1 /*
   2  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2012, 2023 SAP SE. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "code/compiledIC.hpp"
  29 #include "compiler/disassembler.hpp"
  30 #include "gc/shared/collectedHeap.inline.hpp"
  31 #include "gc/shared/barrierSet.hpp"
  32 #include "gc/shared/barrierSetAssembler.hpp"
  33 #include "interpreter/interpreter.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "nativeInst_ppc.hpp"
  36 #include "oops/compressedKlass.inline.hpp"
  37 #include "oops/compressedOops.inline.hpp"
  38 #include "oops/klass.inline.hpp"
  39 #include "oops/methodData.hpp"
  40 #include "prims/methodHandles.hpp"
  41 #include "register_ppc.hpp"
  42 #include "runtime/icache.hpp"
  43 #include "runtime/interfaceSupport.inline.hpp"
  44 #include "runtime/objectMonitor.hpp"
  45 #include "runtime/os.hpp"
  46 #include "runtime/safepoint.hpp"
  47 #include "runtime/safepointMechanism.hpp"
  48 #include "runtime/sharedRuntime.hpp"
  49 #include "runtime/stubRoutines.hpp"
  50 #include "runtime/vm_version.hpp"
  51 #include "utilities/macros.hpp"
  52 #include "utilities/powerOfTwo.hpp"
  53 
  54 #ifdef PRODUCT
  55 #define BLOCK_COMMENT(str) // nothing
  56 #else
  57 #define BLOCK_COMMENT(str) block_comment(str)
  58 #endif
  59 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  60 
  61 #ifdef ASSERT
  62 // On RISC, there's no benefit to verifying instruction boundaries.
  63 bool AbstractAssembler::pd_check_instruction_mark() { return false; }
  64 #endif
  65 
  66 void MacroAssembler::ld_largeoffset_unchecked(Register d, int si31, Register a, int emit_filler_nop) {
  67   assert(Assembler::is_simm(si31, 31) && si31 >= 0, "si31 out of range");
  68   if (Assembler::is_simm(si31, 16)) {
  69     ld(d, si31, a);
  70     if (emit_filler_nop) nop();
  71   } else {
  72     const int hi = MacroAssembler::largeoffset_si16_si16_hi(si31);
  73     const int lo = MacroAssembler::largeoffset_si16_si16_lo(si31);
  74     addis(d, a, hi);
  75     ld(d, lo, d);
  76   }
  77 }
  78 
  79 void MacroAssembler::ld_largeoffset(Register d, int si31, Register a, int emit_filler_nop) {
  80   assert_different_registers(d, a);
  81   ld_largeoffset_unchecked(d, si31, a, emit_filler_nop);
  82 }
  83 
  84 void MacroAssembler::load_sized_value(Register dst, RegisterOrConstant offs, Register base,
  85                                       size_t size_in_bytes, bool is_signed) {
  86   switch (size_in_bytes) {
  87   case  8:              ld(dst, offs, base);                         break;
  88   case  4:  is_signed ? lwa(dst, offs, base) : lwz(dst, offs, base); break;
  89   case  2:  is_signed ? lha(dst, offs, base) : lhz(dst, offs, base); break;
  90   case  1:  lbz(dst, offs, base); if (is_signed) extsb(dst, dst);    break; // lba doesn't exist :(
  91   default:  ShouldNotReachHere();
  92   }
  93 }
  94 
  95 void MacroAssembler::store_sized_value(Register dst, RegisterOrConstant offs, Register base,
  96                                        size_t size_in_bytes) {
  97   switch (size_in_bytes) {
  98   case  8:  std(dst, offs, base); break;
  99   case  4:  stw(dst, offs, base); break;
 100   case  2:  sth(dst, offs, base); break;
 101   case  1:  stb(dst, offs, base); break;
 102   default:  ShouldNotReachHere();
 103   }
 104 }
 105 
 106 void MacroAssembler::align(int modulus, int max, int rem) {
 107   int padding = (rem + modulus - (offset() % modulus)) % modulus;
 108   if (padding > max) return;
 109   for (int c = (padding >> 2); c > 0; --c) { nop(); }
 110 }
 111 
 112 void MacroAssembler::align_prefix() {
 113   if (is_aligned(offset() + BytesPerInstWord, 64)) { nop(); }
 114 }
 115 
 116 // Issue instructions that calculate given TOC from global TOC.
 117 void MacroAssembler::calculate_address_from_global_toc(Register dst, address addr, bool hi16, bool lo16,
 118                                                        bool add_relocation, bool emit_dummy_addr) {
 119   int offset = -1;
 120   if (emit_dummy_addr) {
 121     offset = -128; // dummy address
 122   } else if (addr != (address)(intptr_t)-1) {
 123     offset = MacroAssembler::offset_to_global_toc(addr);
 124   }
 125 
 126   if (hi16) {
 127     addis(dst, R29_TOC, MacroAssembler::largeoffset_si16_si16_hi(offset));
 128   }
 129   if (lo16) {
 130     if (add_relocation) {
 131       // Relocate at the addi to avoid confusion with a load from the method's TOC.
 132       relocate(internal_word_Relocation::spec(addr));
 133     }
 134     addi(dst, dst, MacroAssembler::largeoffset_si16_si16_lo(offset));
 135   }
 136 }
 137 
 138 address MacroAssembler::patch_calculate_address_from_global_toc_at(address a, address bound, address addr) {
 139   const int offset = MacroAssembler::offset_to_global_toc(addr);
 140 
 141   const address inst2_addr = a;
 142   const int inst2 = *(int *)inst2_addr;
 143 
 144   // The relocation points to the second instruction, the addi,
 145   // and the addi reads and writes the same register dst.
 146   const int dst = inv_rt_field(inst2);
 147   assert(is_addi(inst2) && inv_ra_field(inst2) == dst, "must be addi reading and writing dst");
 148 
 149   // Now, find the preceding addis which writes to dst.
 150   int inst1 = 0;
 151   address inst1_addr = inst2_addr - BytesPerInstWord;
 152   while (inst1_addr >= bound) {
 153     inst1 = *(int *) inst1_addr;
 154     if (is_addis(inst1) && inv_rt_field(inst1) == dst) {
 155       // Stop, found the addis which writes dst.
 156       break;
 157     }
 158     inst1_addr -= BytesPerInstWord;
 159   }
 160 
 161   assert(is_addis(inst1) && inv_ra_field(inst1) == 29 /* R29 */, "source must be global TOC");
 162   set_imm((int *)inst1_addr, MacroAssembler::largeoffset_si16_si16_hi(offset));
 163   set_imm((int *)inst2_addr, MacroAssembler::largeoffset_si16_si16_lo(offset));
 164   return inst1_addr;
 165 }
 166 
 167 address MacroAssembler::get_address_of_calculate_address_from_global_toc_at(address a, address bound) {
 168   const address inst2_addr = a;
 169   const int inst2 = *(int *)inst2_addr;
 170 
 171   // The relocation points to the second instruction, the addi,
 172   // and the addi reads and writes the same register dst.
 173   const int dst = inv_rt_field(inst2);
 174   assert(is_addi(inst2) && inv_ra_field(inst2) == dst, "must be addi reading and writing dst");
 175 
 176   // Now, find the preceding addis which writes to dst.
 177   int inst1 = 0;
 178   address inst1_addr = inst2_addr - BytesPerInstWord;
 179   while (inst1_addr >= bound) {
 180     inst1 = *(int *) inst1_addr;
 181     if (is_addis(inst1) && inv_rt_field(inst1) == dst) {
 182       // stop, found the addis which writes dst
 183       break;
 184     }
 185     inst1_addr -= BytesPerInstWord;
 186   }
 187 
 188   assert(is_addis(inst1) && inv_ra_field(inst1) == 29 /* R29 */, "source must be global TOC");
 189 
 190   int offset = (get_imm(inst1_addr, 0) << 16) + get_imm(inst2_addr, 0);
 191   // -1 is a special case
 192   if (offset == -1) {
 193     return (address)(intptr_t)-1;
 194   } else {
 195     return global_toc() + offset;
 196   }
 197 }
 198 
 199 #ifdef _LP64
 200 // Patch compressed oops or klass constants.
 201 // Assembler sequence is
 202 // 1) compressed oops:
 203 //    lis  rx = const.hi
 204 //    ori rx = rx | const.lo
 205 // 2) compressed klass:
 206 //    lis  rx = const.hi
 207 //    clrldi rx = rx & 0xFFFFffff // clearMS32b, optional
 208 //    ori rx = rx | const.lo
 209 // Clrldi will be passed by.
 210 address MacroAssembler::patch_set_narrow_oop(address a, address bound, narrowOop data) {
 211   assert(UseCompressedOops, "Should only patch compressed oops");
 212 
 213   const address inst2_addr = a;
 214   const int inst2 = *(int *)inst2_addr;
 215 
 216   // The relocation points to the second instruction, the ori,
 217   // and the ori reads and writes the same register dst.
 218   const int dst = inv_rta_field(inst2);
 219   assert(is_ori(inst2) && inv_rs_field(inst2) == dst, "must be ori reading and writing dst");
 220   // Now, find the preceding addis which writes to dst.
 221   int inst1 = 0;
 222   address inst1_addr = inst2_addr - BytesPerInstWord;
 223   bool inst1_found = false;
 224   while (inst1_addr >= bound) {
 225     inst1 = *(int *)inst1_addr;
 226     if (is_lis(inst1) && inv_rs_field(inst1) == dst) { inst1_found = true; break; }
 227     inst1_addr -= BytesPerInstWord;
 228   }
 229   assert(inst1_found, "inst is not lis");
 230 
 231   uint32_t data_value = CompressedOops::narrow_oop_value(data);
 232   int xc = (data_value >> 16) & 0xffff;
 233   int xd = (data_value >>  0) & 0xffff;
 234 
 235   set_imm((int *)inst1_addr, (short)(xc)); // see enc_load_con_narrow_hi/_lo
 236   set_imm((int *)inst2_addr,        (xd)); // unsigned int
 237   return inst1_addr;
 238 }
 239 
 240 // Get compressed oop constant.
 241 narrowOop MacroAssembler::get_narrow_oop(address a, address bound) {
 242   assert(UseCompressedOops, "Should only patch compressed oops");
 243 
 244   const address inst2_addr = a;
 245   const int inst2 = *(int *)inst2_addr;
 246 
 247   // The relocation points to the second instruction, the ori,
 248   // and the ori reads and writes the same register dst.
 249   const int dst = inv_rta_field(inst2);
 250   assert(is_ori(inst2) && inv_rs_field(inst2) == dst, "must be ori reading and writing dst");
 251   // Now, find the preceding lis which writes to dst.
 252   int inst1 = 0;
 253   address inst1_addr = inst2_addr - BytesPerInstWord;
 254   bool inst1_found = false;
 255 
 256   while (inst1_addr >= bound) {
 257     inst1 = *(int *) inst1_addr;
 258     if (is_lis(inst1) && inv_rs_field(inst1) == dst) { inst1_found = true; break;}
 259     inst1_addr -= BytesPerInstWord;
 260   }
 261   assert(inst1_found, "inst is not lis");
 262 
 263   uint xl = ((unsigned int) (get_imm(inst2_addr, 0) & 0xffff));
 264   uint xh = (((get_imm(inst1_addr, 0)) & 0xffff) << 16);
 265 
 266   return CompressedOops::narrow_oop_cast(xl | xh);
 267 }
 268 #endif // _LP64
 269 
 270 // Returns true if successful.
 271 bool MacroAssembler::load_const_from_method_toc(Register dst, AddressLiteral& a,
 272                                                 Register toc, bool fixed_size) {
 273   int toc_offset = 0;
 274   // Use RelocationHolder::none for the constant pool entry, otherwise
 275   // we will end up with a failing NativeCall::verify(x) where x is
 276   // the address of the constant pool entry.
 277   // FIXME: We should insert relocation information for oops at the constant
 278   // pool entries instead of inserting it at the loads; patching of a constant
 279   // pool entry should be less expensive.
 280   address const_address = address_constant((address)a.value(), RelocationHolder::none);
 281   if (const_address == nullptr) { return false; } // allocation failure
 282   // Relocate at the pc of the load.
 283   relocate(a.rspec());
 284   toc_offset = (int)(const_address - code()->consts()->start());
 285   ld_largeoffset_unchecked(dst, toc_offset, toc, fixed_size);
 286   return true;
 287 }
 288 
 289 bool MacroAssembler::is_load_const_from_method_toc_at(address a) {
 290   const address inst1_addr = a;
 291   const int inst1 = *(int *)inst1_addr;
 292 
 293    // The relocation points to the ld or the addis.
 294    return (is_ld(inst1)) ||
 295           (is_addis(inst1) && inv_ra_field(inst1) != 0);
 296 }
 297 
 298 int MacroAssembler::get_offset_of_load_const_from_method_toc_at(address a) {
 299   assert(is_load_const_from_method_toc_at(a), "must be load_const_from_method_toc");
 300 
 301   const address inst1_addr = a;
 302   const int inst1 = *(int *)inst1_addr;
 303 
 304   if (is_ld(inst1)) {
 305     return inv_d1_field(inst1);
 306   } else if (is_addis(inst1)) {
 307     const int dst = inv_rt_field(inst1);
 308 
 309     // Now, find the succeeding ld which reads and writes to dst.
 310     address inst2_addr = inst1_addr + BytesPerInstWord;
 311     int inst2 = 0;
 312     while (true) {
 313       inst2 = *(int *) inst2_addr;
 314       if (is_ld(inst2) && inv_ra_field(inst2) == dst && inv_rt_field(inst2) == dst) {
 315         // Stop, found the ld which reads and writes dst.
 316         break;
 317       }
 318       inst2_addr += BytesPerInstWord;
 319     }
 320     return (inv_d1_field(inst1) << 16) + inv_d1_field(inst2);
 321   }
 322   ShouldNotReachHere();
 323   return 0;
 324 }
 325 
 326 // Get the constant from a `load_const' sequence.
 327 long MacroAssembler::get_const(address a) {
 328   assert(is_load_const_at(a), "not a load of a constant");
 329   const int *p = (const int*) a;
 330   unsigned long x = (((unsigned long) (get_imm(a,0) & 0xffff)) << 48);
 331   if (is_ori(*(p+1))) {
 332     x |= (((unsigned long) (get_imm(a,1) & 0xffff)) << 32);
 333     x |= (((unsigned long) (get_imm(a,3) & 0xffff)) << 16);
 334     x |= (((unsigned long) (get_imm(a,4) & 0xffff)));
 335   } else if (is_lis(*(p+1))) {
 336     x |= (((unsigned long) (get_imm(a,2) & 0xffff)) << 32);
 337     x |= (((unsigned long) (get_imm(a,1) & 0xffff)) << 16);
 338     x |= (((unsigned long) (get_imm(a,3) & 0xffff)));
 339   } else {
 340     ShouldNotReachHere();
 341     return (long) 0;
 342   }
 343   return (long) x;
 344 }
 345 
 346 // Patch the 64 bit constant of a `load_const' sequence. This is a low
 347 // level procedure. It neither flushes the instruction cache nor is it
 348 // mt safe.
 349 void MacroAssembler::patch_const(address a, long x) {
 350   assert(is_load_const_at(a), "not a load of a constant");
 351   int *p = (int*) a;
 352   if (is_ori(*(p+1))) {
 353     set_imm(0 + p, (x >> 48) & 0xffff);
 354     set_imm(1 + p, (x >> 32) & 0xffff);
 355     set_imm(3 + p, (x >> 16) & 0xffff);
 356     set_imm(4 + p, x & 0xffff);
 357   } else if (is_lis(*(p+1))) {
 358     set_imm(0 + p, (x >> 48) & 0xffff);
 359     set_imm(2 + p, (x >> 32) & 0xffff);
 360     set_imm(1 + p, (x >> 16) & 0xffff);
 361     set_imm(3 + p, x & 0xffff);
 362   } else {
 363     ShouldNotReachHere();
 364   }
 365 }
 366 
 367 AddressLiteral MacroAssembler::allocate_metadata_address(Metadata* obj) {
 368   assert(oop_recorder() != nullptr, "this assembler needs a Recorder");
 369   int index = oop_recorder()->allocate_metadata_index(obj);
 370   RelocationHolder rspec = metadata_Relocation::spec(index);
 371   return AddressLiteral((address)obj, rspec);
 372 }
 373 
 374 AddressLiteral MacroAssembler::constant_metadata_address(Metadata* obj) {
 375   assert(oop_recorder() != nullptr, "this assembler needs a Recorder");
 376   int index = oop_recorder()->find_index(obj);
 377   RelocationHolder rspec = metadata_Relocation::spec(index);
 378   return AddressLiteral((address)obj, rspec);
 379 }
 380 
 381 AddressLiteral MacroAssembler::allocate_oop_address(jobject obj) {
 382   assert(oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 383   int oop_index = oop_recorder()->allocate_oop_index(obj);
 384   return AddressLiteral(address(obj), oop_Relocation::spec(oop_index));
 385 }
 386 
 387 AddressLiteral MacroAssembler::constant_oop_address(jobject obj) {
 388   assert(oop_recorder() != nullptr, "this assembler needs an OopRecorder");
 389   int oop_index = oop_recorder()->find_index(obj);
 390   return AddressLiteral(address(obj), oop_Relocation::spec(oop_index));
 391 }
 392 
 393 #ifndef PRODUCT
 394 void MacroAssembler::pd_print_patched_instruction(address branch) {
 395   Unimplemented(); // TODO: PPC port
 396 }
 397 #endif // ndef PRODUCT
 398 
 399 // Conditional far branch for destinations encodable in 24+2 bits.
 400 void MacroAssembler::bc_far(int boint, int biint, Label& dest, int optimize) {
 401 
 402   // If requested by flag optimize, relocate the bc_far as a
 403   // runtime_call and prepare for optimizing it when the code gets
 404   // relocated.
 405   if (optimize == bc_far_optimize_on_relocate) {
 406     relocate(relocInfo::runtime_call_type);
 407   }
 408 
 409   // variant 2:
 410   //
 411   //    b!cxx SKIP
 412   //    bxx   DEST
 413   //  SKIP:
 414   //
 415 
 416   const int opposite_boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(boint)),
 417                                                 opposite_bcond(inv_boint_bcond(boint)));
 418 
 419   // We emit two branches.
 420   // First, a conditional branch which jumps around the far branch.
 421   const address not_taken_pc = pc() + 2 * BytesPerInstWord;
 422   const address bc_pc        = pc();
 423   bc(opposite_boint, biint, not_taken_pc);
 424 
 425   const int bc_instr = *(int*)bc_pc;
 426   assert(not_taken_pc == (address)inv_bd_field(bc_instr, (intptr_t)bc_pc), "postcondition");
 427   assert(opposite_boint == inv_bo_field(bc_instr), "postcondition");
 428   assert(boint == add_bhint_to_boint(opposite_bhint(inv_boint_bhint(inv_bo_field(bc_instr))),
 429                                      opposite_bcond(inv_boint_bcond(inv_bo_field(bc_instr)))),
 430          "postcondition");
 431   assert(biint == inv_bi_field(bc_instr), "postcondition");
 432 
 433   // Second, an unconditional far branch which jumps to dest.
 434   // Note: target(dest) remembers the current pc (see CodeSection::target)
 435   //       and returns the current pc if the label is not bound yet; when
 436   //       the label gets bound, the unconditional far branch will be patched.
 437   const address target_pc = target(dest);
 438   const address b_pc  = pc();
 439   b(target_pc);
 440 
 441   assert(not_taken_pc == pc(),                     "postcondition");
 442   assert(dest.is_bound() || target_pc == b_pc, "postcondition");
 443 }
 444 
 445 // 1 or 2 instructions
 446 void MacroAssembler::bc_far_optimized(int boint, int biint, Label& dest) {
 447   if (dest.is_bound() && is_within_range_of_bcxx(target(dest), pc())) {
 448     bc(boint, biint, dest);
 449   } else {
 450     bc_far(boint, biint, dest, MacroAssembler::bc_far_optimize_on_relocate);
 451   }
 452 }
 453 
 454 bool MacroAssembler::is_bc_far_at(address instruction_addr) {
 455   return is_bc_far_variant1_at(instruction_addr) ||
 456          is_bc_far_variant2_at(instruction_addr) ||
 457          is_bc_far_variant3_at(instruction_addr);
 458 }
 459 
 460 address MacroAssembler::get_dest_of_bc_far_at(address instruction_addr) {
 461   if (is_bc_far_variant1_at(instruction_addr)) {
 462     const address instruction_1_addr = instruction_addr;
 463     const int instruction_1 = *(int*)instruction_1_addr;
 464     return (address)inv_bd_field(instruction_1, (intptr_t)instruction_1_addr);
 465   } else if (is_bc_far_variant2_at(instruction_addr)) {
 466     const address instruction_2_addr = instruction_addr + 4;
 467     return bxx_destination(instruction_2_addr);
 468   } else if (is_bc_far_variant3_at(instruction_addr)) {
 469     return instruction_addr + 8;
 470   }
 471   // variant 4 ???
 472   ShouldNotReachHere();
 473   return nullptr;
 474 }
 475 void MacroAssembler::set_dest_of_bc_far_at(address instruction_addr, address dest) {
 476 
 477   if (is_bc_far_variant3_at(instruction_addr)) {
 478     // variant 3, far cond branch to the next instruction, already patched to nops:
 479     //
 480     //    nop
 481     //    endgroup
 482     //  SKIP/DEST:
 483     //
 484     return;
 485   }
 486 
 487   // first, extract boint and biint from the current branch
 488   int boint = 0;
 489   int biint = 0;
 490 
 491   ResourceMark rm;
 492   const int code_size = 2 * BytesPerInstWord;
 493   CodeBuffer buf(instruction_addr, code_size);
 494   MacroAssembler masm(&buf);
 495   if (is_bc_far_variant2_at(instruction_addr) && dest == instruction_addr + 8) {
 496     // Far branch to next instruction: Optimize it by patching nops (produce variant 3).
 497     masm.nop();
 498     masm.endgroup();
 499   } else {
 500     if (is_bc_far_variant1_at(instruction_addr)) {
 501       // variant 1, the 1st instruction contains the destination address:
 502       //
 503       //    bcxx  DEST
 504       //    nop
 505       //
 506       const int instruction_1 = *(int*)(instruction_addr);
 507       boint = inv_bo_field(instruction_1);
 508       biint = inv_bi_field(instruction_1);
 509     } else if (is_bc_far_variant2_at(instruction_addr)) {
 510       // variant 2, the 2nd instruction contains the destination address:
 511       //
 512       //    b!cxx SKIP
 513       //    bxx   DEST
 514       //  SKIP:
 515       //
 516       const int instruction_1 = *(int*)(instruction_addr);
 517       boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(inv_bo_field(instruction_1))),
 518           opposite_bcond(inv_boint_bcond(inv_bo_field(instruction_1))));
 519       biint = inv_bi_field(instruction_1);
 520     } else {
 521       // variant 4???
 522       ShouldNotReachHere();
 523     }
 524 
 525     // second, set the new branch destination and optimize the code
 526     if (dest != instruction_addr + 4 && // the bc_far is still unbound!
 527         masm.is_within_range_of_bcxx(dest, instruction_addr)) {
 528       // variant 1:
 529       //
 530       //    bcxx  DEST
 531       //    nop
 532       //
 533       masm.bc(boint, biint, dest);
 534       masm.nop();
 535     } else {
 536       // variant 2:
 537       //
 538       //    b!cxx SKIP
 539       //    bxx   DEST
 540       //  SKIP:
 541       //
 542       const int opposite_boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(boint)),
 543                                                     opposite_bcond(inv_boint_bcond(boint)));
 544       const address not_taken_pc = masm.pc() + 2 * BytesPerInstWord;
 545       masm.bc(opposite_boint, biint, not_taken_pc);
 546       masm.b(dest);
 547     }
 548   }
 549   ICache::ppc64_flush_icache_bytes(instruction_addr, code_size);
 550 }
 551 
 552 // Emit a NOT mt-safe patchable 64 bit absolute call/jump.
 553 void MacroAssembler::bxx64_patchable(address dest, relocInfo::relocType rt, bool link) {
 554   // get current pc
 555   uint64_t start_pc = (uint64_t) pc();
 556 
 557   const address pc_of_bl = (address) (start_pc + (6*BytesPerInstWord)); // bl is last
 558   const address pc_of_b  = (address) (start_pc + (0*BytesPerInstWord)); // b is first
 559 
 560   // relocate here
 561   if (rt != relocInfo::none) {
 562     relocate(rt);
 563   }
 564 
 565   if ( ReoptimizeCallSequences &&
 566        (( link && is_within_range_of_b(dest, pc_of_bl)) ||
 567         (!link && is_within_range_of_b(dest, pc_of_b)))) {
 568     // variant 2:
 569     // Emit an optimized, pc-relative call/jump.
 570 
 571     if (link) {
 572       // some padding
 573       nop();
 574       nop();
 575       nop();
 576       nop();
 577       nop();
 578       nop();
 579 
 580       // do the call
 581       assert(pc() == pc_of_bl, "just checking");
 582       bl(dest, relocInfo::none);
 583     } else {
 584       // do the jump
 585       assert(pc() == pc_of_b, "just checking");
 586       b(dest, relocInfo::none);
 587 
 588       // some padding
 589       nop();
 590       nop();
 591       nop();
 592       nop();
 593       nop();
 594       nop();
 595     }
 596 
 597     // Assert that we can identify the emitted call/jump.
 598     assert(is_bxx64_patchable_variant2_at((address)start_pc, link),
 599            "can't identify emitted call");
 600   } else {
 601     // variant 1:
 602     mr(R0, R11);  // spill R11 -> R0.
 603 
 604     // Load the destination address into CTR,
 605     // calculate destination relative to global toc.
 606     calculate_address_from_global_toc(R11, dest, true, true, false);
 607 
 608     mtctr(R11);
 609     mr(R11, R0);  // spill R11 <- R0.
 610     nop();
 611 
 612     // do the call/jump
 613     if (link) {
 614       bctrl();
 615     } else{
 616       bctr();
 617     }
 618     // Assert that we can identify the emitted call/jump.
 619     assert(is_bxx64_patchable_variant1b_at((address)start_pc, link),
 620            "can't identify emitted call");
 621   }
 622 
 623   // Assert that we can identify the emitted call/jump.
 624   assert(is_bxx64_patchable_at((address)start_pc, link),
 625          "can't identify emitted call");
 626   assert(get_dest_of_bxx64_patchable_at((address)start_pc, link) == dest,
 627          "wrong encoding of dest address");
 628 }
 629 
 630 // Identify a bxx64_patchable instruction.
 631 bool MacroAssembler::is_bxx64_patchable_at(address instruction_addr, bool link) {
 632   return is_bxx64_patchable_variant1b_at(instruction_addr, link)
 633     //|| is_bxx64_patchable_variant1_at(instruction_addr, link)
 634       || is_bxx64_patchable_variant2_at(instruction_addr, link);
 635 }
 636 
 637 // Does the call64_patchable instruction use a pc-relative encoding of
 638 // the call destination?
 639 bool MacroAssembler::is_bxx64_patchable_pcrelative_at(address instruction_addr, bool link) {
 640   // variant 2 is pc-relative
 641   return is_bxx64_patchable_variant2_at(instruction_addr, link);
 642 }
 643 
 644 // Identify variant 1.
 645 bool MacroAssembler::is_bxx64_patchable_variant1_at(address instruction_addr, bool link) {
 646   unsigned int* instr = (unsigned int*) instruction_addr;
 647   return (link ? is_bctrl(instr[6]) : is_bctr(instr[6])) // bctr[l]
 648       && is_mtctr(instr[5]) // mtctr
 649     && is_load_const_at(instruction_addr);
 650 }
 651 
 652 // Identify variant 1b: load destination relative to global toc.
 653 bool MacroAssembler::is_bxx64_patchable_variant1b_at(address instruction_addr, bool link) {
 654   unsigned int* instr = (unsigned int*) instruction_addr;
 655   return (link ? is_bctrl(instr[6]) : is_bctr(instr[6])) // bctr[l]
 656     && is_mtctr(instr[3]) // mtctr
 657     && is_calculate_address_from_global_toc_at(instruction_addr + 2*BytesPerInstWord, instruction_addr);
 658 }
 659 
 660 // Identify variant 2.
 661 bool MacroAssembler::is_bxx64_patchable_variant2_at(address instruction_addr, bool link) {
 662   unsigned int* instr = (unsigned int*) instruction_addr;
 663   if (link) {
 664     return is_bl (instr[6])  // bl dest is last
 665       && is_nop(instr[0])  // nop
 666       && is_nop(instr[1])  // nop
 667       && is_nop(instr[2])  // nop
 668       && is_nop(instr[3])  // nop
 669       && is_nop(instr[4])  // nop
 670       && is_nop(instr[5]); // nop
 671   } else {
 672     return is_b  (instr[0])  // b  dest is first
 673       && is_nop(instr[1])  // nop
 674       && is_nop(instr[2])  // nop
 675       && is_nop(instr[3])  // nop
 676       && is_nop(instr[4])  // nop
 677       && is_nop(instr[5])  // nop
 678       && is_nop(instr[6]); // nop
 679   }
 680 }
 681 
 682 // Set dest address of a bxx64_patchable instruction.
 683 void MacroAssembler::set_dest_of_bxx64_patchable_at(address instruction_addr, address dest, bool link) {
 684   ResourceMark rm;
 685   int code_size = MacroAssembler::bxx64_patchable_size;
 686   CodeBuffer buf(instruction_addr, code_size);
 687   MacroAssembler masm(&buf);
 688   masm.bxx64_patchable(dest, relocInfo::none, link);
 689   ICache::ppc64_flush_icache_bytes(instruction_addr, code_size);
 690 }
 691 
 692 // Get dest address of a bxx64_patchable instruction.
 693 address MacroAssembler::get_dest_of_bxx64_patchable_at(address instruction_addr, bool link) {
 694   if (is_bxx64_patchable_variant1_at(instruction_addr, link)) {
 695     return (address) (unsigned long) get_const(instruction_addr);
 696   } else if (is_bxx64_patchable_variant2_at(instruction_addr, link)) {
 697     unsigned int* instr = (unsigned int*) instruction_addr;
 698     if (link) {
 699       const int instr_idx = 6; // bl is last
 700       int branchoffset = branch_destination(instr[instr_idx], 0);
 701       return instruction_addr + branchoffset + instr_idx*BytesPerInstWord;
 702     } else {
 703       const int instr_idx = 0; // b is first
 704       int branchoffset = branch_destination(instr[instr_idx], 0);
 705       return instruction_addr + branchoffset + instr_idx*BytesPerInstWord;
 706     }
 707   // Load dest relative to global toc.
 708   } else if (is_bxx64_patchable_variant1b_at(instruction_addr, link)) {
 709     return get_address_of_calculate_address_from_global_toc_at(instruction_addr + 2*BytesPerInstWord,
 710                                                                instruction_addr);
 711   } else {
 712     ShouldNotReachHere();
 713     return nullptr;
 714   }
 715 }
 716 
 717 void MacroAssembler::clobber_volatile_gprs(Register excluded_register) {
 718   const int magic_number = 0x42;
 719 
 720   // Preserve stack pointer register (R1_SP) and system thread id register (R13);
 721   // although they're technically volatile
 722   for (int i = 2; i < 13; i++) {
 723     Register reg = as_Register(i);
 724     if (reg == excluded_register) {
 725       continue;
 726     }
 727 
 728     li(reg, magic_number);
 729   }
 730 }
 731 
 732 void MacroAssembler::clobber_carg_stack_slots(Register tmp) {
 733   const int magic_number = 0x43;
 734 
 735   li(tmp, magic_number);
 736   for (int m = 0; m <= 7; m++) {
 737     std(tmp, frame::native_abi_minframe_size + m * 8, R1_SP);
 738   }
 739 }
 740 
 741 // Uses ordering which corresponds to ABI:
 742 //    _savegpr0_14:  std  r14,-144(r1)
 743 //    _savegpr0_15:  std  r15,-136(r1)
 744 //    _savegpr0_16:  std  r16,-128(r1)
 745 void MacroAssembler::save_nonvolatile_gprs(Register dst, int offset) {
 746   std(R14, offset, dst);   offset += 8;
 747   std(R15, offset, dst);   offset += 8;
 748   std(R16, offset, dst);   offset += 8;
 749   std(R17, offset, dst);   offset += 8;
 750   std(R18, offset, dst);   offset += 8;
 751   std(R19, offset, dst);   offset += 8;
 752   std(R20, offset, dst);   offset += 8;
 753   std(R21, offset, dst);   offset += 8;
 754   std(R22, offset, dst);   offset += 8;
 755   std(R23, offset, dst);   offset += 8;
 756   std(R24, offset, dst);   offset += 8;
 757   std(R25, offset, dst);   offset += 8;
 758   std(R26, offset, dst);   offset += 8;
 759   std(R27, offset, dst);   offset += 8;
 760   std(R28, offset, dst);   offset += 8;
 761   std(R29, offset, dst);   offset += 8;
 762   std(R30, offset, dst);   offset += 8;
 763   std(R31, offset, dst);   offset += 8;
 764 
 765   stfd(F14, offset, dst);   offset += 8;
 766   stfd(F15, offset, dst);   offset += 8;
 767   stfd(F16, offset, dst);   offset += 8;
 768   stfd(F17, offset, dst);   offset += 8;
 769   stfd(F18, offset, dst);   offset += 8;
 770   stfd(F19, offset, dst);   offset += 8;
 771   stfd(F20, offset, dst);   offset += 8;
 772   stfd(F21, offset, dst);   offset += 8;
 773   stfd(F22, offset, dst);   offset += 8;
 774   stfd(F23, offset, dst);   offset += 8;
 775   stfd(F24, offset, dst);   offset += 8;
 776   stfd(F25, offset, dst);   offset += 8;
 777   stfd(F26, offset, dst);   offset += 8;
 778   stfd(F27, offset, dst);   offset += 8;
 779   stfd(F28, offset, dst);   offset += 8;
 780   stfd(F29, offset, dst);   offset += 8;
 781   stfd(F30, offset, dst);   offset += 8;
 782   stfd(F31, offset, dst);
 783 }
 784 
 785 // Uses ordering which corresponds to ABI:
 786 //    _restgpr0_14:  ld   r14,-144(r1)
 787 //    _restgpr0_15:  ld   r15,-136(r1)
 788 //    _restgpr0_16:  ld   r16,-128(r1)
 789 void MacroAssembler::restore_nonvolatile_gprs(Register src, int offset) {
 790   ld(R14, offset, src);   offset += 8;
 791   ld(R15, offset, src);   offset += 8;
 792   ld(R16, offset, src);   offset += 8;
 793   ld(R17, offset, src);   offset += 8;
 794   ld(R18, offset, src);   offset += 8;
 795   ld(R19, offset, src);   offset += 8;
 796   ld(R20, offset, src);   offset += 8;
 797   ld(R21, offset, src);   offset += 8;
 798   ld(R22, offset, src);   offset += 8;
 799   ld(R23, offset, src);   offset += 8;
 800   ld(R24, offset, src);   offset += 8;
 801   ld(R25, offset, src);   offset += 8;
 802   ld(R26, offset, src);   offset += 8;
 803   ld(R27, offset, src);   offset += 8;
 804   ld(R28, offset, src);   offset += 8;
 805   ld(R29, offset, src);   offset += 8;
 806   ld(R30, offset, src);   offset += 8;
 807   ld(R31, offset, src);   offset += 8;
 808 
 809   // FP registers
 810   lfd(F14, offset, src);   offset += 8;
 811   lfd(F15, offset, src);   offset += 8;
 812   lfd(F16, offset, src);   offset += 8;
 813   lfd(F17, offset, src);   offset += 8;
 814   lfd(F18, offset, src);   offset += 8;
 815   lfd(F19, offset, src);   offset += 8;
 816   lfd(F20, offset, src);   offset += 8;
 817   lfd(F21, offset, src);   offset += 8;
 818   lfd(F22, offset, src);   offset += 8;
 819   lfd(F23, offset, src);   offset += 8;
 820   lfd(F24, offset, src);   offset += 8;
 821   lfd(F25, offset, src);   offset += 8;
 822   lfd(F26, offset, src);   offset += 8;
 823   lfd(F27, offset, src);   offset += 8;
 824   lfd(F28, offset, src);   offset += 8;
 825   lfd(F29, offset, src);   offset += 8;
 826   lfd(F30, offset, src);   offset += 8;
 827   lfd(F31, offset, src);
 828 }
 829 
 830 // For verify_oops.
 831 void MacroAssembler::save_volatile_gprs(Register dst, int offset, bool include_fp_regs, bool include_R3_RET_reg) {
 832   std(R2,  offset, dst);   offset += 8;
 833   if (include_R3_RET_reg) {
 834     std(R3, offset, dst);  offset += 8;
 835   }
 836   std(R4,  offset, dst);   offset += 8;
 837   std(R5,  offset, dst);   offset += 8;
 838   std(R6,  offset, dst);   offset += 8;
 839   std(R7,  offset, dst);   offset += 8;
 840   std(R8,  offset, dst);   offset += 8;
 841   std(R9,  offset, dst);   offset += 8;
 842   std(R10, offset, dst);   offset += 8;
 843   std(R11, offset, dst);   offset += 8;
 844   std(R12, offset, dst);   offset += 8;
 845 
 846   if (include_fp_regs) {
 847     stfd(F0, offset, dst);   offset += 8;
 848     stfd(F1, offset, dst);   offset += 8;
 849     stfd(F2, offset, dst);   offset += 8;
 850     stfd(F3, offset, dst);   offset += 8;
 851     stfd(F4, offset, dst);   offset += 8;
 852     stfd(F5, offset, dst);   offset += 8;
 853     stfd(F6, offset, dst);   offset += 8;
 854     stfd(F7, offset, dst);   offset += 8;
 855     stfd(F8, offset, dst);   offset += 8;
 856     stfd(F9, offset, dst);   offset += 8;
 857     stfd(F10, offset, dst);  offset += 8;
 858     stfd(F11, offset, dst);  offset += 8;
 859     stfd(F12, offset, dst);  offset += 8;
 860     stfd(F13, offset, dst);
 861   }
 862 }
 863 
 864 // For verify_oops.
 865 void MacroAssembler::restore_volatile_gprs(Register src, int offset, bool include_fp_regs, bool include_R3_RET_reg) {
 866   ld(R2,  offset, src);   offset += 8;
 867   if (include_R3_RET_reg) {
 868     ld(R3,  offset, src);   offset += 8;
 869   }
 870   ld(R4,  offset, src);   offset += 8;
 871   ld(R5,  offset, src);   offset += 8;
 872   ld(R6,  offset, src);   offset += 8;
 873   ld(R7,  offset, src);   offset += 8;
 874   ld(R8,  offset, src);   offset += 8;
 875   ld(R9,  offset, src);   offset += 8;
 876   ld(R10, offset, src);   offset += 8;
 877   ld(R11, offset, src);   offset += 8;
 878   ld(R12, offset, src);   offset += 8;
 879 
 880   if (include_fp_regs) {
 881     lfd(F0, offset, src);   offset += 8;
 882     lfd(F1, offset, src);   offset += 8;
 883     lfd(F2, offset, src);   offset += 8;
 884     lfd(F3, offset, src);   offset += 8;
 885     lfd(F4, offset, src);   offset += 8;
 886     lfd(F5, offset, src);   offset += 8;
 887     lfd(F6, offset, src);   offset += 8;
 888     lfd(F7, offset, src);   offset += 8;
 889     lfd(F8, offset, src);   offset += 8;
 890     lfd(F9, offset, src);   offset += 8;
 891     lfd(F10, offset, src);  offset += 8;
 892     lfd(F11, offset, src);  offset += 8;
 893     lfd(F12, offset, src);  offset += 8;
 894     lfd(F13, offset, src);
 895   }
 896 }
 897 
 898 void MacroAssembler::save_LR_CR(Register tmp) {
 899   mfcr(tmp);
 900   std(tmp, _abi0(cr), R1_SP);
 901   mflr(tmp);
 902   std(tmp, _abi0(lr), R1_SP);
 903   // Tmp must contain lr on exit! (see return_addr and prolog in ppc64.ad)
 904 }
 905 
 906 void MacroAssembler::restore_LR_CR(Register tmp) {
 907   assert(tmp != R1_SP, "must be distinct");
 908   ld(tmp, _abi0(lr), R1_SP);
 909   mtlr(tmp);
 910   ld(tmp, _abi0(cr), R1_SP);
 911   mtcr(tmp);
 912 }
 913 
 914 address MacroAssembler::get_PC_trash_LR(Register result) {
 915   Label L;
 916   bl(L);
 917   bind(L);
 918   address lr_pc = pc();
 919   mflr(result);
 920   return lr_pc;
 921 }
 922 
 923 void MacroAssembler::resize_frame(Register offset, Register tmp) {
 924 #ifdef ASSERT
 925   assert_different_registers(offset, tmp, R1_SP);
 926   andi_(tmp, offset, frame::alignment_in_bytes-1);
 927   asm_assert_eq("resize_frame: unaligned");
 928 #endif
 929 
 930   // tmp <- *(SP)
 931   ld(tmp, _abi0(callers_sp), R1_SP);
 932   // addr <- SP + offset;
 933   // *(addr) <- tmp;
 934   // SP <- addr
 935   stdux(tmp, R1_SP, offset);
 936 }
 937 
 938 void MacroAssembler::resize_frame(int offset, Register tmp) {
 939   assert(is_simm(offset, 16), "too big an offset");
 940   assert_different_registers(tmp, R1_SP);
 941   assert((offset & (frame::alignment_in_bytes-1))==0, "resize_frame: unaligned");
 942   // tmp <- *(SP)
 943   ld(tmp, _abi0(callers_sp), R1_SP);
 944   // addr <- SP + offset;
 945   // *(addr) <- tmp;
 946   // SP <- addr
 947   stdu(tmp, offset, R1_SP);
 948 }
 949 
 950 void MacroAssembler::resize_frame_absolute(Register addr, Register tmp1, Register tmp2) {
 951   // (addr == tmp1) || (addr == tmp2) is allowed here!
 952   assert(tmp1 != tmp2, "must be distinct");
 953 
 954   // compute offset w.r.t. current stack pointer
 955   // tmp_1 <- addr - SP (!)
 956   subf(tmp1, R1_SP, addr);
 957 
 958   // atomically update SP keeping back link.
 959   resize_frame(tmp1/* offset */, tmp2/* tmp */);
 960 }
 961 
 962 void MacroAssembler::push_frame(Register bytes, Register tmp) {
 963 #ifdef ASSERT
 964   assert(bytes != R0, "r0 not allowed here");
 965   andi_(R0, bytes, frame::alignment_in_bytes-1);
 966   asm_assert_eq("push_frame(Reg, Reg): unaligned");
 967 #endif
 968   neg(tmp, bytes);
 969   stdux(R1_SP, R1_SP, tmp);
 970 }
 971 
 972 // Push a frame of size `bytes'.
 973 void MacroAssembler::push_frame(unsigned int bytes, Register tmp) {
 974   long offset = align_addr(bytes, frame::alignment_in_bytes);
 975   if (is_simm(-offset, 16)) {
 976     stdu(R1_SP, -offset, R1_SP);
 977   } else {
 978     load_const_optimized(tmp, -offset);
 979     stdux(R1_SP, R1_SP, tmp);
 980   }
 981 }
 982 
 983 // Push a frame of size `bytes' plus native_abi_reg_args on top.
 984 void MacroAssembler::push_frame_reg_args(unsigned int bytes, Register tmp) {
 985   push_frame(bytes + frame::native_abi_reg_args_size, tmp);
 986 }
 987 
 988 // Setup up a new C frame with a spill area for non-volatile GPRs and
 989 // additional space for local variables.
 990 void MacroAssembler::push_frame_reg_args_nonvolatiles(unsigned int bytes,
 991                                                       Register tmp) {
 992   push_frame(bytes + frame::native_abi_reg_args_size + frame::spill_nonvolatiles_size, tmp);
 993 }
 994 
 995 // Pop current C frame.
 996 void MacroAssembler::pop_frame() {
 997   ld(R1_SP, _abi0(callers_sp), R1_SP);
 998 }
 999 
1000 #if defined(ABI_ELFv2)
1001 address MacroAssembler::branch_to(Register r_function_entry, bool and_link) {
1002   // TODO(asmundak): make sure the caller uses R12 as function descriptor
1003   // most of the times.
1004   if (R12 != r_function_entry) {
1005     mr(R12, r_function_entry);
1006   }
1007   mtctr(R12);
1008   // Do a call or a branch.
1009   if (and_link) {
1010     bctrl();
1011   } else {
1012     bctr();
1013   }
1014   _last_calls_return_pc = pc();
1015 
1016   return _last_calls_return_pc;
1017 }
1018 
1019 // Call a C function via a function descriptor and use full C
1020 // calling conventions. Updates and returns _last_calls_return_pc.
1021 address MacroAssembler::call_c(Register r_function_entry) {
1022   return branch_to(r_function_entry, /*and_link=*/true);
1023 }
1024 
1025 // For tail calls: only branch, don't link, so callee returns to caller of this function.
1026 address MacroAssembler::call_c_and_return_to_caller(Register r_function_entry) {
1027   return branch_to(r_function_entry, /*and_link=*/false);
1028 }
1029 
1030 address MacroAssembler::call_c(address function_entry, relocInfo::relocType rt) {
1031   load_const(R12, function_entry, R0);
1032   return branch_to(R12,  /*and_link=*/true);
1033 }
1034 
1035 #else
1036 // Generic version of a call to C function via a function descriptor
1037 // with variable support for C calling conventions (TOC, ENV, etc.).
1038 // Updates and returns _last_calls_return_pc.
1039 address MacroAssembler::branch_to(Register function_descriptor, bool and_link, bool save_toc_before_call,
1040                                   bool restore_toc_after_call, bool load_toc_of_callee, bool load_env_of_callee) {
1041   // we emit standard ptrgl glue code here
1042   assert((function_descriptor != R0), "function_descriptor cannot be R0");
1043 
1044   // retrieve necessary entries from the function descriptor
1045   ld(R0, in_bytes(FunctionDescriptor::entry_offset()), function_descriptor);
1046   mtctr(R0);
1047 
1048   if (load_toc_of_callee) {
1049     ld(R2_TOC, in_bytes(FunctionDescriptor::toc_offset()), function_descriptor);
1050   }
1051   if (load_env_of_callee) {
1052     ld(R11, in_bytes(FunctionDescriptor::env_offset()), function_descriptor);
1053   } else if (load_toc_of_callee) {
1054     li(R11, 0);
1055   }
1056 
1057   // do a call or a branch
1058   if (and_link) {
1059     bctrl();
1060   } else {
1061     bctr();
1062   }
1063   _last_calls_return_pc = pc();
1064 
1065   return _last_calls_return_pc;
1066 }
1067 
1068 // Call a C function via a function descriptor and use full C calling
1069 // conventions.
1070 // We don't use the TOC in generated code, so there is no need to save
1071 // and restore its value.
1072 address MacroAssembler::call_c(Register fd) {
1073   return branch_to(fd, /*and_link=*/true,
1074                        /*save toc=*/false,
1075                        /*restore toc=*/false,
1076                        /*load toc=*/true,
1077                        /*load env=*/true);
1078 }
1079 
1080 address MacroAssembler::call_c_and_return_to_caller(Register fd) {
1081   return branch_to(fd, /*and_link=*/false,
1082                        /*save toc=*/false,
1083                        /*restore toc=*/false,
1084                        /*load toc=*/true,
1085                        /*load env=*/true);
1086 }
1087 
1088 address MacroAssembler::call_c(const FunctionDescriptor* fd, relocInfo::relocType rt) {
1089   if (rt != relocInfo::none) {
1090     // this call needs to be relocatable
1091     if (!ReoptimizeCallSequences
1092         || (rt != relocInfo::runtime_call_type && rt != relocInfo::none)
1093         || fd == nullptr   // support code-size estimation
1094         || !fd->is_friend_function()
1095         || fd->entry() == nullptr) {
1096       // it's not a friend function as defined by class FunctionDescriptor,
1097       // so do a full call-c here.
1098       load_const(R11, (address)fd, R0);
1099 
1100       bool has_env = (fd != nullptr && fd->env() != nullptr);
1101       return branch_to(R11, /*and_link=*/true,
1102                             /*save toc=*/false,
1103                             /*restore toc=*/false,
1104                             /*load toc=*/true,
1105                             /*load env=*/has_env);
1106     } else {
1107       // It's a friend function. Load the entry point and don't care about
1108       // toc and env. Use an optimizable call instruction, but ensure the
1109       // same code-size as in the case of a non-friend function.
1110       nop();
1111       nop();
1112       nop();
1113       bl64_patchable(fd->entry(), rt);
1114       _last_calls_return_pc = pc();
1115       return _last_calls_return_pc;
1116     }
1117   } else {
1118     // This call does not need to be relocatable, do more aggressive
1119     // optimizations.
1120     if (!ReoptimizeCallSequences
1121       || !fd->is_friend_function()) {
1122       // It's not a friend function as defined by class FunctionDescriptor,
1123       // so do a full call-c here.
1124       load_const(R11, (address)fd, R0);
1125       return branch_to(R11, /*and_link=*/true,
1126                             /*save toc=*/false,
1127                             /*restore toc=*/false,
1128                             /*load toc=*/true,
1129                             /*load env=*/true);
1130     } else {
1131       // it's a friend function, load the entry point and don't care about
1132       // toc and env.
1133       address dest = fd->entry();
1134       if (is_within_range_of_b(dest, pc())) {
1135         bl(dest);
1136       } else {
1137         bl64_patchable(dest, rt);
1138       }
1139       _last_calls_return_pc = pc();
1140       return _last_calls_return_pc;
1141     }
1142   }
1143 }
1144 
1145 // Call a C function.  All constants needed reside in TOC.
1146 //
1147 // Read the address to call from the TOC.
1148 // Read env from TOC, if fd specifies an env.
1149 // Read new TOC from TOC.
1150 address MacroAssembler::call_c_using_toc(const FunctionDescriptor* fd,
1151                                          relocInfo::relocType rt, Register toc) {
1152   if (!ReoptimizeCallSequences
1153     || (rt != relocInfo::runtime_call_type && rt != relocInfo::none)
1154     || !fd->is_friend_function()) {
1155     // It's not a friend function as defined by class FunctionDescriptor,
1156     // so do a full call-c here.
1157     assert(fd->entry() != nullptr, "function must be linked");
1158 
1159     AddressLiteral fd_entry(fd->entry());
1160     bool success = load_const_from_method_toc(R11, fd_entry, toc, /*fixed_size*/ true);
1161     mtctr(R11);
1162     if (fd->env() == nullptr) {
1163       li(R11, 0);
1164       nop();
1165     } else {
1166       AddressLiteral fd_env(fd->env());
1167       success = success && load_const_from_method_toc(R11, fd_env, toc, /*fixed_size*/ true);
1168     }
1169     AddressLiteral fd_toc(fd->toc());
1170     // Set R2_TOC (load from toc)
1171     success = success && load_const_from_method_toc(R2_TOC, fd_toc, toc, /*fixed_size*/ true);
1172     bctrl();
1173     _last_calls_return_pc = pc();
1174     if (!success) { return nullptr; }
1175   } else {
1176     // It's a friend function, load the entry point and don't care about
1177     // toc and env. Use an optimizable call instruction, but ensure the
1178     // same code-size as in the case of a non-friend function.
1179     nop();
1180     bl64_patchable(fd->entry(), rt);
1181     _last_calls_return_pc = pc();
1182   }
1183   return _last_calls_return_pc;
1184 }
1185 #endif // ABI_ELFv2
1186 
1187 void MacroAssembler::post_call_nop() {
1188   // Make inline again when loom is always enabled.
1189   if (!Continuations::enabled()) {
1190     return;
1191   }
1192   // We use CMPI/CMPLI instructions to encode post call nops.
1193   // Refer to NativePostCallNop for details.
1194   relocate(post_call_nop_Relocation::spec());
1195   InlineSkippedInstructionsCounter skipCounter(this);
1196   Assembler::emit_int32(Assembler::CMPLI_OPCODE | Assembler::opp_u_field(1, 9, 9));
1197   assert(is_post_call_nop(*(int*)(pc() - 4)), "post call not not found");
1198 }
1199 
1200 int MacroAssembler::ic_check_size() {
1201   bool implicit_null_checks_available = ImplicitNullChecks && os::zero_page_read_protected(),
1202        use_fast_receiver_null_check   = implicit_null_checks_available || TrapBasedNullChecks,
1203        use_trap_based_null_check      = !implicit_null_checks_available && TrapBasedNullChecks;
1204 
1205   int num_ins;
1206   if (use_fast_receiver_null_check && TrapBasedICMissChecks) {
1207     num_ins = 3;
1208     if (use_trap_based_null_check) num_ins += 1;
1209   } else {
1210     num_ins = 7;
1211     if (!implicit_null_checks_available) num_ins += 2;
1212   }
1213   return num_ins * BytesPerInstWord;
1214 }
1215 
1216 int MacroAssembler::ic_check(int end_alignment) {
1217   bool implicit_null_checks_available = ImplicitNullChecks && os::zero_page_read_protected(),
1218        use_fast_receiver_null_check   = implicit_null_checks_available || TrapBasedNullChecks,
1219        use_trap_based_null_check      = !implicit_null_checks_available && TrapBasedNullChecks;
1220 
1221   Register receiver = R3_ARG1;
1222   Register data = R19_inline_cache_reg;
1223   Register tmp1 = R11_scratch1;
1224   Register tmp2 = R12_scratch2;
1225 
1226   // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed
1227   // before the inline cache check, so we don't have to execute any nop instructions when dispatching
1228   // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align
1229   // before the inline cache check here, and not after
1230   align(end_alignment, end_alignment, end_alignment - ic_check_size());
1231 
1232   int uep_offset = offset();
1233 
1234   if (use_fast_receiver_null_check && TrapBasedICMissChecks) {
1235     // Fast version which uses SIGTRAP
1236 
1237     if (use_trap_based_null_check) {
1238       trap_null_check(receiver);
1239     }
1240     if (UseCompressedClassPointers) {
1241       lwz(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
1242     } else {
1243       ld(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
1244     }
1245     ld(tmp2, in_bytes(CompiledICData::speculated_klass_offset()), data);
1246     trap_ic_miss_check(tmp1, tmp2);
1247 
1248   } else {
1249     // Slower version which doesn't use SIGTRAP
1250 
1251     // Load stub address using toc (fixed instruction size, unlike load_const_optimized)
1252     calculate_address_from_global_toc(tmp1, SharedRuntime::get_ic_miss_stub(),
1253                                       true, true, false); // 2 instructions
1254     mtctr(tmp1);
1255 
1256     if (!implicit_null_checks_available) {
1257       cmpdi(CCR0, receiver, 0);
1258       beqctr(CCR0);
1259     }
1260     if (UseCompressedClassPointers) {
1261       lwz(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
1262     } else {
1263       ld(tmp1, oopDesc::klass_offset_in_bytes(), receiver);
1264     }
1265     ld(tmp2, in_bytes(CompiledICData::speculated_klass_offset()), data);
1266     cmpd(CCR0, tmp1, tmp2);
1267     bnectr(CCR0);
1268   }
1269 
1270   assert((offset() % end_alignment) == 0, "Misaligned verified entry point");
1271 
1272   return uep_offset;
1273 }
1274 
1275 void MacroAssembler::call_VM_base(Register oop_result,
1276                                   Register last_java_sp,
1277                                   address  entry_point,
1278                                   bool     check_exceptions) {
1279   BLOCK_COMMENT("call_VM {");
1280   // Determine last_java_sp register.
1281   if (!last_java_sp->is_valid()) {
1282     last_java_sp = R1_SP;
1283   }
1284   set_top_ijava_frame_at_SP_as_last_Java_frame(last_java_sp, R11_scratch1);
1285 
1286   // ARG1 must hold thread address.
1287   mr(R3_ARG1, R16_thread);
1288 #if defined(ABI_ELFv2)
1289   address return_pc = call_c(entry_point, relocInfo::none);
1290 #else
1291   address return_pc = call_c((FunctionDescriptor*)entry_point, relocInfo::none);
1292 #endif
1293 
1294   reset_last_Java_frame();
1295 
1296   // Check for pending exceptions.
1297   if (check_exceptions) {
1298     // We don't check for exceptions here.
1299     ShouldNotReachHere();
1300   }
1301 
1302   // Get oop result if there is one and reset the value in the thread.
1303   if (oop_result->is_valid()) {
1304     get_vm_result(oop_result);
1305   }
1306 
1307   _last_calls_return_pc = return_pc;
1308   BLOCK_COMMENT("} call_VM");
1309 }
1310 
1311 void MacroAssembler::call_VM_leaf_base(address entry_point) {
1312   BLOCK_COMMENT("call_VM_leaf {");
1313 #if defined(ABI_ELFv2)
1314   call_c(entry_point, relocInfo::none);
1315 #else
1316   call_c(CAST_FROM_FN_PTR(FunctionDescriptor*, entry_point), relocInfo::none);
1317 #endif
1318   BLOCK_COMMENT("} call_VM_leaf");
1319 }
1320 
1321 void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) {
1322   call_VM_base(oop_result, noreg, entry_point, check_exceptions);
1323 }
1324 
1325 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1,
1326                              bool check_exceptions) {
1327   // R3_ARG1 is reserved for the thread.
1328   mr_if_needed(R4_ARG2, arg_1);
1329   call_VM(oop_result, entry_point, check_exceptions);
1330 }
1331 
1332 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2,
1333                              bool check_exceptions) {
1334   // R3_ARG1 is reserved for the thread
1335   assert_different_registers(arg_2, R4_ARG2);
1336   mr_if_needed(R4_ARG2, arg_1);
1337   mr_if_needed(R5_ARG3, arg_2);
1338   call_VM(oop_result, entry_point, check_exceptions);
1339 }
1340 
1341 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, Register arg_3,
1342                              bool check_exceptions) {
1343   // R3_ARG1 is reserved for the thread
1344   assert_different_registers(arg_2, R4_ARG2);
1345   assert_different_registers(arg_3, R4_ARG2, R5_ARG3);
1346   mr_if_needed(R4_ARG2, arg_1);
1347   mr_if_needed(R5_ARG3, arg_2);
1348   mr_if_needed(R6_ARG4, arg_3);
1349   call_VM(oop_result, entry_point, check_exceptions);
1350 }
1351 
1352 void MacroAssembler::call_VM_leaf(address entry_point) {
1353   call_VM_leaf_base(entry_point);
1354 }
1355 
1356 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1) {
1357   mr_if_needed(R3_ARG1, arg_1);
1358   call_VM_leaf(entry_point);
1359 }
1360 
1361 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2) {
1362   assert_different_registers(arg_2, R3_ARG1);
1363   mr_if_needed(R3_ARG1, arg_1);
1364   mr_if_needed(R4_ARG2, arg_2);
1365   call_VM_leaf(entry_point);
1366 }
1367 
1368 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3) {
1369   assert_different_registers(arg_2, R3_ARG1);
1370   assert_different_registers(arg_3, R3_ARG1, R4_ARG2);
1371   mr_if_needed(R3_ARG1, arg_1);
1372   mr_if_needed(R4_ARG2, arg_2);
1373   mr_if_needed(R5_ARG3, arg_3);
1374   call_VM_leaf(entry_point);
1375 }
1376 
1377 // Check whether instruction is a read access to the polling page
1378 // which was emitted by load_from_polling_page(..).
1379 bool MacroAssembler::is_load_from_polling_page(int instruction, void* ucontext,
1380                                                address* polling_address_ptr) {
1381   if (!is_ld(instruction))
1382     return false; // It's not a ld. Fail.
1383 
1384   int rt = inv_rt_field(instruction);
1385   int ra = inv_ra_field(instruction);
1386   int ds = inv_ds_field(instruction);
1387   if (!(ds == 0 && ra != 0 && rt == 0)) {
1388     return false; // It's not a ld(r0, X, ra). Fail.
1389   }
1390 
1391   if (!ucontext) {
1392     // Set polling address.
1393     if (polling_address_ptr != nullptr) {
1394       *polling_address_ptr = nullptr;
1395     }
1396     return true; // No ucontext given. Can't check value of ra. Assume true.
1397   }
1398 
1399 #ifdef LINUX
1400   // Ucontext given. Check that register ra contains the address of
1401   // the safepoing polling page.
1402   ucontext_t* uc = (ucontext_t*) ucontext;
1403   // Set polling address.
1404   address addr = (address)uc->uc_mcontext.regs->gpr[ra] + (ssize_t)ds;
1405   if (polling_address_ptr != nullptr) {
1406     *polling_address_ptr = addr;
1407   }
1408   return SafepointMechanism::is_poll_address(addr);
1409 #else
1410   // Not on Linux, ucontext must be null.
1411   ShouldNotReachHere();
1412   return false;
1413 #endif
1414 }
1415 
1416 void MacroAssembler::bang_stack_with_offset(int offset) {
1417   // When increasing the stack, the old stack pointer will be written
1418   // to the new top of stack according to the PPC64 abi.
1419   // Therefore, stack banging is not necessary when increasing
1420   // the stack by <= os::vm_page_size() bytes.
1421   // When increasing the stack by a larger amount, this method is
1422   // called repeatedly to bang the intermediate pages.
1423 
1424   // Stack grows down, caller passes positive offset.
1425   assert(offset > 0, "must bang with positive offset");
1426 
1427   long stdoffset = -offset;
1428 
1429   if (is_simm(stdoffset, 16)) {
1430     // Signed 16 bit offset, a simple std is ok.
1431     if (UseLoadInstructionsForStackBangingPPC64) {
1432       ld(R0, (int)(signed short)stdoffset, R1_SP);
1433     } else {
1434       std(R0,(int)(signed short)stdoffset, R1_SP);
1435     }
1436   } else if (is_simm(stdoffset, 31)) {
1437     const int hi = MacroAssembler::largeoffset_si16_si16_hi(stdoffset);
1438     const int lo = MacroAssembler::largeoffset_si16_si16_lo(stdoffset);
1439 
1440     Register tmp = R11;
1441     addis(tmp, R1_SP, hi);
1442     if (UseLoadInstructionsForStackBangingPPC64) {
1443       ld(R0,  lo, tmp);
1444     } else {
1445       std(R0, lo, tmp);
1446     }
1447   } else {
1448     ShouldNotReachHere();
1449   }
1450 }
1451 
1452 // If instruction is a stack bang of the form
1453 //    std    R0,    x(Ry),       (see bang_stack_with_offset())
1454 //    stdu   R1_SP, x(R1_SP),    (see push_frame(), resize_frame())
1455 // or stdux  R1_SP, Rx, R1_SP    (see push_frame(), resize_frame())
1456 // return the banged address. Otherwise, return 0.
1457 address MacroAssembler::get_stack_bang_address(int instruction, void *ucontext) {
1458 #ifdef LINUX
1459   ucontext_t* uc = (ucontext_t*) ucontext;
1460   int rs = inv_rs_field(instruction);
1461   int ra = inv_ra_field(instruction);
1462   if (   (is_ld(instruction)   && rs == 0 &&  UseLoadInstructionsForStackBangingPPC64)
1463       || (is_std(instruction)  && rs == 0 && !UseLoadInstructionsForStackBangingPPC64)
1464       || (is_stdu(instruction) && rs == 1)) {
1465     int ds = inv_ds_field(instruction);
1466     // return banged address
1467     return ds+(address)uc->uc_mcontext.regs->gpr[ra];
1468   } else if (is_stdux(instruction) && rs == 1) {
1469     int rb = inv_rb_field(instruction);
1470     address sp = (address)uc->uc_mcontext.regs->gpr[1];
1471     long rb_val = (long)uc->uc_mcontext.regs->gpr[rb];
1472     return ra != 1 || rb_val >= 0 ? nullptr         // not a stack bang
1473                                   : sp + rb_val; // banged address
1474   }
1475   return nullptr; // not a stack bang
1476 #else
1477   // workaround not needed on !LINUX :-)
1478   ShouldNotCallThis();
1479   return nullptr;
1480 #endif
1481 }
1482 
1483 void MacroAssembler::reserved_stack_check(Register return_pc) {
1484   // Test if reserved zone needs to be enabled.
1485   Label no_reserved_zone_enabling;
1486 
1487   ld_ptr(R0, JavaThread::reserved_stack_activation_offset(), R16_thread);
1488   cmpld(CCR0, R1_SP, R0);
1489   blt_predict_taken(CCR0, no_reserved_zone_enabling);
1490 
1491   // Enable reserved zone again, throw stack overflow exception.
1492   push_frame_reg_args(0, R0);
1493   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), R16_thread);
1494   pop_frame();
1495   mtlr(return_pc);
1496   load_const_optimized(R0, StubRoutines::throw_delayed_StackOverflowError_entry());
1497   mtctr(R0);
1498   bctr();
1499 
1500   should_not_reach_here();
1501 
1502   bind(no_reserved_zone_enabling);
1503 }
1504 
1505 void MacroAssembler::getandsetd(Register dest_current_value, Register exchange_value, Register addr_base,
1506                                 bool cmpxchgx_hint) {
1507   Label retry;
1508   bind(retry);
1509   ldarx(dest_current_value, addr_base, cmpxchgx_hint);
1510   stdcx_(exchange_value, addr_base);
1511   if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1512     bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0.
1513   } else {
1514     bne(                  CCR0, retry); // StXcx_ sets CCR0.
1515   }
1516 }
1517 
1518 void MacroAssembler::getandaddd(Register dest_current_value, Register inc_value, Register addr_base,
1519                                 Register tmp, bool cmpxchgx_hint) {
1520   Label retry;
1521   bind(retry);
1522   ldarx(dest_current_value, addr_base, cmpxchgx_hint);
1523   add(tmp, dest_current_value, inc_value);
1524   stdcx_(tmp, addr_base);
1525   if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1526     bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0.
1527   } else {
1528     bne(                  CCR0, retry); // StXcx_ sets CCR0.
1529   }
1530 }
1531 
1532 // Word/sub-word atomic helper functions
1533 
1534 // Temps and addr_base are killed if size < 4 and processor does not support respective instructions.
1535 // Only signed types are supported with size < 4.
1536 // Atomic add always kills tmp1.
1537 void MacroAssembler::atomic_get_and_modify_generic(Register dest_current_value, Register exchange_value,
1538                                                    Register addr_base, Register tmp1, Register tmp2, Register tmp3,
1539                                                    bool cmpxchgx_hint, bool is_add, int size) {
1540   // Sub-word instructions are available since Power 8.
1541   // For older processors, instruction_type != size holds, and we
1542   // emulate the sub-word instructions by constructing a 4-byte value
1543   // that leaves the other bytes unchanged.
1544   const int instruction_type = VM_Version::has_lqarx() ? size : 4;
1545 
1546   Label retry;
1547   Register shift_amount = noreg,
1548            val32 = dest_current_value,
1549            modval = is_add ? tmp1 : exchange_value;
1550 
1551   if (instruction_type != size) {
1552     assert_different_registers(tmp1, tmp2, tmp3, dest_current_value, exchange_value, addr_base);
1553     modval = tmp1;
1554     shift_amount = tmp2;
1555     val32 = tmp3;
1556     // Need some preparation: Compute shift amount, align address. Note: shorts must be 2 byte aligned.
1557 #ifdef VM_LITTLE_ENDIAN
1558     rldic(shift_amount, addr_base, 3, 64-5); // (dest & 3) * 8;
1559     clrrdi(addr_base, addr_base, 2);
1560 #else
1561     xori(shift_amount, addr_base, (size == 1) ? 3 : 2);
1562     clrrdi(addr_base, addr_base, 2);
1563     rldic(shift_amount, shift_amount, 3, 64-5); // byte: ((3-dest) & 3) * 8; short: ((1-dest/2) & 1) * 16;
1564 #endif
1565   }
1566 
1567   // atomic emulation loop
1568   bind(retry);
1569 
1570   switch (instruction_type) {
1571     case 4: lwarx(val32, addr_base, cmpxchgx_hint); break;
1572     case 2: lharx(val32, addr_base, cmpxchgx_hint); break;
1573     case 1: lbarx(val32, addr_base, cmpxchgx_hint); break;
1574     default: ShouldNotReachHere();
1575   }
1576 
1577   if (instruction_type != size) {
1578     srw(dest_current_value, val32, shift_amount);
1579   }
1580 
1581   if (is_add) { add(modval, dest_current_value, exchange_value); }
1582 
1583   if (instruction_type != size) {
1584     // Transform exchange value such that the replacement can be done by one xor instruction.
1585     xorr(modval, dest_current_value, is_add ? modval : exchange_value);
1586     clrldi(modval, modval, (size == 1) ? 56 : 48);
1587     slw(modval, modval, shift_amount);
1588     xorr(modval, val32, modval);
1589   }
1590 
1591   switch (instruction_type) {
1592     case 4: stwcx_(modval, addr_base); break;
1593     case 2: sthcx_(modval, addr_base); break;
1594     case 1: stbcx_(modval, addr_base); break;
1595     default: ShouldNotReachHere();
1596   }
1597 
1598   if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1599     bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0.
1600   } else {
1601     bne(                  CCR0, retry); // StXcx_ sets CCR0.
1602   }
1603 
1604   // l?arx zero-extends, but Java wants byte/short values sign-extended.
1605   if (size == 1) {
1606     extsb(dest_current_value, dest_current_value);
1607   } else if (size == 2) {
1608     extsh(dest_current_value, dest_current_value);
1609   };
1610 }
1611 
1612 // Temps, addr_base and exchange_value are killed if size < 4 and processor does not support respective instructions.
1613 // Only signed types are supported with size < 4.
1614 void MacroAssembler::cmpxchg_loop_body(ConditionRegister flag, Register dest_current_value,
1615                                        Register compare_value, Register exchange_value,
1616                                        Register addr_base, Register tmp1, Register tmp2,
1617                                        Label &retry, Label &failed, bool cmpxchgx_hint, int size) {
1618   // Sub-word instructions are available since Power 8.
1619   // For older processors, instruction_type != size holds, and we
1620   // emulate the sub-word instructions by constructing a 4-byte value
1621   // that leaves the other bytes unchanged.
1622   const int instruction_type = VM_Version::has_lqarx() ? size : 4;
1623 
1624   Register shift_amount = noreg,
1625            val32 = dest_current_value,
1626            modval = exchange_value;
1627 
1628   if (instruction_type != size) {
1629     assert_different_registers(tmp1, tmp2, dest_current_value, compare_value, exchange_value, addr_base);
1630     shift_amount = tmp1;
1631     val32 = tmp2;
1632     modval = tmp2;
1633     // Need some preparation: Compute shift amount, align address. Note: shorts must be 2 byte aligned.
1634 #ifdef VM_LITTLE_ENDIAN
1635     rldic(shift_amount, addr_base, 3, 64-5); // (dest & 3) * 8;
1636     clrrdi(addr_base, addr_base, 2);
1637 #else
1638     xori(shift_amount, addr_base, (size == 1) ? 3 : 2);
1639     clrrdi(addr_base, addr_base, 2);
1640     rldic(shift_amount, shift_amount, 3, 64-5); // byte: ((3-dest) & 3) * 8; short: ((1-dest/2) & 1) * 16;
1641 #endif
1642     // Transform exchange value such that the replacement can be done by one xor instruction.
1643     xorr(exchange_value, compare_value, exchange_value);
1644     clrldi(exchange_value, exchange_value, (size == 1) ? 56 : 48);
1645     slw(exchange_value, exchange_value, shift_amount);
1646   }
1647 
1648   // atomic emulation loop
1649   bind(retry);
1650 
1651   switch (instruction_type) {
1652     case 4: lwarx(val32, addr_base, cmpxchgx_hint); break;
1653     case 2: lharx(val32, addr_base, cmpxchgx_hint); break;
1654     case 1: lbarx(val32, addr_base, cmpxchgx_hint); break;
1655     default: ShouldNotReachHere();
1656   }
1657 
1658   if (instruction_type != size) {
1659     srw(dest_current_value, val32, shift_amount);
1660   }
1661   if (size == 1) {
1662     extsb(dest_current_value, dest_current_value);
1663   } else if (size == 2) {
1664     extsh(dest_current_value, dest_current_value);
1665   };
1666 
1667   cmpw(flag, dest_current_value, compare_value);
1668   if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1669     bne_predict_not_taken(flag, failed);
1670   } else {
1671     bne(                  flag, failed);
1672   }
1673   // branch to done  => (flag == ne), (dest_current_value != compare_value)
1674   // fall through    => (flag == eq), (dest_current_value == compare_value)
1675 
1676   if (instruction_type != size) {
1677     xorr(modval, val32, exchange_value);
1678   }
1679 
1680   switch (instruction_type) {
1681     case 4: stwcx_(modval, addr_base); break;
1682     case 2: sthcx_(modval, addr_base); break;
1683     case 1: stbcx_(modval, addr_base); break;
1684     default: ShouldNotReachHere();
1685   }
1686 }
1687 
1688 // CmpxchgX sets condition register to cmpX(current, compare).
1689 void MacroAssembler::cmpxchg_generic(ConditionRegister flag, Register dest_current_value,
1690                                      Register compare_value, Register exchange_value,
1691                                      Register addr_base, Register tmp1, Register tmp2,
1692                                      int semantics, bool cmpxchgx_hint,
1693                                      Register int_flag_success, bool contention_hint, bool weak, int size) {
1694   Label retry;
1695   Label failed;
1696   Label done;
1697 
1698   // Save one branch if result is returned via register and
1699   // result register is different from the other ones.
1700   bool use_result_reg    = (int_flag_success != noreg);
1701   bool preset_result_reg = (int_flag_success != dest_current_value && int_flag_success != compare_value &&
1702                             int_flag_success != exchange_value && int_flag_success != addr_base &&
1703                             int_flag_success != tmp1 && int_flag_success != tmp2);
1704   assert(!weak || flag == CCR0, "weak only supported with CCR0");
1705   assert(size == 1 || size == 2 || size == 4, "unsupported");
1706 
1707   if (use_result_reg && preset_result_reg) {
1708     li(int_flag_success, 0); // preset (assume cas failed)
1709   }
1710 
1711   // Add simple guard in order to reduce risk of starving under high contention (recommended by IBM).
1712   if (contention_hint) { // Don't try to reserve if cmp fails.
1713     switch (size) {
1714       case 1: lbz(dest_current_value, 0, addr_base); extsb(dest_current_value, dest_current_value); break;
1715       case 2: lha(dest_current_value, 0, addr_base); break;
1716       case 4: lwz(dest_current_value, 0, addr_base); break;
1717       default: ShouldNotReachHere();
1718     }
1719     cmpw(flag, dest_current_value, compare_value);
1720     bne(flag, failed);
1721   }
1722 
1723   // release/fence semantics
1724   if (semantics & MemBarRel) {
1725     release();
1726   }
1727 
1728   cmpxchg_loop_body(flag, dest_current_value, compare_value, exchange_value, addr_base, tmp1, tmp2,
1729                     retry, failed, cmpxchgx_hint, size);
1730   if (!weak || use_result_reg) {
1731     if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1732       bne_predict_not_taken(CCR0, weak ? failed : retry); // StXcx_ sets CCR0.
1733     } else {
1734       bne(                  CCR0, weak ? failed : retry); // StXcx_ sets CCR0.
1735     }
1736   }
1737   // fall through    => (flag == eq), (dest_current_value == compare_value), (swapped)
1738 
1739   // Result in register (must do this at the end because int_flag_success can be the
1740   // same register as one above).
1741   if (use_result_reg) {
1742     li(int_flag_success, 1);
1743   }
1744 
1745   if (semantics & MemBarFenceAfter) {
1746     fence();
1747   } else if (semantics & MemBarAcq) {
1748     isync();
1749   }
1750 
1751   if (use_result_reg && !preset_result_reg) {
1752     b(done);
1753   }
1754 
1755   bind(failed);
1756   if (use_result_reg && !preset_result_reg) {
1757     li(int_flag_success, 0);
1758   }
1759 
1760   bind(done);
1761   // (flag == ne) => (dest_current_value != compare_value), (!swapped)
1762   // (flag == eq) => (dest_current_value == compare_value), ( swapped)
1763 }
1764 
1765 // Performs atomic compare exchange:
1766 //   if (compare_value == *addr_base)
1767 //     *addr_base = exchange_value
1768 //     int_flag_success = 1;
1769 //   else
1770 //     int_flag_success = 0;
1771 //
1772 // ConditionRegister flag       = cmp(compare_value, *addr_base)
1773 // Register dest_current_value  = *addr_base
1774 // Register compare_value       Used to compare with value in memory
1775 // Register exchange_value      Written to memory if compare_value == *addr_base
1776 // Register addr_base           The memory location to compareXChange
1777 // Register int_flag_success    Set to 1 if exchange_value was written to *addr_base
1778 //
1779 // To avoid the costly compare exchange the value is tested beforehand.
1780 // Several special cases exist to avoid that unnecessary information is generated.
1781 //
1782 void MacroAssembler::cmpxchgd(ConditionRegister flag,
1783                               Register dest_current_value, RegisterOrConstant compare_value, Register exchange_value,
1784                               Register addr_base, int semantics, bool cmpxchgx_hint,
1785                               Register int_flag_success, Label* failed_ext, bool contention_hint, bool weak) {
1786   Label retry;
1787   Label failed_int;
1788   Label& failed = (failed_ext != nullptr) ? *failed_ext : failed_int;
1789   Label done;
1790 
1791   // Save one branch if result is returned via register and result register is different from the other ones.
1792   bool use_result_reg    = (int_flag_success!=noreg);
1793   bool preset_result_reg = (int_flag_success!=dest_current_value && int_flag_success!=compare_value.register_or_noreg() &&
1794                             int_flag_success!=exchange_value && int_flag_success!=addr_base);
1795   assert(!weak || flag == CCR0, "weak only supported with CCR0");
1796   assert(int_flag_success == noreg || failed_ext == nullptr, "cannot have both");
1797 
1798   if (use_result_reg && preset_result_reg) {
1799     li(int_flag_success, 0); // preset (assume cas failed)
1800   }
1801 
1802   // Add simple guard in order to reduce risk of starving under high contention (recommended by IBM).
1803   if (contention_hint) { // Don't try to reserve if cmp fails.
1804     ld(dest_current_value, 0, addr_base);
1805     cmpd(flag, compare_value, dest_current_value);
1806     bne(flag, failed);
1807   }
1808 
1809   // release/fence semantics
1810   if (semantics & MemBarRel) {
1811     release();
1812   }
1813 
1814   // atomic emulation loop
1815   bind(retry);
1816 
1817   ldarx(dest_current_value, addr_base, cmpxchgx_hint);
1818   cmpd(flag, compare_value, dest_current_value);
1819   if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1820     bne_predict_not_taken(flag, failed);
1821   } else {
1822     bne(                  flag, failed);
1823   }
1824 
1825   stdcx_(exchange_value, addr_base);
1826   if (!weak || use_result_reg || failed_ext) {
1827     if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1828       bne_predict_not_taken(CCR0, weak ? failed : retry); // stXcx_ sets CCR0
1829     } else {
1830       bne(                  CCR0, weak ? failed : retry); // stXcx_ sets CCR0
1831     }
1832   }
1833 
1834   // result in register (must do this at the end because int_flag_success can be the same register as one above)
1835   if (use_result_reg) {
1836     li(int_flag_success, 1);
1837   }
1838 
1839   if (semantics & MemBarFenceAfter) {
1840     fence();
1841   } else if (semantics & MemBarAcq) {
1842     isync();
1843   }
1844 
1845   if (use_result_reg && !preset_result_reg) {
1846     b(done);
1847   }
1848 
1849   bind(failed_int);
1850   if (use_result_reg && !preset_result_reg) {
1851     li(int_flag_success, 0);
1852   }
1853 
1854   bind(done);
1855   // (flag == ne) => (dest_current_value != compare_value), (!swapped)
1856   // (flag == eq) => (dest_current_value == compare_value), ( swapped)
1857 }
1858 
1859 // Look up the method for a megamorphic invokeinterface call.
1860 // The target method is determined by <intf_klass, itable_index>.
1861 // The receiver klass is in recv_klass.
1862 // On success, the result will be in method_result, and execution falls through.
1863 // On failure, execution transfers to the given label.
1864 void MacroAssembler::lookup_interface_method(Register recv_klass,
1865                                              Register intf_klass,
1866                                              RegisterOrConstant itable_index,
1867                                              Register method_result,
1868                                              Register scan_temp,
1869                                              Register temp2,
1870                                              Label& L_no_such_interface,
1871                                              bool return_method) {
1872   assert_different_registers(recv_klass, intf_klass, method_result, scan_temp);
1873 
1874   // Compute start of first itableOffsetEntry (which is at the end of the vtable).
1875   int vtable_base = in_bytes(Klass::vtable_start_offset());
1876   int itentry_off = in_bytes(itableMethodEntry::method_offset());
1877   int logMEsize   = exact_log2(itableMethodEntry::size() * wordSize);
1878   int scan_step   = itableOffsetEntry::size() * wordSize;
1879   int log_vte_size= exact_log2(vtableEntry::size_in_bytes());
1880 
1881   lwz(scan_temp, in_bytes(Klass::vtable_length_offset()), recv_klass);
1882   // We should store the aligned, prescaled offset in the klass.
1883   // Then the next several instructions would fold away.
1884 
1885   sldi(scan_temp, scan_temp, log_vte_size);
1886   addi(scan_temp, scan_temp, vtable_base);
1887   add(scan_temp, recv_klass, scan_temp);
1888 
1889   // Adjust recv_klass by scaled itable_index, so we can free itable_index.
1890   if (return_method) {
1891     if (itable_index.is_register()) {
1892       Register itable_offset = itable_index.as_register();
1893       sldi(method_result, itable_offset, logMEsize);
1894       if (itentry_off) { addi(method_result, method_result, itentry_off); }
1895       add(method_result, method_result, recv_klass);
1896     } else {
1897       long itable_offset = (long)itable_index.as_constant();
1898       // static address, no relocation
1899       add_const_optimized(method_result, recv_klass, (itable_offset << logMEsize) + itentry_off, temp2);
1900     }
1901   }
1902 
1903   // for (scan = klass->itable(); scan->interface() != nullptr; scan += scan_step) {
1904   //   if (scan->interface() == intf) {
1905   //     result = (klass + scan->offset() + itable_index);
1906   //   }
1907   // }
1908   Label search, found_method;
1909 
1910   for (int peel = 1; peel >= 0; peel--) {
1911     // %%%% Could load both offset and interface in one ldx, if they were
1912     // in the opposite order. This would save a load.
1913     ld(temp2, in_bytes(itableOffsetEntry::interface_offset()), scan_temp);
1914 
1915     // Check that this entry is non-null. A null entry means that
1916     // the receiver class doesn't implement the interface, and wasn't the
1917     // same as when the caller was compiled.
1918     cmpd(CCR0, temp2, intf_klass);
1919 
1920     if (peel) {
1921       beq(CCR0, found_method);
1922     } else {
1923       bne(CCR0, search);
1924       // (invert the test to fall through to found_method...)
1925     }
1926 
1927     if (!peel) break;
1928 
1929     bind(search);
1930 
1931     cmpdi(CCR0, temp2, 0);
1932     beq(CCR0, L_no_such_interface);
1933     addi(scan_temp, scan_temp, scan_step);
1934   }
1935 
1936   bind(found_method);
1937 
1938   // Got a hit.
1939   if (return_method) {
1940     int ito_offset = in_bytes(itableOffsetEntry::offset_offset());
1941     lwz(scan_temp, ito_offset, scan_temp);
1942     ldx(method_result, scan_temp, method_result);
1943   }
1944 }
1945 
1946 // virtual method calling
1947 void MacroAssembler::lookup_virtual_method(Register recv_klass,
1948                                            RegisterOrConstant vtable_index,
1949                                            Register method_result) {
1950 
1951   assert_different_registers(recv_klass, method_result, vtable_index.register_or_noreg());
1952 
1953   const ByteSize base = Klass::vtable_start_offset();
1954   assert(vtableEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
1955 
1956   if (vtable_index.is_register()) {
1957     sldi(vtable_index.as_register(), vtable_index.as_register(), LogBytesPerWord);
1958     add(recv_klass, vtable_index.as_register(), recv_klass);
1959   } else {
1960     addi(recv_klass, recv_klass, vtable_index.as_constant() << LogBytesPerWord);
1961   }
1962   ld(R19_method, in_bytes(base + vtableEntry::method_offset()), recv_klass);
1963 }
1964 
1965 /////////////////////////////////////////// subtype checking ////////////////////////////////////////////
1966 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
1967                                                    Register super_klass,
1968                                                    Register temp1_reg,
1969                                                    Register temp2_reg,
1970                                                    Label* L_success,
1971                                                    Label* L_failure,
1972                                                    Label* L_slow_path,
1973                                                    RegisterOrConstant super_check_offset) {
1974 
1975   const Register check_cache_offset = temp1_reg;
1976   const Register cached_super       = temp2_reg;
1977 
1978   assert_different_registers(sub_klass, super_klass, check_cache_offset, cached_super);
1979 
1980   int sco_offset = in_bytes(Klass::super_check_offset_offset());
1981   int sc_offset  = in_bytes(Klass::secondary_super_cache_offset());
1982 
1983   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
1984   bool need_slow_path = (must_load_sco || super_check_offset.constant_or_zero() == sco_offset);
1985 
1986   Label L_fallthrough;
1987   int label_nulls = 0;
1988   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
1989   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
1990   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
1991   assert(label_nulls <= 1 ||
1992          (L_slow_path == &L_fallthrough && label_nulls <= 2 && !need_slow_path),
1993          "at most one null in the batch, usually");
1994 
1995   // If the pointers are equal, we are done (e.g., String[] elements).
1996   // This self-check enables sharing of secondary supertype arrays among
1997   // non-primary types such as array-of-interface. Otherwise, each such
1998   // type would need its own customized SSA.
1999   // We move this check to the front of the fast path because many
2000   // type checks are in fact trivially successful in this manner,
2001   // so we get a nicely predicted branch right at the start of the check.
2002   cmpd(CCR0, sub_klass, super_klass);
2003   beq(CCR0, *L_success);
2004 
2005   // Check the supertype display:
2006   if (must_load_sco) {
2007     // The super check offset is always positive...
2008     lwz(check_cache_offset, sco_offset, super_klass);
2009     super_check_offset = RegisterOrConstant(check_cache_offset);
2010     // super_check_offset is register.
2011     assert_different_registers(sub_klass, super_klass, cached_super, super_check_offset.as_register());
2012   }
2013   // The loaded value is the offset from Klass.
2014 
2015   ld(cached_super, super_check_offset, sub_klass);
2016   cmpd(CCR0, cached_super, super_klass);
2017 
2018   // This check has worked decisively for primary supers.
2019   // Secondary supers are sought in the super_cache ('super_cache_addr').
2020   // (Secondary supers are interfaces and very deeply nested subtypes.)
2021   // This works in the same check above because of a tricky aliasing
2022   // between the super_cache and the primary super display elements.
2023   // (The 'super_check_addr' can address either, as the case requires.)
2024   // Note that the cache is updated below if it does not help us find
2025   // what we need immediately.
2026   // So if it was a primary super, we can just fail immediately.
2027   // Otherwise, it's the slow path for us (no success at this point).
2028 
2029 #define FINAL_JUMP(label) if (&(label) != &L_fallthrough) { b(label); }
2030 
2031   if (super_check_offset.is_register()) {
2032     beq(CCR0, *L_success);
2033     cmpwi(CCR0, super_check_offset.as_register(), sc_offset);
2034     if (L_failure == &L_fallthrough) {
2035       beq(CCR0, *L_slow_path);
2036     } else {
2037       bne(CCR0, *L_failure);
2038       FINAL_JUMP(*L_slow_path);
2039     }
2040   } else {
2041     if (super_check_offset.as_constant() == sc_offset) {
2042       // Need a slow path; fast failure is impossible.
2043       if (L_slow_path == &L_fallthrough) {
2044         beq(CCR0, *L_success);
2045       } else {
2046         bne(CCR0, *L_slow_path);
2047         FINAL_JUMP(*L_success);
2048       }
2049     } else {
2050       // No slow path; it's a fast decision.
2051       if (L_failure == &L_fallthrough) {
2052         beq(CCR0, *L_success);
2053       } else {
2054         bne(CCR0, *L_failure);
2055         FINAL_JUMP(*L_success);
2056       }
2057     }
2058   }
2059 
2060   bind(L_fallthrough);
2061 #undef FINAL_JUMP
2062 }
2063 
2064 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
2065                                                    Register super_klass,
2066                                                    Register temp1_reg,
2067                                                    Register temp2_reg,
2068                                                    Label* L_success,
2069                                                    Register result_reg) {
2070   const Register array_ptr = temp1_reg; // current value from cache array
2071   const Register temp      = temp2_reg;
2072 
2073   assert_different_registers(sub_klass, super_klass, array_ptr, temp);
2074 
2075   int source_offset = in_bytes(Klass::secondary_supers_offset());
2076   int target_offset = in_bytes(Klass::secondary_super_cache_offset());
2077 
2078   int length_offset = Array<Klass*>::length_offset_in_bytes();
2079   int base_offset   = Array<Klass*>::base_offset_in_bytes();
2080 
2081   Label hit, loop, failure, fallthru;
2082 
2083   ld(array_ptr, source_offset, sub_klass);
2084 
2085   // TODO: PPC port: assert(4 == arrayOopDesc::length_length_in_bytes(), "precondition violated.");
2086   lwz(temp, length_offset, array_ptr);
2087   cmpwi(CCR0, temp, 0);
2088   beq(CCR0, result_reg!=noreg ? failure : fallthru); // length 0
2089 
2090   mtctr(temp); // load ctr
2091 
2092   bind(loop);
2093   // Oops in table are NO MORE compressed.
2094   ld(temp, base_offset, array_ptr);
2095   cmpd(CCR0, temp, super_klass);
2096   beq(CCR0, hit);
2097   addi(array_ptr, array_ptr, BytesPerWord);
2098   bdnz(loop);
2099 
2100   bind(failure);
2101   if (result_reg!=noreg) li(result_reg, 1); // load non-zero result (indicates a miss)
2102   b(fallthru);
2103 
2104   bind(hit);
2105   std(super_klass, target_offset, sub_klass); // save result to cache
2106   if (result_reg != noreg) { li(result_reg, 0); } // load zero result (indicates a hit)
2107   if (L_success != nullptr) { b(*L_success); }
2108   else if (result_reg == noreg) { blr(); } // return with CR0.eq if neither label nor result reg provided
2109 
2110   bind(fallthru);
2111 }
2112 
2113 // Try fast path, then go to slow one if not successful
2114 void MacroAssembler::check_klass_subtype(Register sub_klass,
2115                          Register super_klass,
2116                          Register temp1_reg,
2117                          Register temp2_reg,
2118                          Label& L_success) {
2119   Label L_failure;
2120   check_klass_subtype_fast_path(sub_klass, super_klass, temp1_reg, temp2_reg, &L_success, &L_failure);
2121   check_klass_subtype_slow_path(sub_klass, super_klass, temp1_reg, temp2_reg, &L_success);
2122   bind(L_failure); // Fallthru if not successful.
2123 }
2124 
2125 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
2126   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
2127 
2128   Label L_fallthrough;
2129   if (L_fast_path == nullptr) {
2130     L_fast_path = &L_fallthrough;
2131   } else if (L_slow_path == nullptr) {
2132     L_slow_path = &L_fallthrough;
2133   }
2134 
2135   // Fast path check: class is fully initialized
2136   lbz(R0, in_bytes(InstanceKlass::init_state_offset()), klass);
2137   cmpwi(CCR0, R0, InstanceKlass::fully_initialized);
2138   beq(CCR0, *L_fast_path);
2139 
2140   // Fast path check: current thread is initializer thread
2141   ld(R0, in_bytes(InstanceKlass::init_thread_offset()), klass);
2142   cmpd(CCR0, thread, R0);
2143   if (L_slow_path == &L_fallthrough) {
2144     beq(CCR0, *L_fast_path);
2145   } else if (L_fast_path == &L_fallthrough) {
2146     bne(CCR0, *L_slow_path);
2147   } else {
2148     Unimplemented();
2149   }
2150 
2151   bind(L_fallthrough);
2152 }
2153 
2154 RegisterOrConstant MacroAssembler::argument_offset(RegisterOrConstant arg_slot,
2155                                                    Register temp_reg,
2156                                                    int extra_slot_offset) {
2157   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
2158   int stackElementSize = Interpreter::stackElementSize;
2159   int offset = extra_slot_offset * stackElementSize;
2160   if (arg_slot.is_constant()) {
2161     offset += arg_slot.as_constant() * stackElementSize;
2162     return offset;
2163   } else {
2164     assert(temp_reg != noreg, "must specify");
2165     sldi(temp_reg, arg_slot.as_register(), exact_log2(stackElementSize));
2166     if (offset != 0)
2167       addi(temp_reg, temp_reg, offset);
2168     return temp_reg;
2169   }
2170 }
2171 
2172 void MacroAssembler::tlab_allocate(
2173   Register obj,                      // result: pointer to object after successful allocation
2174   Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
2175   int      con_size_in_bytes,        // object size in bytes if   known at compile time
2176   Register t1,                       // temp register
2177   Label&   slow_case                 // continuation point if fast allocation fails
2178 ) {
2179   // make sure arguments make sense
2180   assert_different_registers(obj, var_size_in_bytes, t1);
2181   assert(0 <= con_size_in_bytes && is_simm16(con_size_in_bytes), "illegal object size");
2182   assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, "object size is not multiple of alignment");
2183 
2184   const Register new_top = t1;
2185   //verify_tlab(); not implemented
2186 
2187   ld(obj, in_bytes(JavaThread::tlab_top_offset()), R16_thread);
2188   ld(R0, in_bytes(JavaThread::tlab_end_offset()), R16_thread);
2189   if (var_size_in_bytes == noreg) {
2190     addi(new_top, obj, con_size_in_bytes);
2191   } else {
2192     add(new_top, obj, var_size_in_bytes);
2193   }
2194   cmpld(CCR0, new_top, R0);
2195   bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::greater), slow_case);
2196 
2197 #ifdef ASSERT
2198   // make sure new free pointer is properly aligned
2199   {
2200     Label L;
2201     andi_(R0, new_top, MinObjAlignmentInBytesMask);
2202     beq(CCR0, L);
2203     stop("updated TLAB free is not properly aligned");
2204     bind(L);
2205   }
2206 #endif // ASSERT
2207 
2208   // update the tlab top pointer
2209   std(new_top, in_bytes(JavaThread::tlab_top_offset()), R16_thread);
2210   //verify_tlab(); not implemented
2211 }
2212 void MacroAssembler::incr_allocated_bytes(RegisterOrConstant size_in_bytes, Register t1, Register t2) {
2213   unimplemented("incr_allocated_bytes");
2214 }
2215 
2216 address MacroAssembler::emit_trampoline_stub(int destination_toc_offset,
2217                                              int insts_call_instruction_offset, Register Rtoc) {
2218   // Start the stub.
2219   address stub = start_a_stub(64);
2220   if (stub == nullptr) { return nullptr; } // CodeCache full: bail out
2221 
2222   // Create a trampoline stub relocation which relates this trampoline stub
2223   // with the call instruction at insts_call_instruction_offset in the
2224   // instructions code-section.
2225   relocate(trampoline_stub_Relocation::spec(code()->insts()->start() + insts_call_instruction_offset));
2226   const int stub_start_offset = offset();
2227 
2228   // For java_to_interp stubs we use R11_scratch1 as scratch register
2229   // and in call trampoline stubs we use R12_scratch2. This way we
2230   // can distinguish them (see is_NativeCallTrampolineStub_at()).
2231   Register reg_scratch = R12_scratch2;
2232 
2233   // Now, create the trampoline stub's code:
2234   // - load the TOC
2235   // - load the call target from the constant pool
2236   // - call
2237   if (Rtoc == noreg) {
2238     calculate_address_from_global_toc(reg_scratch, method_toc());
2239     Rtoc = reg_scratch;
2240   }
2241 
2242   ld_largeoffset_unchecked(reg_scratch, destination_toc_offset, Rtoc, false);
2243   mtctr(reg_scratch);
2244   bctr();
2245 
2246   const address stub_start_addr = addr_at(stub_start_offset);
2247 
2248   // Assert that the encoded destination_toc_offset can be identified and that it is correct.
2249   assert(destination_toc_offset == NativeCallTrampolineStub_at(stub_start_addr)->destination_toc_offset(),
2250          "encoded offset into the constant pool must match");
2251   // Trampoline_stub_size should be good.
2252   assert((uint)(offset() - stub_start_offset) <= trampoline_stub_size, "should be good size");
2253   assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline");
2254 
2255   // End the stub.
2256   end_a_stub();
2257   return stub;
2258 }
2259 
2260 // "The box" is the space on the stack where we copy the object mark.
2261 void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register oop, Register box,
2262                                                Register temp, Register displaced_header, Register current_header) {
2263   assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_lock_lightweight");
2264   assert_different_registers(oop, box, temp, displaced_header, current_header);
2265   Label object_has_monitor;
2266   Label cas_failed;
2267   Label success, failure;
2268 
2269   // Load markWord from object into displaced_header.
2270   ld(displaced_header, oopDesc::mark_offset_in_bytes(), oop);
2271 
2272   if (DiagnoseSyncOnValueBasedClasses != 0) {
2273     load_klass(temp, oop);
2274     lwz(temp, in_bytes(Klass::access_flags_offset()), temp);
2275     testbitdi(flag, R0, temp, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
2276     bne(flag, failure);
2277   }
2278 
2279   // Handle existing monitor.
2280   // The object has an existing monitor iff (mark & monitor_value) != 0.
2281   andi_(temp, displaced_header, markWord::monitor_value);
2282   bne(CCR0, object_has_monitor);
2283 
2284   if (LockingMode == LM_MONITOR) {
2285     // Set NE to indicate 'failure' -> take slow-path.
2286     crandc(flag, Assembler::equal, flag, Assembler::equal);
2287     b(failure);
2288   } else {
2289     assert(LockingMode == LM_LEGACY, "must be");
2290     // Set displaced_header to be (markWord of object | UNLOCK_VALUE).
2291     ori(displaced_header, displaced_header, markWord::unlocked_value);
2292 
2293     // Load Compare Value application register.
2294 
2295     // Initialize the box. (Must happen before we update the object mark!)
2296     std(displaced_header, BasicLock::displaced_header_offset_in_bytes(), box);
2297 
2298     // Must fence, otherwise, preceding store(s) may float below cmpxchg.
2299     // Compare object markWord with mark and if equal exchange scratch1 with object markWord.
2300     cmpxchgd(/*flag=*/flag,
2301              /*current_value=*/current_header,
2302              /*compare_value=*/displaced_header,
2303              /*exchange_value=*/box,
2304              /*where=*/oop,
2305              MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2306              MacroAssembler::cmpxchgx_hint_acquire_lock(),
2307              noreg,
2308              &cas_failed,
2309              /*check without membar and ldarx first*/true);
2310     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2311     // If the compare-and-exchange succeeded, then we found an unlocked
2312     // object and we have now locked it.
2313     b(success);
2314 
2315     bind(cas_failed);
2316     // We did not see an unlocked object so try the fast recursive case.
2317 
2318     // Check if the owner is self by comparing the value in the markWord of object
2319     // (current_header) with the stack pointer.
2320     sub(current_header, current_header, R1_SP);
2321     load_const_optimized(temp, ~(os::vm_page_size()-1) | markWord::lock_mask_in_place);
2322 
2323     and_(R0/*==0?*/, current_header, temp);
2324     // If condition is true we are cont and hence we can store 0 as the
2325     // displaced header in the box, which indicates that it is a recursive lock.
2326     std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), box);
2327 
2328     if (flag != CCR0) {
2329       mcrf(flag, CCR0);
2330     }
2331     beq(CCR0, success);
2332     b(failure);
2333   }
2334 
2335   // Handle existing monitor.
2336   bind(object_has_monitor);
2337   // The object's monitor m is unlocked iff m->owner is null,
2338   // otherwise m->owner may contain a thread or a stack address.
2339 
2340   // Try to CAS m->owner from null to current thread.
2341   addi(temp, displaced_header, in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value);
2342   Register thread_id = displaced_header;
2343   ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2344   cmpxchgd(/*flag=*/flag,
2345            /*current_value=*/current_header,
2346            /*compare_value=*/(intptr_t)0,
2347            /*exchange_value=*/thread_id,
2348            /*where=*/temp,
2349            MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2350            MacroAssembler::cmpxchgx_hint_acquire_lock());
2351 
2352   // Store a non-null value into the box.
2353   std(box, BasicLock::displaced_header_offset_in_bytes(), box);
2354   beq(flag, success);
2355 
2356   // Check for recursive locking.
2357   cmpd(flag, current_header, thread_id);
2358   bne(flag, failure);
2359 
2360   // Current thread already owns the lock. Just increment recursions.
2361   Register recursions = displaced_header;
2362   ld(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2363   addi(recursions, recursions, 1);
2364   std(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2365 
2366   // flag == EQ indicates success, increment held monitor count
2367   // flag == NE indicates failure
2368   bind(success);
2369   inc_held_monitor_count(temp);
2370   bind(failure);
2371 }
2372 
2373 void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
2374                                                  Register temp, Register displaced_header, Register current_header) {
2375   assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight");
2376   assert_different_registers(oop, box, temp, displaced_header, current_header);
2377   Label success, failure, object_has_monitor, notRecursive;
2378 
2379   if (LockingMode == LM_LEGACY) {
2380     // Find the lock address and load the displaced header from the stack.
2381     ld(displaced_header, BasicLock::displaced_header_offset_in_bytes(), box);
2382 
2383     // If the displaced header is 0, we have a recursive unlock.
2384     cmpdi(flag, displaced_header, 0);
2385     beq(flag, success);
2386   }
2387 
2388   // Handle existing monitor.
2389   // The object has an existing monitor iff (mark & monitor_value) != 0.
2390   ld(current_header, oopDesc::mark_offset_in_bytes(), oop);
2391   andi_(R0, current_header, markWord::monitor_value);
2392   bne(CCR0, object_has_monitor);
2393 
2394   if (LockingMode == LM_MONITOR) {
2395     // Set NE to indicate 'failure' -> take slow-path.
2396     crandc(flag, Assembler::equal, flag, Assembler::equal);
2397     b(failure);
2398   } else {
2399     assert(LockingMode == LM_LEGACY, "must be");
2400     // Check if it is still a light weight lock, this is is true if we see
2401     // the stack address of the basicLock in the markWord of the object.
2402     // Cmpxchg sets flag to cmpd(current_header, box).
2403     cmpxchgd(/*flag=*/flag,
2404              /*current_value=*/current_header,
2405              /*compare_value=*/box,
2406              /*exchange_value=*/displaced_header,
2407              /*where=*/oop,
2408              MacroAssembler::MemBarRel,
2409              MacroAssembler::cmpxchgx_hint_release_lock(),
2410              noreg,
2411              &failure);
2412     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2413     b(success);
2414   }
2415 
2416   // Handle existing monitor.
2417   bind(object_has_monitor);
2418   STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
2419   addi(current_header, current_header, -(int)markWord::monitor_value); // monitor
2420   ld(temp,             in_bytes(ObjectMonitor::owner_offset()), current_header);
2421 
2422   // In case of LM_LIGHTWEIGHT, we may reach here with (temp & ObjectMonitor::ANONYMOUS_OWNER) != 0.
2423   // This is handled like owner thread mismatches: We take the slow path.
2424   Register thread_id = displaced_header;
2425   ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2426   cmpd(flag, temp, thread_id);
2427   bne(flag, failure);
2428 
2429   ld(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2430 
2431   addic_(displaced_header, displaced_header, -1);
2432   blt(CCR0, notRecursive); // Not recursive if negative after decrement.
2433   std(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2434   if (flag == CCR0) { // Otherwise, flag is already EQ, here.
2435     crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Set CCR0 EQ
2436   }
2437   b(success);
2438 
2439   bind(notRecursive);
2440   ld(temp,             in_bytes(ObjectMonitor::EntryList_offset()), current_header);
2441   ld(displaced_header, in_bytes(ObjectMonitor::cxq_offset()), current_header);
2442   orr(temp, temp, displaced_header); // Will be 0 if both are 0.
2443   cmpdi(flag, temp, 0);
2444   bne(flag, failure);
2445   release();
2446   std(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2447 
2448   // flag == EQ indicates success, decrement held monitor count
2449   // flag == NE indicates failure
2450   bind(success);
2451   dec_held_monitor_count(temp);
2452   bind(failure);
2453 }
2454 
2455 void MacroAssembler::compiler_fast_lock_lightweight_object(ConditionRegister flag, Register obj, Register tmp1,
2456                                                            Register tmp2, Register tmp3) {
2457   assert_different_registers(obj, tmp1, tmp2, tmp3);
2458   assert(flag == CCR0, "bad condition register");
2459 
2460   // Handle inflated monitor.
2461   Label inflated;
2462   // Finish fast lock successfully. MUST reach to with flag == NE
2463   Label locked;
2464   // Finish fast lock unsuccessfully. MUST branch to with flag == EQ
2465   Label slow_path;
2466 
2467   if (DiagnoseSyncOnValueBasedClasses != 0) {
2468     load_klass(tmp1, obj);
2469     lwz(tmp1, in_bytes(Klass::access_flags_offset()), tmp1);
2470     testbitdi(flag, R0, tmp1, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
2471     bne(flag, slow_path);
2472   }
2473 
2474   const Register mark = tmp1;
2475   const Register t = tmp3; // Usage of R0 allowed!
2476 
2477   { // Lightweight locking
2478 
2479     // Push lock to the lock stack and finish successfully. MUST reach to with flag == EQ
2480     Label push;
2481 
2482     const Register top = tmp2;
2483 
2484     // Check if lock-stack is full.
2485     lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2486     cmplwi(flag, top, LockStack::end_offset() - 1);
2487     bgt(flag, slow_path);
2488 
2489     // The underflow check is elided. The recursive check will always fail
2490     // when the lock stack is empty because of the _bad_oop_sentinel field.
2491 
2492     // Check if recursive.
2493     subi(t, top, oopSize);
2494     ldx(t, R16_thread, t);
2495     cmpd(flag, obj, t);
2496     beq(flag, push);
2497 
2498     // Check for monitor (0b10) or locked (0b00).
2499     ld(mark, oopDesc::mark_offset_in_bytes(), obj);
2500     andi_(t, mark, markWord::lock_mask_in_place);
2501     cmpldi(flag, t, markWord::unlocked_value);
2502     bgt(flag, inflated);
2503     bne(flag, slow_path);
2504 
2505     // Not inflated.
2506 
2507     // Try to lock. Transition lock bits 0b00 => 0b01
2508     assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
2509     atomically_flip_locked_state(/* is_unlock */ false, obj, mark, slow_path, MacroAssembler::MemBarAcq);
2510 
2511     bind(push);
2512     // After successful lock, push object on lock-stack.
2513     stdx(obj, R16_thread, top);
2514     addi(top, top, oopSize);
2515     stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2516     b(locked);
2517   }
2518 
2519   { // Handle inflated monitor.
2520     bind(inflated);
2521 
2522     // mark contains the tagged ObjectMonitor*.
2523     const Register tagged_monitor = mark;
2524     const uintptr_t monitor_tag = markWord::monitor_value;
2525     const Register owner_addr = tmp2;
2526 
2527     // Compute owner address.
2528     addi(owner_addr, tagged_monitor, in_bytes(ObjectMonitor::owner_offset()) - monitor_tag);
2529 
2530     // CAS owner (null => current thread id).
2531     Register thread_id = tmp1;
2532     ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2533     cmpxchgd(/*flag=*/flag,
2534             /*current_value=*/t,
2535             /*compare_value=*/(intptr_t)0,
2536             /*exchange_value=*/thread_id,
2537             /*where=*/owner_addr,
2538             MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2539             MacroAssembler::cmpxchgx_hint_acquire_lock());
2540     beq(flag, locked);
2541 
2542     // Check if recursive.
2543     cmpd(flag, t, thread_id);
2544     bne(flag, slow_path);
2545 
2546     // Recursive.
2547     ld(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2548     addi(tmp1, tmp1, 1);
2549     std(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2550   }
2551 
2552   bind(locked);
2553   inc_held_monitor_count(tmp1);
2554 
2555 #ifdef ASSERT
2556   // Check that locked label is reached with flag == EQ.
2557   Label flag_correct;
2558   beq(flag, flag_correct);
2559   stop("Fast Lock Flag != EQ");
2560 #endif
2561   bind(slow_path);
2562 #ifdef ASSERT
2563   // Check that slow_path label is reached with flag == NE.
2564   bne(flag, flag_correct);
2565   stop("Fast Lock Flag != NE");
2566   bind(flag_correct);
2567 #endif
2568   // C2 uses the value of flag (NE vs EQ) to determine the continuation.
2569 }
2570 
2571 void MacroAssembler::compiler_fast_unlock_lightweight_object(ConditionRegister flag, Register obj, Register tmp1,
2572                                                              Register tmp2, Register tmp3) {
2573   assert_different_registers(obj, tmp1, tmp2, tmp3);
2574   assert(flag == CCR0, "bad condition register");
2575 
2576   // Handle inflated monitor.
2577   Label inflated, inflated_load_monitor;
2578   // Finish fast unlock successfully. MUST reach to with flag == EQ.
2579   Label unlocked;
2580   // Finish fast unlock unsuccessfully. MUST branch to with flag == NE.
2581   Label slow_path;
2582 
2583   const Register mark = tmp1;
2584   const Register top = tmp2;
2585   const Register t = tmp3;
2586 
2587   { // Lightweight unlock
2588     Label push_and_slow;
2589 
2590     // Check if obj is top of lock-stack.
2591     lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2592     subi(top, top, oopSize);
2593     ldx(t, R16_thread, top);
2594     cmpd(flag, obj, t);
2595     // Top of lock stack was not obj. Must be monitor.
2596     bne(flag, inflated_load_monitor);
2597 
2598     // Pop lock-stack.
2599     DEBUG_ONLY(li(t, 0);)
2600     DEBUG_ONLY(stdx(t, R16_thread, top);)
2601     stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2602 
2603     // The underflow check is elided. The recursive check will always fail
2604     // when the lock stack is empty because of the _bad_oop_sentinel field.
2605 
2606     // Check if recursive.
2607     subi(t, top, oopSize);
2608     ldx(t, R16_thread, t);
2609     cmpd(flag, obj, t);
2610     beq(flag, unlocked);
2611 
2612     // Not recursive.
2613 
2614     // Check for monitor (0b10).
2615     ld(mark, oopDesc::mark_offset_in_bytes(), obj);
2616     andi_(t, mark, markWord::monitor_value);
2617     bne(CCR0, inflated);
2618 
2619 #ifdef ASSERT
2620     // Check header not unlocked (0b01).
2621     Label not_unlocked;
2622     andi_(t, mark, markWord::unlocked_value);
2623     beq(CCR0, not_unlocked);
2624     stop("lightweight_unlock already unlocked");
2625     bind(not_unlocked);
2626 #endif
2627 
2628     // Try to unlock. Transition lock bits 0b00 => 0b01
2629     atomically_flip_locked_state(/* is_unlock */ true, obj, mark, push_and_slow, MacroAssembler::MemBarRel);
2630     b(unlocked);
2631 
2632     bind(push_and_slow);
2633     // Restore lock-stack and handle the unlock in runtime.
2634     DEBUG_ONLY(stdx(obj, R16_thread, top);)
2635     addi(top, top, oopSize);
2636     stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2637     b(slow_path);
2638   }
2639 
2640   { // Handle inflated monitor.
2641     bind(inflated_load_monitor);
2642     ld(mark, oopDesc::mark_offset_in_bytes(), obj);
2643 #ifdef ASSERT
2644     andi_(t, mark, markWord::monitor_value);
2645     bne(CCR0, inflated);
2646     stop("Fast Unlock not monitor");
2647 #endif
2648 
2649     bind(inflated);
2650 
2651 #ifdef ASSERT
2652     Label check_done;
2653     subi(top, top, oopSize);
2654     cmplwi(CCR0, top, in_bytes(JavaThread::lock_stack_base_offset()));
2655     blt(CCR0, check_done);
2656     ldx(t, R16_thread, top);
2657     cmpd(flag, obj, t);
2658     bne(flag, inflated);
2659     stop("Fast Unlock lock on stack");
2660     bind(check_done);
2661 #endif
2662 
2663     // mark contains the tagged ObjectMonitor*.
2664     const Register monitor = mark;
2665     const uintptr_t monitor_tag = markWord::monitor_value;
2666 
2667     // Untag the monitor.
2668     subi(monitor, mark, monitor_tag);
2669 
2670     const Register recursions = tmp2;
2671     Label not_recursive;
2672 
2673     // Check if recursive.
2674     ld(recursions, in_bytes(ObjectMonitor::recursions_offset()), monitor);
2675     addic_(recursions, recursions, -1);
2676     blt(CCR0, not_recursive);
2677 
2678     // Recursive unlock.
2679     std(recursions, in_bytes(ObjectMonitor::recursions_offset()), monitor);
2680     crorc(CCR0, Assembler::equal, CCR0, Assembler::equal);
2681     b(unlocked);
2682 
2683     bind(not_recursive);
2684 
2685     Label release_;
2686     const Register t2 = tmp2;
2687 
2688     // Check if the entry lists are empty.
2689     ld(t, in_bytes(ObjectMonitor::EntryList_offset()), monitor);
2690     ld(t2, in_bytes(ObjectMonitor::cxq_offset()), monitor);
2691     orr(t, t, t2);
2692     cmpdi(flag, t, 0);
2693     beq(flag, release_);
2694 
2695     // The owner may be anonymous and we removed the last obj entry in
2696     // the lock-stack. This loses the information about the owner.
2697     // Write the thread to the owner field so the runtime knows the owner.
2698     Register thread_id = tmp2;
2699     ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2700     std(thread_id, in_bytes(ObjectMonitor::owner_offset()), monitor);
2701     b(slow_path);
2702 
2703     bind(release_);
2704     // Set owner to null.
2705     release();
2706     // t contains 0
2707     std(t, in_bytes(ObjectMonitor::owner_offset()), monitor);
2708   }
2709 
2710   bind(unlocked);
2711   dec_held_monitor_count(t);
2712 
2713 #ifdef ASSERT
2714   // Check that unlocked label is reached with flag == EQ.
2715   Label flag_correct;
2716   beq(flag, flag_correct);
2717   stop("Fast Lock Flag != EQ");
2718 #endif
2719   bind(slow_path);
2720 #ifdef ASSERT
2721   // Check that slow_path label is reached with flag == NE.
2722   bne(flag, flag_correct);
2723   stop("Fast Lock Flag != NE");
2724   bind(flag_correct);
2725 #endif
2726   // C2 uses the value of flag (NE vs EQ) to determine the continuation.
2727 }
2728 
2729 void MacroAssembler::safepoint_poll(Label& slow_path, Register temp, bool at_return, bool in_nmethod) {
2730   ld(temp, in_bytes(JavaThread::polling_word_offset()), R16_thread);
2731 
2732   if (at_return) {
2733     if (in_nmethod) {
2734       if (UseSIGTRAP) {
2735         // Use Signal Handler.
2736         relocate(relocInfo::poll_return_type);
2737         td(traptoGreaterThanUnsigned, R1_SP, temp);
2738       } else {
2739         cmpld(CCR0, R1_SP, temp);
2740         // Stub may be out of range for short conditional branch.
2741         bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::greater), slow_path);
2742       }
2743     } else { // Not in nmethod.
2744       // Frame still on stack, need to get fp.
2745       Register fp = R0;
2746       ld(fp, _abi0(callers_sp), R1_SP);
2747       cmpld(CCR0, fp, temp);
2748       bgt(CCR0, slow_path);
2749     }
2750   } else { // Normal safepoint poll. Not at return.
2751     assert(!in_nmethod, "should use load_from_polling_page");
2752     andi_(temp, temp, SafepointMechanism::poll_bit());
2753     bne(CCR0, slow_path);
2754   }
2755 }
2756 
2757 void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp2,
2758                                      MacroAssembler::PreservationLevel preservation_level) {
2759   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2760   bs->resolve_jobject(this, value, tmp1, tmp2, preservation_level);
2761 }
2762 
2763 void MacroAssembler::resolve_global_jobject(Register value, Register tmp1, Register tmp2,
2764                                      MacroAssembler::PreservationLevel preservation_level) {
2765   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2766   bs->resolve_global_jobject(this, value, tmp1, tmp2, preservation_level);
2767 }
2768 
2769 // Values for last_Java_pc, and last_Java_sp must comply to the rules
2770 // in frame_ppc.hpp.
2771 void MacroAssembler::set_last_Java_frame(Register last_Java_sp, Register last_Java_pc) {
2772   // Always set last_Java_pc and flags first because once last_Java_sp
2773   // is visible has_last_Java_frame is true and users will look at the
2774   // rest of the fields. (Note: flags should always be zero before we
2775   // get here so doesn't need to be set.)
2776 
2777   // Verify that last_Java_pc was zeroed on return to Java
2778   asm_assert_mem8_is_zero(in_bytes(JavaThread::last_Java_pc_offset()), R16_thread,
2779                           "last_Java_pc not zeroed before leaving Java");
2780 
2781   // When returning from calling out from Java mode the frame anchor's
2782   // last_Java_pc will always be set to null. It is set here so that
2783   // if we are doing a call to native (not VM) that we capture the
2784   // known pc and don't have to rely on the native call having a
2785   // standard frame linkage where we can find the pc.
2786   if (last_Java_pc != noreg)
2787     std(last_Java_pc, in_bytes(JavaThread::last_Java_pc_offset()), R16_thread);
2788 
2789   // Set last_Java_sp last.
2790   std(last_Java_sp, in_bytes(JavaThread::last_Java_sp_offset()), R16_thread);
2791 }
2792 
2793 void MacroAssembler::reset_last_Java_frame(void) {
2794   asm_assert_mem8_isnot_zero(in_bytes(JavaThread::last_Java_sp_offset()),
2795                              R16_thread, "SP was not set, still zero");
2796 
2797   BLOCK_COMMENT("reset_last_Java_frame {");
2798   li(R0, 0);
2799 
2800   // _last_Java_sp = 0
2801   std(R0, in_bytes(JavaThread::last_Java_sp_offset()), R16_thread);
2802 
2803   // _last_Java_pc = 0
2804   std(R0, in_bytes(JavaThread::last_Java_pc_offset()), R16_thread);
2805   BLOCK_COMMENT("} reset_last_Java_frame");
2806 }
2807 
2808 void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1) {
2809   assert_different_registers(sp, tmp1);
2810 
2811   // sp points to a TOP_IJAVA_FRAME, retrieve frame's PC via
2812   // TOP_IJAVA_FRAME_ABI.
2813   // FIXME: assert that we really have a TOP_IJAVA_FRAME here!
2814   address entry = pc();
2815   load_const_optimized(tmp1, entry);
2816 
2817   set_last_Java_frame(/*sp=*/sp, /*pc=*/tmp1);
2818 }
2819 
2820 void MacroAssembler::get_vm_result(Register oop_result) {
2821   // Read:
2822   //   R16_thread
2823   //   R16_thread->in_bytes(JavaThread::vm_result_offset())
2824   //
2825   // Updated:
2826   //   oop_result
2827   //   R16_thread->in_bytes(JavaThread::vm_result_offset())
2828 
2829   ld(oop_result, in_bytes(JavaThread::vm_result_offset()), R16_thread);
2830   li(R0, 0);
2831   std(R0, in_bytes(JavaThread::vm_result_offset()), R16_thread);
2832 
2833   verify_oop(oop_result, FILE_AND_LINE);
2834 }
2835 
2836 void MacroAssembler::get_vm_result_2(Register metadata_result) {
2837   // Read:
2838   //   R16_thread
2839   //   R16_thread->in_bytes(JavaThread::vm_result_2_offset())
2840   //
2841   // Updated:
2842   //   metadata_result
2843   //   R16_thread->in_bytes(JavaThread::vm_result_2_offset())
2844 
2845   ld(metadata_result, in_bytes(JavaThread::vm_result_2_offset()), R16_thread);
2846   li(R0, 0);
2847   std(R0, in_bytes(JavaThread::vm_result_2_offset()), R16_thread);
2848 }
2849 
2850 Register MacroAssembler::encode_klass_not_null(Register dst, Register src) {
2851   Register current = (src != noreg) ? src : dst; // Klass is in dst if no src provided.
2852   if (CompressedKlassPointers::base() != 0) {
2853     // Use dst as temp if it is free.
2854     sub_const_optimized(dst, current, CompressedKlassPointers::base(), R0);
2855     current = dst;
2856   }
2857   if (CompressedKlassPointers::shift() != 0) {
2858     srdi(dst, current, CompressedKlassPointers::shift());
2859     current = dst;
2860   }
2861   return current;
2862 }
2863 
2864 void MacroAssembler::store_klass(Register dst_oop, Register klass, Register ck) {
2865   if (UseCompressedClassPointers) {
2866     Register compressedKlass = encode_klass_not_null(ck, klass);
2867     stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop);
2868   } else {
2869     std(klass, oopDesc::klass_offset_in_bytes(), dst_oop);
2870   }
2871 }
2872 
2873 void MacroAssembler::store_klass_gap(Register dst_oop, Register val) {
2874   if (UseCompressedClassPointers) {
2875     if (val == noreg) {
2876       val = R0;
2877       li(val, 0);
2878     }
2879     stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop); // klass gap if compressed
2880   }
2881 }
2882 
2883 int MacroAssembler::instr_size_for_decode_klass_not_null() {
2884   static int computed_size = -1;
2885 
2886   // Not yet computed?
2887   if (computed_size == -1) {
2888 
2889     if (!UseCompressedClassPointers) {
2890       computed_size = 0;
2891     } else {
2892       // Determine by scratch emit.
2893       ResourceMark rm;
2894       int code_size = 8 * BytesPerInstWord;
2895       CodeBuffer cb("decode_klass_not_null scratch buffer", code_size, 0);
2896       MacroAssembler* a = new MacroAssembler(&cb);
2897       a->decode_klass_not_null(R11_scratch1);
2898       computed_size = a->offset();
2899     }
2900   }
2901 
2902   return computed_size;
2903 }
2904 
2905 void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
2906   assert(dst != R0, "Dst reg may not be R0, as R0 is used here.");
2907   if (src == noreg) src = dst;
2908   Register shifted_src = src;
2909   if (CompressedKlassPointers::shift() != 0 ||
2910       (CompressedKlassPointers::base() == 0 && src != dst)) {  // Move required.
2911     shifted_src = dst;
2912     sldi(shifted_src, src, CompressedKlassPointers::shift());
2913   }
2914   if (CompressedKlassPointers::base() != 0) {
2915     add_const_optimized(dst, shifted_src, CompressedKlassPointers::base(), R0);
2916   }
2917 }
2918 
2919 void MacroAssembler::load_klass(Register dst, Register src) {
2920   if (UseCompressedClassPointers) {
2921     lwz(dst, oopDesc::klass_offset_in_bytes(), src);
2922     // Attention: no null check here!
2923     decode_klass_not_null(dst, dst);
2924   } else {
2925     ld(dst, oopDesc::klass_offset_in_bytes(), src);
2926   }
2927 }
2928 
2929 void MacroAssembler::load_klass_check_null(Register dst, Register src, Label* is_null) {
2930   null_check(src, oopDesc::klass_offset_in_bytes(), is_null);
2931   load_klass(dst, src);
2932 }
2933 
2934 // ((OopHandle)result).resolve();
2935 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2,
2936                                         MacroAssembler::PreservationLevel preservation_level) {
2937   access_load_at(T_OBJECT, IN_NATIVE, result, noreg, result, tmp1, tmp2, preservation_level);
2938 }
2939 
2940 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2,
2941                                          MacroAssembler::PreservationLevel preservation_level) {
2942   Label resolved;
2943 
2944   // A null weak handle resolves to null.
2945   cmpdi(CCR0, result, 0);
2946   beq(CCR0, resolved);
2947 
2948   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, result, noreg, result, tmp1, tmp2,
2949                  preservation_level);
2950   bind(resolved);
2951 }
2952 
2953 void MacroAssembler::load_method_holder(Register holder, Register method) {
2954   ld(holder, in_bytes(Method::const_offset()), method);
2955   ld(holder, in_bytes(ConstMethod::constants_offset()), holder);
2956   ld(holder, ConstantPool::pool_holder_offset(), holder);
2957 }
2958 
2959 // Clear Array
2960 // For very short arrays. tmp == R0 is allowed.
2961 void MacroAssembler::clear_memory_unrolled(Register base_ptr, int cnt_dwords, Register tmp, int offset) {
2962   if (cnt_dwords > 0) { li(tmp, 0); }
2963   for (int i = 0; i < cnt_dwords; ++i) { std(tmp, offset + i * 8, base_ptr); }
2964 }
2965 
2966 // Version for constant short array length. Kills base_ptr. tmp == R0 is allowed.
2967 void MacroAssembler::clear_memory_constlen(Register base_ptr, int cnt_dwords, Register tmp) {
2968   if (cnt_dwords < 8) {
2969     clear_memory_unrolled(base_ptr, cnt_dwords, tmp);
2970     return;
2971   }
2972 
2973   Label loop;
2974   const long loopcnt   = cnt_dwords >> 1,
2975              remainder = cnt_dwords & 1;
2976 
2977   li(tmp, loopcnt);
2978   mtctr(tmp);
2979   li(tmp, 0);
2980   bind(loop);
2981     std(tmp, 0, base_ptr);
2982     std(tmp, 8, base_ptr);
2983     addi(base_ptr, base_ptr, 16);
2984     bdnz(loop);
2985   if (remainder) { std(tmp, 0, base_ptr); }
2986 }
2987 
2988 // Kills both input registers. tmp == R0 is allowed.
2989 void MacroAssembler::clear_memory_doubleword(Register base_ptr, Register cnt_dwords, Register tmp, long const_cnt) {
2990   // Procedure for large arrays (uses data cache block zero instruction).
2991     Label startloop, fast, fastloop, small_rest, restloop, done;
2992     const int cl_size         = VM_Version::L1_data_cache_line_size(),
2993               cl_dwords       = cl_size >> 3,
2994               cl_dw_addr_bits = exact_log2(cl_dwords),
2995               dcbz_min        = 1,  // Min count of dcbz executions, needs to be >0.
2996               min_cnt         = ((dcbz_min + 1) << cl_dw_addr_bits) - 1;
2997 
2998   if (const_cnt >= 0) {
2999     // Constant case.
3000     if (const_cnt < min_cnt) {
3001       clear_memory_constlen(base_ptr, const_cnt, tmp);
3002       return;
3003     }
3004     load_const_optimized(cnt_dwords, const_cnt, tmp);
3005   } else {
3006     // cnt_dwords already loaded in register. Need to check size.
3007     cmpdi(CCR1, cnt_dwords, min_cnt); // Big enough? (ensure >= dcbz_min lines included).
3008     blt(CCR1, small_rest);
3009   }
3010     rldicl_(tmp, base_ptr, 64-3, 64-cl_dw_addr_bits); // Extract dword offset within first cache line.
3011     beq(CCR0, fast);                                  // Already 128byte aligned.
3012 
3013     subfic(tmp, tmp, cl_dwords);
3014     mtctr(tmp);                        // Set ctr to hit 128byte boundary (0<ctr<cl_dwords).
3015     subf(cnt_dwords, tmp, cnt_dwords); // rest.
3016     li(tmp, 0);
3017 
3018   bind(startloop);                     // Clear at the beginning to reach 128byte boundary.
3019     std(tmp, 0, base_ptr);             // Clear 8byte aligned block.
3020     addi(base_ptr, base_ptr, 8);
3021     bdnz(startloop);
3022 
3023   bind(fast);                                  // Clear 128byte blocks.
3024     srdi(tmp, cnt_dwords, cl_dw_addr_bits);    // Loop count for 128byte loop (>0).
3025     andi(cnt_dwords, cnt_dwords, cl_dwords-1); // Rest in dwords.
3026     mtctr(tmp);                                // Load counter.
3027 
3028   bind(fastloop);
3029     dcbz(base_ptr);                    // Clear 128byte aligned block.
3030     addi(base_ptr, base_ptr, cl_size);
3031     bdnz(fastloop);
3032 
3033   bind(small_rest);
3034     cmpdi(CCR0, cnt_dwords, 0);        // size 0?
3035     beq(CCR0, done);                   // rest == 0
3036     li(tmp, 0);
3037     mtctr(cnt_dwords);                 // Load counter.
3038 
3039   bind(restloop);                      // Clear rest.
3040     std(tmp, 0, base_ptr);             // Clear 8byte aligned block.
3041     addi(base_ptr, base_ptr, 8);
3042     bdnz(restloop);
3043 
3044   bind(done);
3045 }
3046 
3047 /////////////////////////////////////////// String intrinsics ////////////////////////////////////////////
3048 
3049 // Helpers for Intrinsic Emitters
3050 //
3051 // Revert the byte order of a 32bit value in a register
3052 //   src: 0x44556677
3053 //   dst: 0x77665544
3054 // Three steps to obtain the result:
3055 //  1) Rotate src (as doubleword) left 5 bytes. That puts the leftmost byte of the src word
3056 //     into the rightmost byte position. Afterwards, everything left of the rightmost byte is cleared.
3057 //     This value initializes dst.
3058 //  2) Rotate src (as word) left 3 bytes. That puts the rightmost byte of the src word into the leftmost
3059 //     byte position. Furthermore, byte 5 is rotated into byte 6 position where it is supposed to go.
3060 //     This value is mask inserted into dst with a [0..23] mask of 1s.
3061 //  3) Rotate src (as word) left 1 byte. That puts byte 6 into byte 5 position.
3062 //     This value is mask inserted into dst with a [8..15] mask of 1s.
3063 void MacroAssembler::load_reverse_32(Register dst, Register src) {
3064   assert_different_registers(dst, src);
3065 
3066   rldicl(dst, src, (4+1)*8, 56);       // Rotate byte 4 into position 7 (rightmost), clear all to the left.
3067   rlwimi(dst, src,     3*8,  0, 23);   // Insert byte 5 into position 6, 7 into 4, leave pos 7 alone.
3068   rlwimi(dst, src,     1*8,  8, 15);   // Insert byte 6 into position 5, leave the rest alone.
3069 }
3070 
3071 // Calculate the column addresses of the crc32 lookup table into distinct registers.
3072 // This loop-invariant calculation is moved out of the loop body, reducing the loop
3073 // body size from 20 to 16 instructions.
3074 // Returns the offset that was used to calculate the address of column tc3.
3075 // Due to register shortage, setting tc3 may overwrite table. With the return offset
3076 // at hand, the original table address can be easily reconstructed.
3077 int MacroAssembler::crc32_table_columns(Register table, Register tc0, Register tc1, Register tc2, Register tc3) {
3078   assert(!VM_Version::has_vpmsumb(), "Vector version should be used instead!");
3079 
3080   // Point to 4 byte folding tables (byte-reversed version for Big Endian)
3081   // Layout: See StubRoutines::ppc::generate_crc_constants.
3082 #ifdef VM_LITTLE_ENDIAN
3083   const int ix0 = 3 * CRC32_TABLE_SIZE;
3084   const int ix1 = 2 * CRC32_TABLE_SIZE;
3085   const int ix2 = 1 * CRC32_TABLE_SIZE;
3086   const int ix3 = 0 * CRC32_TABLE_SIZE;
3087 #else
3088   const int ix0 = 1 * CRC32_TABLE_SIZE;
3089   const int ix1 = 2 * CRC32_TABLE_SIZE;
3090   const int ix2 = 3 * CRC32_TABLE_SIZE;
3091   const int ix3 = 4 * CRC32_TABLE_SIZE;
3092 #endif
3093   assert_different_registers(table, tc0, tc1, tc2);
3094   assert(table == tc3, "must be!");
3095 
3096   addi(tc0, table, ix0);
3097   addi(tc1, table, ix1);
3098   addi(tc2, table, ix2);
3099   if (ix3 != 0) addi(tc3, table, ix3);
3100 
3101   return ix3;
3102 }
3103 
3104 /**
3105  * uint32_t crc;
3106  * table[crc & 0xFF] ^ (crc >> 8);
3107  */
3108 void MacroAssembler::fold_byte_crc32(Register crc, Register val, Register table, Register tmp) {
3109   assert_different_registers(crc, table, tmp);
3110   assert_different_registers(val, table);
3111 
3112   if (crc == val) {                   // Must rotate first to use the unmodified value.
3113     rlwinm(tmp, val, 2, 24-2, 31-2);  // Insert (rightmost) byte 7 of val, shifted left by 2, into byte 6..7 of tmp, clear the rest.
3114                                       // As we use a word (4-byte) instruction, we have to adapt the mask bit positions.
3115     srwi(crc, crc, 8);                // Unsigned shift, clear leftmost 8 bits.
3116   } else {
3117     srwi(crc, crc, 8);                // Unsigned shift, clear leftmost 8 bits.
3118     rlwinm(tmp, val, 2, 24-2, 31-2);  // Insert (rightmost) byte 7 of val, shifted left by 2, into byte 6..7 of tmp, clear the rest.
3119   }
3120   lwzx(tmp, table, tmp);
3121   xorr(crc, crc, tmp);
3122 }
3123 
3124 /**
3125  * Emits code to update CRC-32 with a byte value according to constants in table.
3126  *
3127  * @param [in,out]crc   Register containing the crc.
3128  * @param [in]val       Register containing the byte to fold into the CRC.
3129  * @param [in]table     Register containing the table of crc constants.
3130  *
3131  * uint32_t crc;
3132  * val = crc_table[(val ^ crc) & 0xFF];
3133  * crc = val ^ (crc >> 8);
3134  */
3135 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
3136   BLOCK_COMMENT("update_byte_crc32:");
3137   xorr(val, val, crc);
3138   fold_byte_crc32(crc, val, table, val);
3139 }
3140 
3141 /**
3142  * @param crc   register containing existing CRC (32-bit)
3143  * @param buf   register pointing to input byte buffer (byte*)
3144  * @param len   register containing number of bytes
3145  * @param table register pointing to CRC table
3146  */
3147 void MacroAssembler::update_byteLoop_crc32(Register crc, Register buf, Register len, Register table,
3148                                            Register data, bool loopAlignment) {
3149   assert_different_registers(crc, buf, len, table, data);
3150 
3151   Label L_mainLoop, L_done;
3152   const int mainLoop_stepping  = 1;
3153   const int mainLoop_alignment = loopAlignment ? 32 : 4; // (InputForNewCode > 4 ? InputForNewCode : 32) : 4;
3154 
3155   // Process all bytes in a single-byte loop.
3156   clrldi_(len, len, 32);                         // Enforce 32 bit. Anything to do?
3157   beq(CCR0, L_done);
3158 
3159   mtctr(len);
3160   align(mainLoop_alignment);
3161   BIND(L_mainLoop);
3162     lbz(data, 0, buf);                           // Byte from buffer, zero-extended.
3163     addi(buf, buf, mainLoop_stepping);           // Advance buffer position.
3164     update_byte_crc32(crc, data, table);
3165     bdnz(L_mainLoop);                            // Iterate.
3166 
3167   bind(L_done);
3168 }
3169 
3170 /**
3171  * Emits code to update CRC-32 with a 4-byte value according to constants in table
3172  * Implementation according to jdk/src/share/native/java/util/zip/zlib-1.2.8/crc32.c
3173  */
3174 // A note on the lookup table address(es):
3175 // The implementation uses 4 table columns (byte-reversed versions for Big Endian).
3176 // To save the effort of adding the column offset to the table address each time
3177 // a table element is looked up, it is possible to pass the pre-calculated
3178 // column addresses.
3179 // Uses R9..R12 as work register. Must be saved/restored by caller, if necessary.
3180 void MacroAssembler::update_1word_crc32(Register crc, Register buf, Register table, int bufDisp, int bufInc,
3181                                         Register t0,  Register t1,  Register t2,  Register t3,
3182                                         Register tc0, Register tc1, Register tc2, Register tc3) {
3183   assert_different_registers(crc, t3);
3184 
3185   // XOR crc with next four bytes of buffer.
3186   lwz(t3, bufDisp, buf);
3187   if (bufInc != 0) {
3188     addi(buf, buf, bufInc);
3189   }
3190   xorr(t3, t3, crc);
3191 
3192   // Chop crc into 4 single-byte pieces, shifted left 2 bits, to form the table indices.
3193   rlwinm(t0, t3,  2,         24-2, 31-2);  // ((t1 >>  0) & 0xff) << 2
3194   rlwinm(t1, t3,  32+(2- 8), 24-2, 31-2);  // ((t1 >>  8) & 0xff) << 2
3195   rlwinm(t2, t3,  32+(2-16), 24-2, 31-2);  // ((t1 >> 16) & 0xff) << 2
3196   rlwinm(t3, t3,  32+(2-24), 24-2, 31-2);  // ((t1 >> 24) & 0xff) << 2
3197 
3198   // Use the pre-calculated column addresses.
3199   // Load pre-calculated table values.
3200   lwzx(t0, tc0, t0);
3201   lwzx(t1, tc1, t1);
3202   lwzx(t2, tc2, t2);
3203   lwzx(t3, tc3, t3);
3204 
3205   // Calculate new crc from table values.
3206   xorr(t0,  t0, t1);
3207   xorr(t2,  t2, t3);
3208   xorr(crc, t0, t2);  // Now crc contains the final checksum value.
3209 }
3210 
3211 /**
3212  * @param crc   register containing existing CRC (32-bit)
3213  * @param buf   register pointing to input byte buffer (byte*)
3214  * @param len   register containing number of bytes
3215  * @param table register pointing to CRC table
3216  *
3217  * uses R9..R12 as work register. Must be saved/restored by caller!
3218  */
3219 void MacroAssembler::kernel_crc32_1word(Register crc, Register buf, Register len, Register table,
3220                                         Register t0,  Register t1,  Register t2,  Register t3,
3221                                         Register tc0, Register tc1, Register tc2, Register tc3,
3222                                         bool invertCRC) {
3223   assert_different_registers(crc, buf, len, table);
3224 
3225   Label L_mainLoop, L_tail;
3226   Register  tmp          = t0;
3227   Register  data         = t0;
3228   Register  tmp2         = t1;
3229   const int mainLoop_stepping  = 4;
3230   const int tailLoop_stepping  = 1;
3231   const int log_stepping       = exact_log2(mainLoop_stepping);
3232   const int mainLoop_alignment = 32; // InputForNewCode > 4 ? InputForNewCode : 32;
3233   const int complexThreshold   = 2*mainLoop_stepping;
3234 
3235   // Don't test for len <= 0 here. This pathological case should not occur anyway.
3236   // Optimizing for it by adding a test and a branch seems to be a waste of CPU cycles
3237   // for all well-behaved cases. The situation itself is detected and handled correctly
3238   // within update_byteLoop_crc32.
3239   assert(tailLoop_stepping == 1, "check tailLoop_stepping!");
3240 
3241   BLOCK_COMMENT("kernel_crc32_1word {");
3242 
3243   if (invertCRC) {
3244     nand(crc, crc, crc);                      // 1s complement of crc
3245   }
3246 
3247   // Check for short (<mainLoop_stepping) buffer.
3248   cmpdi(CCR0, len, complexThreshold);
3249   blt(CCR0, L_tail);
3250 
3251   // Pre-mainLoop alignment did show a slight (1%) positive effect on performance.
3252   // We leave the code in for reference. Maybe we need alignment when we exploit vector instructions.
3253   {
3254     // Align buf addr to mainLoop_stepping boundary.
3255     neg(tmp2, buf);                              // Calculate # preLoop iterations for alignment.
3256     rldicl(tmp2, tmp2, 0, 64-log_stepping);      // Rotate tmp2 0 bits, insert into tmp2, anding with mask with 1s from 62..63.
3257 
3258     if (complexThreshold > mainLoop_stepping) {
3259       sub(len, len, tmp2);                       // Remaining bytes for main loop (>=mainLoop_stepping is guaranteed).
3260     } else {
3261       sub(tmp, len, tmp2);                       // Remaining bytes for main loop.
3262       cmpdi(CCR0, tmp, mainLoop_stepping);
3263       blt(CCR0, L_tail);                         // For less than one mainloop_stepping left, do only tail processing
3264       mr(len, tmp);                              // remaining bytes for main loop (>=mainLoop_stepping is guaranteed).
3265     }
3266     update_byteLoop_crc32(crc, buf, tmp2, table, data, false);
3267   }
3268 
3269   srdi(tmp2, len, log_stepping);                 // #iterations for mainLoop
3270   andi(len, len, mainLoop_stepping-1);           // remaining bytes for tailLoop
3271   mtctr(tmp2);
3272 
3273 #ifdef VM_LITTLE_ENDIAN
3274   Register crc_rv = crc;
3275 #else
3276   Register crc_rv = tmp;                         // Load_reverse needs separate registers to work on.
3277                                                  // Occupies tmp, but frees up crc.
3278   load_reverse_32(crc_rv, crc);                  // Revert byte order because we are dealing with big-endian data.
3279   tmp = crc;
3280 #endif
3281 
3282   int reconstructTableOffset = crc32_table_columns(table, tc0, tc1, tc2, tc3);
3283 
3284   align(mainLoop_alignment);                     // Octoword-aligned loop address. Shows 2% improvement.
3285   BIND(L_mainLoop);
3286     update_1word_crc32(crc_rv, buf, table, 0, mainLoop_stepping, crc_rv, t1, t2, t3, tc0, tc1, tc2, tc3);
3287     bdnz(L_mainLoop);
3288 
3289 #ifndef VM_LITTLE_ENDIAN
3290   load_reverse_32(crc, crc_rv);                  // Revert byte order because we are dealing with big-endian data.
3291   tmp = crc_rv;                                  // Tmp uses it's original register again.
3292 #endif
3293 
3294   // Restore original table address for tailLoop.
3295   if (reconstructTableOffset != 0) {
3296     addi(table, table, -reconstructTableOffset);
3297   }
3298 
3299   // Process last few (<complexThreshold) bytes of buffer.
3300   BIND(L_tail);
3301   update_byteLoop_crc32(crc, buf, len, table, data, false);
3302 
3303   if (invertCRC) {
3304     nand(crc, crc, crc);                      // 1s complement of crc
3305   }
3306   BLOCK_COMMENT("} kernel_crc32_1word");
3307 }
3308 
3309 /**
3310  * @param crc             register containing existing CRC (32-bit)
3311  * @param buf             register pointing to input byte buffer (byte*)
3312  * @param len             register containing number of bytes
3313  * @param constants       register pointing to precomputed constants
3314  * @param t0-t6           temp registers
3315  */
3316 void MacroAssembler::kernel_crc32_vpmsum(Register crc, Register buf, Register len, Register constants,
3317                                          Register t0, Register t1, Register t2, Register t3,
3318                                          Register t4, Register t5, Register t6, bool invertCRC) {
3319   assert_different_registers(crc, buf, len, constants);
3320 
3321   Label L_tail;
3322 
3323   BLOCK_COMMENT("kernel_crc32_vpmsum {");
3324 
3325   if (invertCRC) {
3326     nand(crc, crc, crc);                      // 1s complement of crc
3327   }
3328 
3329   // Enforce 32 bit.
3330   clrldi(len, len, 32);
3331 
3332   // Align if we have enough bytes for the fast version.
3333   const int alignment = 16,
3334             threshold = 32;
3335   Register prealign = t0;
3336 
3337   neg(prealign, buf);
3338   addi(t1, len, -threshold);
3339   andi(prealign, prealign, alignment - 1);
3340   cmpw(CCR0, t1, prealign);
3341   blt(CCR0, L_tail); // len - prealign < threshold?
3342 
3343   subf(len, prealign, len);
3344   update_byteLoop_crc32(crc, buf, prealign, constants, t2, false);
3345 
3346   // Calculate from first aligned address as far as possible.
3347   addi(constants, constants, CRC32_TABLE_SIZE); // Point to vector constants.
3348   kernel_crc32_vpmsum_aligned(crc, buf, len, constants, t0, t1, t2, t3, t4, t5, t6);
3349   addi(constants, constants, -CRC32_TABLE_SIZE); // Point to table again.
3350 
3351   // Remaining bytes.
3352   BIND(L_tail);
3353   update_byteLoop_crc32(crc, buf, len, constants, t2, false);
3354 
3355   if (invertCRC) {
3356     nand(crc, crc, crc);                      // 1s complement of crc
3357   }
3358 
3359   BLOCK_COMMENT("} kernel_crc32_vpmsum");
3360 }
3361 
3362 /**
3363  * @param crc             register containing existing CRC (32-bit)
3364  * @param buf             register pointing to input byte buffer (byte*)
3365  * @param len             register containing number of bytes (will get updated to remaining bytes)
3366  * @param constants       register pointing to CRC table for 128-bit aligned memory
3367  * @param t0-t6           temp registers
3368  */
3369 void MacroAssembler::kernel_crc32_vpmsum_aligned(Register crc, Register buf, Register len, Register constants,
3370     Register t0, Register t1, Register t2, Register t3, Register t4, Register t5, Register t6) {
3371 
3372   // Save non-volatile vector registers (frameless).
3373   Register offset = t1;
3374   int offsetInt = 0;
3375   offsetInt -= 16; li(offset, offsetInt); stvx(VR20, offset, R1_SP);
3376   offsetInt -= 16; li(offset, offsetInt); stvx(VR21, offset, R1_SP);
3377   offsetInt -= 16; li(offset, offsetInt); stvx(VR22, offset, R1_SP);
3378   offsetInt -= 16; li(offset, offsetInt); stvx(VR23, offset, R1_SP);
3379   offsetInt -= 16; li(offset, offsetInt); stvx(VR24, offset, R1_SP);
3380   offsetInt -= 16; li(offset, offsetInt); stvx(VR25, offset, R1_SP);
3381 #ifndef VM_LITTLE_ENDIAN
3382   offsetInt -= 16; li(offset, offsetInt); stvx(VR26, offset, R1_SP);
3383 #endif
3384   offsetInt -= 8; std(R14, offsetInt, R1_SP);
3385   offsetInt -= 8; std(R15, offsetInt, R1_SP);
3386 
3387   // Implementation uses an inner loop which uses between 256 and 16 * unroll_factor
3388   // bytes per iteration. The basic scheme is:
3389   // lvx: load vector (Big Endian needs reversal)
3390   // vpmsumw: carry-less 32 bit multiplications with constant representing a large CRC shift
3391   // vxor: xor partial results together to get unroll_factor2 vectors
3392 
3393   // Outer loop performs the CRC shifts needed to combine the unroll_factor2 vectors.
3394 
3395   // Using 16 * unroll_factor / unroll_factor_2 bytes for constants.
3396   const int unroll_factor = CRC32_UNROLL_FACTOR,
3397             unroll_factor2 = CRC32_UNROLL_FACTOR2;
3398 
3399   const int outer_consts_size = (unroll_factor2 - 1) * 16,
3400             inner_consts_size = (unroll_factor / unroll_factor2) * 16;
3401 
3402   // Support registers.
3403   Register offs[] = { noreg, t0, t1, t2, t3, t4, t5, t6 };
3404   Register num_bytes = R14,
3405            loop_count = R15,
3406            cur_const = crc; // will live in VCRC
3407   // Constant array for outer loop: unroll_factor2 - 1 registers,
3408   // Constant array for inner loop: unroll_factor / unroll_factor2 registers.
3409   VectorRegister consts0[] = { VR16, VR17, VR18, VR19, VR20, VR21, VR22 },
3410                  consts1[] = { VR23, VR24 };
3411   // Data register arrays: 2 arrays with unroll_factor2 registers.
3412   VectorRegister data0[] = { VR0, VR1, VR2, VR3, VR4, VR5, VR6, VR7 },
3413                  data1[] = { VR8, VR9, VR10, VR11, VR12, VR13, VR14, VR15 };
3414 
3415   VectorRegister VCRC = data0[0];
3416   VectorRegister Vc = VR25;
3417   VectorRegister swap_bytes = VR26; // Only for Big Endian.
3418 
3419   // We have at least 1 iteration (ensured by caller).
3420   Label L_outer_loop, L_inner_loop, L_last;
3421 
3422   // If supported set DSCR pre-fetch to deepest.
3423   if (VM_Version::has_mfdscr()) {
3424     load_const_optimized(t0, VM_Version::_dscr_val | 7);
3425     mtdscr(t0);
3426   }
3427 
3428   mtvrwz(VCRC, crc); // crc lives in VCRC, now
3429 
3430   for (int i = 1; i < unroll_factor2; ++i) {
3431     li(offs[i], 16 * i);
3432   }
3433 
3434   // Load consts for outer loop
3435   lvx(consts0[0], constants);
3436   for (int i = 1; i < unroll_factor2 - 1; ++i) {
3437     lvx(consts0[i], offs[i], constants);
3438   }
3439 
3440   load_const_optimized(num_bytes, 16 * unroll_factor);
3441 
3442   // Reuse data registers outside of the loop.
3443   VectorRegister Vtmp = data1[0];
3444   VectorRegister Vtmp2 = data1[1];
3445   VectorRegister zeroes = data1[2];
3446 
3447   vspltisb(Vtmp, 0);
3448   vsldoi(VCRC, Vtmp, VCRC, 8); // 96 bit zeroes, 32 bit CRC.
3449 
3450   // Load vector for vpermxor (to xor both 64 bit parts together)
3451   lvsl(Vtmp, buf);   // 000102030405060708090a0b0c0d0e0f
3452   vspltisb(Vc, 4);
3453   vsl(Vc, Vtmp, Vc); // 00102030405060708090a0b0c0d0e0f0
3454   xxspltd(Vc->to_vsr(), Vc->to_vsr(), 0);
3455   vor(Vc, Vtmp, Vc); // 001122334455667708192a3b4c5d6e7f
3456 
3457 #ifdef VM_LITTLE_ENDIAN
3458 #define BE_swap_bytes(x)
3459 #else
3460   vspltisb(Vtmp2, 0xf);
3461   vxor(swap_bytes, Vtmp, Vtmp2);
3462 #define BE_swap_bytes(x) vperm(x, x, x, swap_bytes)
3463 #endif
3464 
3465   cmpd(CCR0, len, num_bytes);
3466   blt(CCR0, L_last);
3467 
3468   addi(cur_const, constants, outer_consts_size); // Point to consts for inner loop
3469   load_const_optimized(loop_count, unroll_factor / (2 * unroll_factor2) - 1); // One double-iteration peeled off.
3470 
3471   // ********** Main loop start **********
3472   align(32);
3473   bind(L_outer_loop);
3474 
3475   // Begin of unrolled first iteration (no xor).
3476   lvx(data1[0], buf);
3477   for (int i = 1; i < unroll_factor2 / 2; ++i) {
3478     lvx(data1[i], offs[i], buf);
3479   }
3480   vpermxor(VCRC, VCRC, VCRC, Vc); // xor both halves to 64 bit result.
3481   lvx(consts1[0], cur_const);
3482   mtctr(loop_count);
3483   for (int i = 0; i < unroll_factor2 / 2; ++i) {
3484     BE_swap_bytes(data1[i]);
3485     if (i == 0) { vxor(data1[0], data1[0], VCRC); } // xor in previous CRC.
3486     lvx(data1[i + unroll_factor2 / 2], offs[i + unroll_factor2 / 2], buf);
3487     vpmsumw(data0[i], data1[i], consts1[0]);
3488   }
3489   addi(buf, buf, 16 * unroll_factor2);
3490   subf(len, num_bytes, len);
3491   lvx(consts1[1], offs[1], cur_const);
3492   addi(cur_const, cur_const, 32);
3493   // Begin of unrolled second iteration (head).
3494   for (int i = 0; i < unroll_factor2 / 2; ++i) {
3495     BE_swap_bytes(data1[i + unroll_factor2 / 2]);
3496     if (i == 0) { lvx(data1[0], buf); } else { lvx(data1[i], offs[i], buf); }
3497     vpmsumw(data0[i + unroll_factor2 / 2], data1[i + unroll_factor2 / 2], consts1[0]);
3498   }
3499   for (int i = 0; i < unroll_factor2 / 2; ++i) {
3500     BE_swap_bytes(data1[i]);
3501     lvx(data1[i + unroll_factor2 / 2], offs[i + unroll_factor2 / 2], buf);
3502     vpmsumw(data1[i], data1[i], consts1[1]);
3503   }
3504   addi(buf, buf, 16 * unroll_factor2);
3505 
3506   // Generate most performance relevant code. Loads + half of the vpmsumw have been generated.
3507   // Double-iteration allows using the 2 constant registers alternatingly.
3508   align(32);
3509   bind(L_inner_loop);
3510   for (int j = 1; j < 3; ++j) { // j < unroll_factor / unroll_factor2 - 1 for complete unrolling.
3511     if (j & 1) {
3512       lvx(consts1[0], cur_const);
3513     } else {
3514       lvx(consts1[1], offs[1], cur_const);
3515       addi(cur_const, cur_const, 32);
3516     }
3517     for (int i = 0; i < unroll_factor2; ++i) {
3518       int idx = i + unroll_factor2 / 2, inc = 0; // For modulo-scheduled input.
3519       if (idx >= unroll_factor2) { idx -= unroll_factor2; inc = 1; }
3520       BE_swap_bytes(data1[idx]);
3521       vxor(data0[i], data0[i], data1[i]);
3522       if (i == 0) lvx(data1[0], buf); else lvx(data1[i], offs[i], buf);
3523       vpmsumw(data1[idx], data1[idx], consts1[(j + inc) & 1]);
3524     }
3525     addi(buf, buf, 16 * unroll_factor2);
3526   }
3527   bdnz(L_inner_loop);
3528 
3529   addi(cur_const, constants, outer_consts_size); // Reset
3530 
3531   // Tail of last iteration (no loads).
3532   for (int i = 0; i < unroll_factor2 / 2; ++i) {
3533     BE_swap_bytes(data1[i + unroll_factor2 / 2]);
3534     vxor(data0[i], data0[i], data1[i]);
3535     vpmsumw(data1[i + unroll_factor2 / 2], data1[i + unroll_factor2 / 2], consts1[1]);
3536   }
3537   for (int i = 0; i < unroll_factor2 / 2; ++i) {
3538     vpmsumw(data0[i], data0[i], consts0[unroll_factor2 - 2 - i]); // First half of fixup shifts.
3539     vxor(data0[i + unroll_factor2 / 2], data0[i + unroll_factor2 / 2], data1[i + unroll_factor2 / 2]);
3540   }
3541 
3542   // Last data register is ok, other ones need fixup shift.
3543   for (int i = unroll_factor2 / 2; i < unroll_factor2 - 1; ++i) {
3544     vpmsumw(data0[i], data0[i], consts0[unroll_factor2 - 2 - i]);
3545   }
3546 
3547   // Combine to 128 bit result vector VCRC = data0[0].
3548   for (int i = 1; i < unroll_factor2; i<<=1) {
3549     for (int j = 0; j <= unroll_factor2 - 2*i; j+=2*i) {
3550       vxor(data0[j], data0[j], data0[j+i]);
3551     }
3552   }
3553   cmpd(CCR0, len, num_bytes);
3554   bge(CCR0, L_outer_loop);
3555 
3556   // Last chance with lower num_bytes.
3557   bind(L_last);
3558   srdi(loop_count, len, exact_log2(16 * 2 * unroll_factor2)); // Use double-iterations.
3559   // Point behind last const for inner loop.
3560   add_const_optimized(cur_const, constants, outer_consts_size + inner_consts_size);
3561   sldi(R0, loop_count, exact_log2(16 * 2)); // Bytes of constants to be used.
3562   clrrdi(num_bytes, len, exact_log2(16 * 2 * unroll_factor2));
3563   subf(cur_const, R0, cur_const); // Point to constant to be used first.
3564 
3565   addic_(loop_count, loop_count, -1); // One double-iteration peeled off.
3566   bgt(CCR0, L_outer_loop);
3567   // ********** Main loop end **********
3568 
3569   // Restore DSCR pre-fetch value.
3570   if (VM_Version::has_mfdscr()) {
3571     load_const_optimized(t0, VM_Version::_dscr_val);
3572     mtdscr(t0);
3573   }
3574 
3575   // ********** Simple loop for remaining 16 byte blocks **********
3576   {
3577     Label L_loop, L_done;
3578 
3579     srdi_(t0, len, 4); // 16 bytes per iteration
3580     clrldi(len, len, 64-4);
3581     beq(CCR0, L_done);
3582 
3583     // Point to const (same as last const for inner loop).
3584     add_const_optimized(cur_const, constants, outer_consts_size + inner_consts_size - 16);
3585     mtctr(t0);
3586     lvx(Vtmp2, cur_const);
3587 
3588     align(32);
3589     bind(L_loop);
3590 
3591     lvx(Vtmp, buf);
3592     addi(buf, buf, 16);
3593     vpermxor(VCRC, VCRC, VCRC, Vc); // xor both halves to 64 bit result.
3594     BE_swap_bytes(Vtmp);
3595     vxor(VCRC, VCRC, Vtmp);
3596     vpmsumw(VCRC, VCRC, Vtmp2);
3597     bdnz(L_loop);
3598 
3599     bind(L_done);
3600   }
3601   // ********** Simple loop end **********
3602 #undef BE_swap_bytes
3603 
3604   // Point to Barrett constants
3605   add_const_optimized(cur_const, constants, outer_consts_size + inner_consts_size);
3606 
3607   vspltisb(zeroes, 0);
3608 
3609   // Combine to 64 bit result.
3610   vpermxor(VCRC, VCRC, VCRC, Vc); // xor both halves to 64 bit result.
3611 
3612   // Reduce to 32 bit CRC: Remainder by multiply-high.
3613   lvx(Vtmp, cur_const);
3614   vsldoi(Vtmp2, zeroes, VCRC, 12);  // Extract high 32 bit.
3615   vpmsumd(Vtmp2, Vtmp2, Vtmp);      // Multiply by inverse long poly.
3616   vsldoi(Vtmp2, zeroes, Vtmp2, 12); // Extract high 32 bit.
3617   vsldoi(Vtmp, zeroes, Vtmp, 8);
3618   vpmsumd(Vtmp2, Vtmp2, Vtmp);      // Multiply quotient by long poly.
3619   vxor(VCRC, VCRC, Vtmp2);          // Remainder fits into 32 bit.
3620 
3621   // Move result. len is already updated.
3622   vsldoi(VCRC, VCRC, zeroes, 8);
3623   mfvrd(crc, VCRC);
3624 
3625   // Restore non-volatile Vector registers (frameless).
3626   offsetInt = 0;
3627   offsetInt -= 16; li(offset, offsetInt); lvx(VR20, offset, R1_SP);
3628   offsetInt -= 16; li(offset, offsetInt); lvx(VR21, offset, R1_SP);
3629   offsetInt -= 16; li(offset, offsetInt); lvx(VR22, offset, R1_SP);
3630   offsetInt -= 16; li(offset, offsetInt); lvx(VR23, offset, R1_SP);
3631   offsetInt -= 16; li(offset, offsetInt); lvx(VR24, offset, R1_SP);
3632   offsetInt -= 16; li(offset, offsetInt); lvx(VR25, offset, R1_SP);
3633 #ifndef VM_LITTLE_ENDIAN
3634   offsetInt -= 16; li(offset, offsetInt); lvx(VR26, offset, R1_SP);
3635 #endif
3636   offsetInt -= 8;  ld(R14, offsetInt, R1_SP);
3637   offsetInt -= 8;  ld(R15, offsetInt, R1_SP);
3638 }
3639 
3640 void MacroAssembler::crc32(Register crc, Register buf, Register len, Register t0, Register t1, Register t2,
3641                            Register t3, Register t4, Register t5, Register t6, Register t7, bool is_crc32c) {
3642   load_const_optimized(t0, is_crc32c ? StubRoutines::crc32c_table_addr()
3643                                      : StubRoutines::crc_table_addr()   , R0);
3644 
3645   if (VM_Version::has_vpmsumb()) {
3646     kernel_crc32_vpmsum(crc, buf, len, t0, t1, t2, t3, t4, t5, t6, t7, !is_crc32c);
3647   } else {
3648     kernel_crc32_1word(crc, buf, len, t0, t1, t2, t3, t4, t5, t6, t7, t0, !is_crc32c);
3649   }
3650 }
3651 
3652 void MacroAssembler::kernel_crc32_singleByteReg(Register crc, Register val, Register table, bool invertCRC) {
3653   assert_different_registers(crc, val, table);
3654 
3655   BLOCK_COMMENT("kernel_crc32_singleByteReg:");
3656   if (invertCRC) {
3657     nand(crc, crc, crc);                // 1s complement of crc
3658   }
3659 
3660   update_byte_crc32(crc, val, table);
3661 
3662   if (invertCRC) {
3663     nand(crc, crc, crc);                // 1s complement of crc
3664   }
3665 }
3666 
3667 // dest_lo += src1 + src2
3668 // dest_hi += carry1 + carry2
3669 void MacroAssembler::add2_with_carry(Register dest_hi,
3670                                      Register dest_lo,
3671                                      Register src1, Register src2) {
3672   li(R0, 0);
3673   addc(dest_lo, dest_lo, src1);
3674   adde(dest_hi, dest_hi, R0);
3675   addc(dest_lo, dest_lo, src2);
3676   adde(dest_hi, dest_hi, R0);
3677 }
3678 
3679 // Multiply 64 bit by 64 bit first loop.
3680 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart,
3681                                            Register x_xstart,
3682                                            Register y, Register y_idx,
3683                                            Register z,
3684                                            Register carry,
3685                                            Register product_high, Register product,
3686                                            Register idx, Register kdx,
3687                                            Register tmp) {
3688   //  jlong carry, x[], y[], z[];
3689   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) {
3690   //    huge_128 product = y[idx] * x[xstart] + carry;
3691   //    z[kdx] = (jlong)product;
3692   //    carry  = (jlong)(product >>> 64);
3693   //  }
3694   //  z[xstart] = carry;
3695 
3696   Label L_first_loop, L_first_loop_exit;
3697   Label L_one_x, L_one_y, L_multiply;
3698 
3699   addic_(xstart, xstart, -1);
3700   blt(CCR0, L_one_x);   // Special case: length of x is 1.
3701 
3702   // Load next two integers of x.
3703   sldi(tmp, xstart, LogBytesPerInt);
3704   ldx(x_xstart, x, tmp);
3705 #ifdef VM_LITTLE_ENDIAN
3706   rldicl(x_xstart, x_xstart, 32, 0);
3707 #endif
3708 
3709   align(32, 16);
3710   bind(L_first_loop);
3711 
3712   cmpdi(CCR0, idx, 1);
3713   blt(CCR0, L_first_loop_exit);
3714   addi(idx, idx, -2);
3715   beq(CCR0, L_one_y);
3716 
3717   // Load next two integers of y.
3718   sldi(tmp, idx, LogBytesPerInt);
3719   ldx(y_idx, y, tmp);
3720 #ifdef VM_LITTLE_ENDIAN
3721   rldicl(y_idx, y_idx, 32, 0);
3722 #endif
3723 
3724 
3725   bind(L_multiply);
3726   multiply64(product_high, product, x_xstart, y_idx);
3727 
3728   li(tmp, 0);
3729   addc(product, product, carry);         // Add carry to result.
3730   adde(product_high, product_high, tmp); // Add carry of the last addition.
3731   addi(kdx, kdx, -2);
3732 
3733   // Store result.
3734 #ifdef VM_LITTLE_ENDIAN
3735   rldicl(product, product, 32, 0);
3736 #endif
3737   sldi(tmp, kdx, LogBytesPerInt);
3738   stdx(product, z, tmp);
3739   mr_if_needed(carry, product_high);
3740   b(L_first_loop);
3741 
3742 
3743   bind(L_one_y); // Load one 32 bit portion of y as (0,value).
3744 
3745   lwz(y_idx, 0, y);
3746   b(L_multiply);
3747 
3748 
3749   bind(L_one_x); // Load one 32 bit portion of x as (0,value).
3750 
3751   lwz(x_xstart, 0, x);
3752   b(L_first_loop);
3753 
3754   bind(L_first_loop_exit);
3755 }
3756 
3757 // Multiply 64 bit by 64 bit and add 128 bit.
3758 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y,
3759                                             Register z, Register yz_idx,
3760                                             Register idx, Register carry,
3761                                             Register product_high, Register product,
3762                                             Register tmp, int offset) {
3763 
3764   //  huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
3765   //  z[kdx] = (jlong)product;
3766 
3767   sldi(tmp, idx, LogBytesPerInt);
3768   if (offset) {
3769     addi(tmp, tmp, offset);
3770   }
3771   ldx(yz_idx, y, tmp);
3772 #ifdef VM_LITTLE_ENDIAN
3773   rldicl(yz_idx, yz_idx, 32, 0);
3774 #endif
3775 
3776   multiply64(product_high, product, x_xstart, yz_idx);
3777   ldx(yz_idx, z, tmp);
3778 #ifdef VM_LITTLE_ENDIAN
3779   rldicl(yz_idx, yz_idx, 32, 0);
3780 #endif
3781 
3782   add2_with_carry(product_high, product, carry, yz_idx);
3783 
3784   sldi(tmp, idx, LogBytesPerInt);
3785   if (offset) {
3786     addi(tmp, tmp, offset);
3787   }
3788 #ifdef VM_LITTLE_ENDIAN
3789   rldicl(product, product, 32, 0);
3790 #endif
3791   stdx(product, z, tmp);
3792 }
3793 
3794 // Multiply 128 bit by 128 bit. Unrolled inner loop.
3795 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart,
3796                                              Register y, Register z,
3797                                              Register yz_idx, Register idx, Register carry,
3798                                              Register product_high, Register product,
3799                                              Register carry2, Register tmp) {
3800 
3801   //  jlong carry, x[], y[], z[];
3802   //  int kdx = ystart+1;
3803   //  for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
3804   //    huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
3805   //    z[kdx+idx+1] = (jlong)product;
3806   //    jlong carry2 = (jlong)(product >>> 64);
3807   //    product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
3808   //    z[kdx+idx] = (jlong)product;
3809   //    carry = (jlong)(product >>> 64);
3810   //  }
3811   //  idx += 2;
3812   //  if (idx > 0) {
3813   //    product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
3814   //    z[kdx+idx] = (jlong)product;
3815   //    carry = (jlong)(product >>> 64);
3816   //  }
3817 
3818   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
3819   const Register jdx = R0;
3820 
3821   // Scale the index.
3822   srdi_(jdx, idx, 2);
3823   beq(CCR0, L_third_loop_exit);
3824   mtctr(jdx);
3825 
3826   align(32, 16);
3827   bind(L_third_loop);
3828 
3829   addi(idx, idx, -4);
3830 
3831   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product_high, product, tmp, 8);
3832   mr_if_needed(carry2, product_high);
3833 
3834   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product_high, product, tmp, 0);
3835   mr_if_needed(carry, product_high);
3836   bdnz(L_third_loop);
3837 
3838   bind(L_third_loop_exit);  // Handle any left-over operand parts.
3839 
3840   andi_(idx, idx, 0x3);
3841   beq(CCR0, L_post_third_loop_done);
3842 
3843   Label L_check_1;
3844 
3845   addic_(idx, idx, -2);
3846   blt(CCR0, L_check_1);
3847 
3848   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product_high, product, tmp, 0);
3849   mr_if_needed(carry, product_high);
3850 
3851   bind(L_check_1);
3852 
3853   addi(idx, idx, 0x2);
3854   andi_(idx, idx, 0x1);
3855   addic_(idx, idx, -1);
3856   blt(CCR0, L_post_third_loop_done);
3857 
3858   sldi(tmp, idx, LogBytesPerInt);
3859   lwzx(yz_idx, y, tmp);
3860   multiply64(product_high, product, x_xstart, yz_idx);
3861   lwzx(yz_idx, z, tmp);
3862 
3863   add2_with_carry(product_high, product, yz_idx, carry);
3864 
3865   sldi(tmp, idx, LogBytesPerInt);
3866   stwx(product, z, tmp);
3867   srdi(product, product, 32);
3868 
3869   sldi(product_high, product_high, 32);
3870   orr(product, product, product_high);
3871   mr_if_needed(carry, product);
3872 
3873   bind(L_post_third_loop_done);
3874 }   // multiply_128_x_128_loop
3875 
3876 void MacroAssembler::muladd(Register out, Register in,
3877                             Register offset, Register len, Register k,
3878                             Register tmp1, Register tmp2, Register carry) {
3879 
3880   // Labels
3881   Label LOOP, SKIP;
3882 
3883   // Make sure length is positive.
3884   cmpdi  (CCR0,    len,     0);
3885 
3886   // Prepare variables
3887   subi   (offset,  offset,  4);
3888   li     (carry,   0);
3889   ble    (CCR0,    SKIP);
3890 
3891   mtctr  (len);
3892   subi   (len,     len,     1    );
3893   sldi   (len,     len,     2    );
3894 
3895   // Main loop
3896   bind(LOOP);
3897   lwzx   (tmp1,    len,     in   );
3898   lwzx   (tmp2,    offset,  out  );
3899   mulld  (tmp1,    tmp1,    k    );
3900   add    (tmp2,    carry,   tmp2 );
3901   add    (tmp2,    tmp1,    tmp2 );
3902   stwx   (tmp2,    offset,  out  );
3903   srdi   (carry,   tmp2,    32   );
3904   subi   (offset,  offset,  4    );
3905   subi   (len,     len,     4    );
3906   bdnz   (LOOP);
3907   bind(SKIP);
3908 }
3909 
3910 void MacroAssembler::multiply_to_len(Register x, Register xlen,
3911                                      Register y, Register ylen,
3912                                      Register z, Register zlen,
3913                                      Register tmp1, Register tmp2,
3914                                      Register tmp3, Register tmp4,
3915                                      Register tmp5, Register tmp6,
3916                                      Register tmp7, Register tmp8,
3917                                      Register tmp9, Register tmp10,
3918                                      Register tmp11, Register tmp12,
3919                                      Register tmp13) {
3920 
3921   ShortBranchVerifier sbv(this);
3922 
3923   assert_different_registers(x, xlen, y, ylen, z, zlen,
3924                              tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
3925   assert_different_registers(x, xlen, y, ylen, z, zlen,
3926                              tmp1, tmp2, tmp3, tmp4, tmp5, tmp7);
3927   assert_different_registers(x, xlen, y, ylen, z, zlen,
3928                              tmp1, tmp2, tmp3, tmp4, tmp5, tmp8);
3929 
3930   const Register idx = tmp1;
3931   const Register kdx = tmp2;
3932   const Register xstart = tmp3;
3933 
3934   const Register y_idx = tmp4;
3935   const Register carry = tmp5;
3936   const Register product = tmp6;
3937   const Register product_high = tmp7;
3938   const Register x_xstart = tmp8;
3939   const Register tmp = tmp9;
3940 
3941   // First Loop.
3942   //
3943   //  final static long LONG_MASK = 0xffffffffL;
3944   //  int xstart = xlen - 1;
3945   //  int ystart = ylen - 1;
3946   //  long carry = 0;
3947   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
3948   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
3949   //    z[kdx] = (int)product;
3950   //    carry = product >>> 32;
3951   //  }
3952   //  z[xstart] = (int)carry;
3953 
3954   mr_if_needed(idx, ylen);        // idx = ylen
3955   mr_if_needed(kdx, zlen);        // kdx = xlen + ylen
3956   li(carry, 0);                   // carry = 0
3957 
3958   Label L_done;
3959 
3960   addic_(xstart, xlen, -1);
3961   blt(CCR0, L_done);
3962 
3963   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z,
3964                         carry, product_high, product, idx, kdx, tmp);
3965 
3966   Label L_second_loop;
3967 
3968   cmpdi(CCR0, kdx, 0);
3969   beq(CCR0, L_second_loop);
3970 
3971   Label L_carry;
3972 
3973   addic_(kdx, kdx, -1);
3974   beq(CCR0, L_carry);
3975 
3976   // Store lower 32 bits of carry.
3977   sldi(tmp, kdx, LogBytesPerInt);
3978   stwx(carry, z, tmp);
3979   srdi(carry, carry, 32);
3980   addi(kdx, kdx, -1);
3981 
3982 
3983   bind(L_carry);
3984 
3985   // Store upper 32 bits of carry.
3986   sldi(tmp, kdx, LogBytesPerInt);
3987   stwx(carry, z, tmp);
3988 
3989   // Second and third (nested) loops.
3990   //
3991   //  for (int i = xstart-1; i >= 0; i--) { // Second loop
3992   //    carry = 0;
3993   //    for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
3994   //      long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
3995   //                     (z[k] & LONG_MASK) + carry;
3996   //      z[k] = (int)product;
3997   //      carry = product >>> 32;
3998   //    }
3999   //    z[i] = (int)carry;
4000   //  }
4001   //
4002   //  i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
4003 
4004   bind(L_second_loop);
4005 
4006   li(carry, 0);                   // carry = 0;
4007 
4008   addic_(xstart, xstart, -1);     // i = xstart-1;
4009   blt(CCR0, L_done);
4010 
4011   Register zsave = tmp10;
4012 
4013   mr(zsave, z);
4014 
4015 
4016   Label L_last_x;
4017 
4018   sldi(tmp, xstart, LogBytesPerInt);
4019   add(z, z, tmp);                 // z = z + k - j
4020   addi(z, z, 4);
4021   addic_(xstart, xstart, -1);     // i = xstart-1;
4022   blt(CCR0, L_last_x);
4023 
4024   sldi(tmp, xstart, LogBytesPerInt);
4025   ldx(x_xstart, x, tmp);
4026 #ifdef VM_LITTLE_ENDIAN
4027   rldicl(x_xstart, x_xstart, 32, 0);
4028 #endif
4029 
4030 
4031   Label L_third_loop_prologue;
4032 
4033   bind(L_third_loop_prologue);
4034 
4035   Register xsave = tmp11;
4036   Register xlensave = tmp12;
4037   Register ylensave = tmp13;
4038 
4039   mr(xsave, x);
4040   mr(xlensave, xstart);
4041   mr(ylensave, ylen);
4042 
4043 
4044   multiply_128_x_128_loop(x_xstart, y, z, y_idx, ylen,
4045                           carry, product_high, product, x, tmp);
4046 
4047   mr(z, zsave);
4048   mr(x, xsave);
4049   mr(xlen, xlensave);   // This is the decrement of the loop counter!
4050   mr(ylen, ylensave);
4051 
4052   addi(tmp3, xlen, 1);
4053   sldi(tmp, tmp3, LogBytesPerInt);
4054   stwx(carry, z, tmp);
4055   addic_(tmp3, tmp3, -1);
4056   blt(CCR0, L_done);
4057 
4058   srdi(carry, carry, 32);
4059   sldi(tmp, tmp3, LogBytesPerInt);
4060   stwx(carry, z, tmp);
4061   b(L_second_loop);
4062 
4063   // Next infrequent code is moved outside loops.
4064   bind(L_last_x);
4065 
4066   lwz(x_xstart, 0, x);
4067   b(L_third_loop_prologue);
4068 
4069   bind(L_done);
4070 }   // multiply_to_len
4071 
4072 void MacroAssembler::asm_assert(bool check_equal, const char *msg) {
4073 #ifdef ASSERT
4074   Label ok;
4075   if (check_equal) {
4076     beq(CCR0, ok);
4077   } else {
4078     bne(CCR0, ok);
4079   }
4080   stop(msg);
4081   bind(ok);
4082 #endif
4083 }
4084 
4085 void MacroAssembler::asm_assert_mems_zero(bool check_equal, int size, int mem_offset,
4086                                           Register mem_base, const char* msg) {
4087 #ifdef ASSERT
4088   switch (size) {
4089     case 4:
4090       lwz(R0, mem_offset, mem_base);
4091       cmpwi(CCR0, R0, 0);
4092       break;
4093     case 8:
4094       ld(R0, mem_offset, mem_base);
4095       cmpdi(CCR0, R0, 0);
4096       break;
4097     default:
4098       ShouldNotReachHere();
4099   }
4100   asm_assert(check_equal, msg);
4101 #endif // ASSERT
4102 }
4103 
4104 void MacroAssembler::verify_coop(Register coop, const char* msg) {
4105   if (!VerifyOops) { return; }
4106   if (UseCompressedOops) { decode_heap_oop(coop); }
4107   verify_oop(coop, msg);
4108   if (UseCompressedOops) { encode_heap_oop(coop, coop); }
4109 }
4110 
4111 // READ: oop. KILL: R0. Volatile floats perhaps.
4112 void MacroAssembler::verify_oop(Register oop, const char* msg) {
4113   if (!VerifyOops) {
4114     return;
4115   }
4116 
4117   address/* FunctionDescriptor** */fd = StubRoutines::verify_oop_subroutine_entry_address();
4118   const Register tmp = R11; // Will be preserved.
4119   const int nbytes_save = MacroAssembler::num_volatile_regs * 8;
4120 
4121   BLOCK_COMMENT("verify_oop {");
4122 
4123   save_volatile_gprs(R1_SP, -nbytes_save); // except R0
4124 
4125   mr_if_needed(R4_ARG2, oop);
4126   save_LR_CR(tmp); // save in old frame
4127   push_frame_reg_args(nbytes_save, tmp);
4128   // load FunctionDescriptor** / entry_address *
4129   load_const_optimized(tmp, fd, R0);
4130   // load FunctionDescriptor* / entry_address
4131   ld(tmp, 0, tmp);
4132   load_const_optimized(R3_ARG1, (address)msg, R0);
4133   // Call destination for its side effect.
4134   call_c(tmp);
4135 
4136   pop_frame();
4137   restore_LR_CR(tmp);
4138   restore_volatile_gprs(R1_SP, -nbytes_save); // except R0
4139 
4140   BLOCK_COMMENT("} verify_oop");
4141 }
4142 
4143 void MacroAssembler::verify_oop_addr(RegisterOrConstant offs, Register base, const char* msg) {
4144   if (!VerifyOops) {
4145     return;
4146   }
4147 
4148   address/* FunctionDescriptor** */fd = StubRoutines::verify_oop_subroutine_entry_address();
4149   const Register tmp = R11; // Will be preserved.
4150   const int nbytes_save = MacroAssembler::num_volatile_regs * 8;
4151   save_volatile_gprs(R1_SP, -nbytes_save); // except R0
4152 
4153   ld(R4_ARG2, offs, base);
4154   save_LR_CR(tmp); // save in old frame
4155   push_frame_reg_args(nbytes_save, tmp);
4156   // load FunctionDescriptor** / entry_address *
4157   load_const_optimized(tmp, fd, R0);
4158   // load FunctionDescriptor* / entry_address
4159   ld(tmp, 0, tmp);
4160   load_const_optimized(R3_ARG1, (address)msg, R0);
4161   // Call destination for its side effect.
4162   call_c(tmp);
4163 
4164   pop_frame();
4165   restore_LR_CR(tmp);
4166   restore_volatile_gprs(R1_SP, -nbytes_save); // except R0
4167 }
4168 
4169 // Call a C-function that prints output.
4170 void MacroAssembler::stop(int type, const char* msg) {
4171   bool msg_present = (msg != nullptr);
4172 
4173 #ifndef PRODUCT
4174   block_comment(err_msg("stop(type %d): %s {", type, msg_present ? msg : "null"));
4175 #else
4176   block_comment("stop {");
4177 #endif
4178 
4179   if (msg_present) {
4180     type |= stop_msg_present;
4181   }
4182   tdi_unchecked(traptoUnconditional, 0/*reg 0*/, type);
4183   if (msg_present) {
4184     emit_int64((uintptr_t)msg);
4185   }
4186 
4187   block_comment("} stop;");
4188 }
4189 
4190 #ifndef PRODUCT
4191 // Write pattern 0x0101010101010101 in memory region [low-before, high+after].
4192 // Val, addr are temp registers.
4193 // If low == addr, addr is killed.
4194 // High is preserved.
4195 void MacroAssembler::zap_from_to(Register low, int before, Register high, int after, Register val, Register addr) {
4196   if (!ZapMemory) return;
4197 
4198   assert_different_registers(low, val);
4199 
4200   BLOCK_COMMENT("zap memory region {");
4201   load_const_optimized(val, 0x0101010101010101);
4202   int size = before + after;
4203   if (low == high && size < 5 && size > 0) {
4204     int offset = -before*BytesPerWord;
4205     for (int i = 0; i < size; ++i) {
4206       std(val, offset, low);
4207       offset += (1*BytesPerWord);
4208     }
4209   } else {
4210     addi(addr, low, -before*BytesPerWord);
4211     assert_different_registers(high, val);
4212     if (after) addi(high, high, after * BytesPerWord);
4213     Label loop;
4214     bind(loop);
4215     std(val, 0, addr);
4216     addi(addr, addr, 8);
4217     cmpd(CCR6, addr, high);
4218     ble(CCR6, loop);
4219     if (after) addi(high, high, -after * BytesPerWord);  // Correct back to old value.
4220   }
4221   BLOCK_COMMENT("} zap memory region");
4222 }
4223 
4224 #endif // !PRODUCT
4225 
4226 void SkipIfEqualZero::skip_to_label_if_equal_zero(MacroAssembler* masm, Register temp,
4227                                                   const bool* flag_addr, Label& label) {
4228   int simm16_offset = masm->load_const_optimized(temp, (address)flag_addr, R0, true);
4229   assert(sizeof(bool) == 1, "PowerPC ABI");
4230   masm->lbz(temp, simm16_offset, temp);
4231   masm->cmpwi(CCR0, temp, 0);
4232   masm->beq(CCR0, label);
4233 }
4234 
4235 SkipIfEqualZero::SkipIfEqualZero(MacroAssembler* masm, Register temp, const bool* flag_addr) : _masm(masm), _label() {
4236   skip_to_label_if_equal_zero(masm, temp, flag_addr, _label);
4237 }
4238 
4239 SkipIfEqualZero::~SkipIfEqualZero() {
4240   _masm->bind(_label);
4241 }
4242 
4243 void MacroAssembler::cache_wb(Address line) {
4244   assert(line.index() == noreg, "index should be noreg");
4245   assert(line.disp() == 0, "displacement should be 0");
4246   assert(VM_Version::supports_data_cache_line_flush(), "CPU or OS does not support flush to persistent memory");
4247   // Data Cache Store, not really a flush, so it works like a sync of cache
4248   // line and persistent mem, i.e. copying the cache line to persistent whilst
4249   // not invalidating the cache line.
4250   dcbst(line.base());
4251 }
4252 
4253 void MacroAssembler::cache_wbsync(bool is_presync) {
4254   assert(VM_Version::supports_data_cache_line_flush(), "CPU or OS does not support sync related to persistent memory");
4255   // We only need a post sync barrier. Post means _after_ a cache line flush or
4256   // store instruction, pre means a barrier emitted before such a instructions.
4257   if (!is_presync) {
4258     fence();
4259   }
4260 }
4261 
4262 void MacroAssembler::push_cont_fastpath() {
4263   Label done;
4264   ld_ptr(R0, JavaThread::cont_fastpath_offset(), R16_thread);
4265   cmpld(CCR0, R1_SP, R0);
4266   ble(CCR0, done);
4267   st_ptr(R1_SP, JavaThread::cont_fastpath_offset(), R16_thread);
4268   bind(done);
4269 }
4270 
4271 void MacroAssembler::pop_cont_fastpath() {
4272   Label done;
4273   ld_ptr(R0, JavaThread::cont_fastpath_offset(), R16_thread);
4274   cmpld(CCR0, R1_SP, R0);
4275   ble(CCR0, done);
4276   li(R0, 0);
4277   st_ptr(R0, JavaThread::cont_fastpath_offset(), R16_thread);
4278   bind(done);
4279 }
4280 
4281 // Note: Must preserve CCR0 EQ (invariant).
4282 void MacroAssembler::inc_held_monitor_count(Register tmp) {
4283   ld(tmp, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread);
4284 #ifdef ASSERT
4285   Label ok;
4286   cmpdi(CCR0, tmp, 0);
4287   bge_predict_taken(CCR0, ok);
4288   stop("held monitor count is negativ at increment");
4289   bind(ok);
4290   crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Restore CCR0 EQ
4291 #endif
4292   addi(tmp, tmp, 1);
4293   std(tmp, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread);
4294 }
4295 
4296 // Note: Must preserve CCR0 EQ (invariant).
4297 void MacroAssembler::dec_held_monitor_count(Register tmp) {
4298   ld(tmp, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread);
4299 #ifdef ASSERT
4300   Label ok;
4301   cmpdi(CCR0, tmp, 0);
4302   bgt_predict_taken(CCR0, ok);
4303   stop("held monitor count is <= 0 at decrement");
4304   bind(ok);
4305   crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Restore CCR0 EQ
4306 #endif
4307   addi(tmp, tmp, -1);
4308   std(tmp, in_bytes(JavaThread::held_monitor_count_offset()), R16_thread);
4309 }
4310 
4311 // Function to flip between unlocked and locked state (fast locking).
4312 // Branches to failed if the state is not as expected with CCR0 NE.
4313 // Falls through upon success with CCR0 EQ.
4314 // This requires fewer instructions and registers and is easier to use than the
4315 // cmpxchg based implementation.
4316 void MacroAssembler::atomically_flip_locked_state(bool is_unlock, Register obj, Register tmp, Label& failed, int semantics) {
4317   assert_different_registers(obj, tmp, R0);
4318   Label retry;
4319 
4320   if (semantics & MemBarRel) {
4321     release();
4322   }
4323 
4324   bind(retry);
4325   STATIC_ASSERT(markWord::locked_value == 0); // Or need to change this!
4326   if (!is_unlock) {
4327     ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_acquire_lock());
4328     xori(tmp, tmp, markWord::unlocked_value); // flip unlocked bit
4329     andi_(R0, tmp, markWord::lock_mask_in_place);
4330     bne(CCR0, failed); // failed if new header doesn't contain locked_value (which is 0)
4331   } else {
4332     ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_release_lock());
4333     andi_(R0, tmp, markWord::lock_mask_in_place);
4334     bne(CCR0, failed); // failed if old header doesn't contain locked_value (which is 0)
4335     ori(tmp, tmp, markWord::unlocked_value); // set unlocked bit
4336   }
4337   stdcx_(tmp, obj);
4338   bne(CCR0, retry);
4339 
4340   if (semantics & MemBarFenceAfter) {
4341     fence();
4342   } else if (semantics & MemBarAcq) {
4343     isync();
4344   }
4345 }
4346 
4347 // Implements lightweight-locking.
4348 //
4349 //  - obj: the object to be locked
4350 //  - t1, t2: temporary register
4351 void MacroAssembler::lightweight_lock(Register obj, Register t1, Register t2, Label& slow) {
4352   assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
4353   assert_different_registers(obj, t1, t2);
4354 
4355   Label push;
4356   const Register top = t1;
4357   const Register mark = t2;
4358   const Register t = R0;
4359 
4360   // Check if the lock-stack is full.
4361   lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4362   cmplwi(CCR0, top, LockStack::end_offset());
4363   bge(CCR0, slow);
4364 
4365   // The underflow check is elided. The recursive check will always fail
4366   // when the lock stack is empty because of the _bad_oop_sentinel field.
4367 
4368   // Check for recursion.
4369   subi(t, top, oopSize);
4370   ldx(t, R16_thread, t);
4371   cmpd(CCR0, obj, t);
4372   beq(CCR0, push);
4373 
4374   // Check header for monitor (0b10) or locked (0b00).
4375   ld(mark, oopDesc::mark_offset_in_bytes(), obj);
4376   xori(t, mark, markWord::unlocked_value);
4377   andi_(t, t, markWord::lock_mask_in_place);
4378   bne(CCR0, slow);
4379 
4380   // Try to lock. Transition lock bits 0b00 => 0b01
4381   atomically_flip_locked_state(/* is_unlock */ false, obj, mark, slow, MacroAssembler::MemBarAcq);
4382 
4383   bind(push);
4384   // After successful lock, push object on lock-stack
4385   stdx(obj, R16_thread, top);
4386   addi(top, top, oopSize);
4387   stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4388 }
4389 
4390 // Implements lightweight-unlocking.
4391 //
4392 // - obj: the object to be unlocked
4393 //  - t1: temporary register
4394 void MacroAssembler::lightweight_unlock(Register obj, Register t1, Label& slow) {
4395   assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
4396   assert_different_registers(obj, t1);
4397 
4398 #ifdef ASSERT
4399   {
4400     // The following checks rely on the fact that LockStack is only ever modified by
4401     // its owning thread, even if the lock got inflated concurrently; removal of LockStack
4402     // entries after inflation will happen delayed in that case.
4403 
4404     // Check for lock-stack underflow.
4405     Label stack_ok;
4406     lwz(t1, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4407     cmplwi(CCR0, t1, LockStack::start_offset());
4408     bge(CCR0, stack_ok);
4409     stop("Lock-stack underflow");
4410     bind(stack_ok);
4411   }
4412 #endif
4413 
4414   Label unlocked, push_and_slow;
4415   const Register top = t1;
4416   const Register mark = R0;
4417   Register t = R0;
4418 
4419   // Check if obj is top of lock-stack.
4420   lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4421   subi(top, top, oopSize);
4422   ldx(t, R16_thread, top);
4423   cmpd(CCR0, obj, t);
4424   bne(CCR0, slow);
4425 
4426   // Pop lock-stack.
4427   DEBUG_ONLY(li(t, 0);)
4428   DEBUG_ONLY(stdx(t, R16_thread, top);)
4429   stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4430 
4431   // The underflow check is elided. The recursive check will always fail
4432   // when the lock stack is empty because of the _bad_oop_sentinel field.
4433 
4434   // Check if recursive.
4435   subi(t, top, oopSize);
4436   ldx(t, R16_thread, t);
4437   cmpd(CCR0, obj, t);
4438   beq(CCR0, unlocked);
4439 
4440   // Use top as tmp
4441   t = top;
4442 
4443   // Not recursive. Check header for monitor (0b10).
4444   ld(mark, oopDesc::mark_offset_in_bytes(), obj);
4445   andi_(t, mark, markWord::monitor_value);
4446   bne(CCR0, push_and_slow);
4447 
4448 #ifdef ASSERT
4449   // Check header not unlocked (0b01).
4450   Label not_unlocked;
4451   andi_(t, mark, markWord::unlocked_value);
4452   beq(CCR0, not_unlocked);
4453   stop("lightweight_unlock already unlocked");
4454   bind(not_unlocked);
4455 #endif
4456 
4457   // Try to unlock. Transition lock bits 0b00 => 0b01
4458   atomically_flip_locked_state(/* is_unlock */ true, obj, t, push_and_slow, MacroAssembler::MemBarRel);
4459   b(unlocked);
4460 
4461   bind(push_and_slow);
4462 
4463   // Restore lock-stack and handle the unlock in runtime.
4464   lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4465   DEBUG_ONLY(stdx(obj, R16_thread, top);)
4466   addi(top, top, oopSize);
4467   stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4468   b(slow);
4469 
4470   bind(unlocked);
4471 }
--- EOF ---