1 /*
   2  * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2012, 2022 SAP SE. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "compiler/disassembler.hpp"
  29 #include "gc/shared/collectedHeap.inline.hpp"
  30 #include "gc/shared/barrierSet.hpp"
  31 #include "gc/shared/barrierSetAssembler.hpp"
  32 #include "interpreter/interpreter.hpp"
  33 #include "memory/resourceArea.hpp"
  34 #include "nativeInst_ppc.hpp"
  35 #include "oops/klass.inline.hpp"
  36 #include "oops/methodData.hpp"
  37 #include "prims/methodHandles.hpp"
  38 #include "runtime/icache.hpp"
  39 #include "runtime/interfaceSupport.inline.hpp"
  40 #include "runtime/objectMonitor.hpp"
  41 #include "runtime/os.hpp"
  42 #include "runtime/safepoint.hpp"
  43 #include "runtime/safepointMechanism.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubRoutines.hpp"
  46 #include "runtime/vm_version.hpp"
  47 #include "utilities/macros.hpp"
  48 #include "utilities/powerOfTwo.hpp"
  49 
  50 #ifdef PRODUCT
  51 #define BLOCK_COMMENT(str) // nothing
  52 #else
  53 #define BLOCK_COMMENT(str) block_comment(str)
  54 #endif
  55 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  56 
  57 #ifdef ASSERT
  58 // On RISC, there's no benefit to verifying instruction boundaries.
  59 bool AbstractAssembler::pd_check_instruction_mark() { return false; }
  60 #endif
  61 
  62 void MacroAssembler::ld_largeoffset_unchecked(Register d, int si31, Register a, int emit_filler_nop) {
  63   assert(Assembler::is_simm(si31, 31) && si31 >= 0, "si31 out of range");
  64   if (Assembler::is_simm(si31, 16)) {
  65     ld(d, si31, a);
  66     if (emit_filler_nop) nop();
  67   } else {
  68     const int hi = MacroAssembler::largeoffset_si16_si16_hi(si31);
  69     const int lo = MacroAssembler::largeoffset_si16_si16_lo(si31);
  70     addis(d, a, hi);
  71     ld(d, lo, d);
  72   }
  73 }
  74 
  75 void MacroAssembler::ld_largeoffset(Register d, int si31, Register a, int emit_filler_nop) {
  76   assert_different_registers(d, a);
  77   ld_largeoffset_unchecked(d, si31, a, emit_filler_nop);
  78 }
  79 
  80 void MacroAssembler::load_sized_value(Register dst, RegisterOrConstant offs, Register base,
  81                                       size_t size_in_bytes, bool is_signed) {
  82   switch (size_in_bytes) {
  83   case  8:              ld(dst, offs, base);                         break;
  84   case  4:  is_signed ? lwa(dst, offs, base) : lwz(dst, offs, base); break;
  85   case  2:  is_signed ? lha(dst, offs, base) : lhz(dst, offs, base); break;
  86   case  1:  lbz(dst, offs, base); if (is_signed) extsb(dst, dst);    break; // lba doesn't exist :(
  87   default:  ShouldNotReachHere();
  88   }
  89 }
  90 
  91 void MacroAssembler::store_sized_value(Register dst, RegisterOrConstant offs, Register base,
  92                                        size_t size_in_bytes) {
  93   switch (size_in_bytes) {
  94   case  8:  std(dst, offs, base); break;
  95   case  4:  stw(dst, offs, base); break;
  96   case  2:  sth(dst, offs, base); break;
  97   case  1:  stb(dst, offs, base); break;
  98   default:  ShouldNotReachHere();
  99   }
 100 }
 101 
 102 void MacroAssembler::align(int modulus, int max, int rem) {
 103   int padding = (rem + modulus - (offset() % modulus)) % modulus;
 104   if (padding > max) return;
 105   for (int c = (padding >> 2); c > 0; --c) { nop(); }
 106 }
 107 
 108 void MacroAssembler::align_prefix() {
 109   if (is_aligned(offset() + BytesPerInstWord, 64)) { nop(); }
 110 }
 111 
 112 // Issue instructions that calculate given TOC from global TOC.
 113 void MacroAssembler::calculate_address_from_global_toc(Register dst, address addr, bool hi16, bool lo16,
 114                                                        bool add_relocation, bool emit_dummy_addr) {
 115   int offset = -1;
 116   if (emit_dummy_addr) {
 117     offset = -128; // dummy address
 118   } else if (addr != (address)(intptr_t)-1) {
 119     offset = MacroAssembler::offset_to_global_toc(addr);
 120   }
 121 
 122   if (hi16) {
 123     addis(dst, R29_TOC, MacroAssembler::largeoffset_si16_si16_hi(offset));
 124   }
 125   if (lo16) {
 126     if (add_relocation) {
 127       // Relocate at the addi to avoid confusion with a load from the method's TOC.
 128       relocate(internal_word_Relocation::spec(addr));
 129     }
 130     addi(dst, dst, MacroAssembler::largeoffset_si16_si16_lo(offset));
 131   }
 132 }
 133 
 134 address MacroAssembler::patch_calculate_address_from_global_toc_at(address a, address bound, address addr) {
 135   const int offset = MacroAssembler::offset_to_global_toc(addr);
 136 
 137   const address inst2_addr = a;
 138   const int inst2 = *(int *)inst2_addr;
 139 
 140   // The relocation points to the second instruction, the addi,
 141   // and the addi reads and writes the same register dst.
 142   const int dst = inv_rt_field(inst2);
 143   assert(is_addi(inst2) && inv_ra_field(inst2) == dst, "must be addi reading and writing dst");
 144 
 145   // Now, find the preceding addis which writes to dst.
 146   int inst1 = 0;
 147   address inst1_addr = inst2_addr - BytesPerInstWord;
 148   while (inst1_addr >= bound) {
 149     inst1 = *(int *) inst1_addr;
 150     if (is_addis(inst1) && inv_rt_field(inst1) == dst) {
 151       // Stop, found the addis which writes dst.
 152       break;
 153     }
 154     inst1_addr -= BytesPerInstWord;
 155   }
 156 
 157   assert(is_addis(inst1) && inv_ra_field(inst1) == 29 /* R29 */, "source must be global TOC");
 158   set_imm((int *)inst1_addr, MacroAssembler::largeoffset_si16_si16_hi(offset));
 159   set_imm((int *)inst2_addr, MacroAssembler::largeoffset_si16_si16_lo(offset));
 160   return inst1_addr;
 161 }
 162 
 163 address MacroAssembler::get_address_of_calculate_address_from_global_toc_at(address a, address bound) {
 164   const address inst2_addr = a;
 165   const int inst2 = *(int *)inst2_addr;
 166 
 167   // The relocation points to the second instruction, the addi,
 168   // and the addi reads and writes the same register dst.
 169   const int dst = inv_rt_field(inst2);
 170   assert(is_addi(inst2) && inv_ra_field(inst2) == dst, "must be addi reading and writing dst");
 171 
 172   // Now, find the preceding addis which writes to dst.
 173   int inst1 = 0;
 174   address inst1_addr = inst2_addr - BytesPerInstWord;
 175   while (inst1_addr >= bound) {
 176     inst1 = *(int *) inst1_addr;
 177     if (is_addis(inst1) && inv_rt_field(inst1) == dst) {
 178       // stop, found the addis which writes dst
 179       break;
 180     }
 181     inst1_addr -= BytesPerInstWord;
 182   }
 183 
 184   assert(is_addis(inst1) && inv_ra_field(inst1) == 29 /* R29 */, "source must be global TOC");
 185 
 186   int offset = (get_imm(inst1_addr, 0) << 16) + get_imm(inst2_addr, 0);
 187   // -1 is a special case
 188   if (offset == -1) {
 189     return (address)(intptr_t)-1;
 190   } else {
 191     return global_toc() + offset;
 192   }
 193 }
 194 
 195 #ifdef _LP64
 196 // Patch compressed oops or klass constants.
 197 // Assembler sequence is
 198 // 1) compressed oops:
 199 //    lis  rx = const.hi
 200 //    ori rx = rx | const.lo
 201 // 2) compressed klass:
 202 //    lis  rx = const.hi
 203 //    clrldi rx = rx & 0xFFFFffff // clearMS32b, optional
 204 //    ori rx = rx | const.lo
 205 // Clrldi will be passed by.
 206 address MacroAssembler::patch_set_narrow_oop(address a, address bound, narrowOop data) {
 207   assert(UseCompressedOops, "Should only patch compressed oops");
 208 
 209   const address inst2_addr = a;
 210   const int inst2 = *(int *)inst2_addr;
 211 
 212   // The relocation points to the second instruction, the ori,
 213   // and the ori reads and writes the same register dst.
 214   const int dst = inv_rta_field(inst2);
 215   assert(is_ori(inst2) && inv_rs_field(inst2) == dst, "must be ori reading and writing dst");
 216   // Now, find the preceding addis which writes to dst.
 217   int inst1 = 0;
 218   address inst1_addr = inst2_addr - BytesPerInstWord;
 219   bool inst1_found = false;
 220   while (inst1_addr >= bound) {
 221     inst1 = *(int *)inst1_addr;
 222     if (is_lis(inst1) && inv_rs_field(inst1) == dst) { inst1_found = true; break; }
 223     inst1_addr -= BytesPerInstWord;
 224   }
 225   assert(inst1_found, "inst is not lis");
 226 
 227   uint32_t data_value = CompressedOops::narrow_oop_value(data);
 228   int xc = (data_value >> 16) & 0xffff;
 229   int xd = (data_value >>  0) & 0xffff;
 230 
 231   set_imm((int *)inst1_addr, (short)(xc)); // see enc_load_con_narrow_hi/_lo
 232   set_imm((int *)inst2_addr,        (xd)); // unsigned int
 233   return inst1_addr;
 234 }
 235 
 236 // Get compressed oop constant.
 237 narrowOop MacroAssembler::get_narrow_oop(address a, address bound) {
 238   assert(UseCompressedOops, "Should only patch compressed oops");
 239 
 240   const address inst2_addr = a;
 241   const int inst2 = *(int *)inst2_addr;
 242 
 243   // The relocation points to the second instruction, the ori,
 244   // and the ori reads and writes the same register dst.
 245   const int dst = inv_rta_field(inst2);
 246   assert(is_ori(inst2) && inv_rs_field(inst2) == dst, "must be ori reading and writing dst");
 247   // Now, find the preceding lis which writes to dst.
 248   int inst1 = 0;
 249   address inst1_addr = inst2_addr - BytesPerInstWord;
 250   bool inst1_found = false;
 251 
 252   while (inst1_addr >= bound) {
 253     inst1 = *(int *) inst1_addr;
 254     if (is_lis(inst1) && inv_rs_field(inst1) == dst) { inst1_found = true; break;}
 255     inst1_addr -= BytesPerInstWord;
 256   }
 257   assert(inst1_found, "inst is not lis");
 258 
 259   uint xl = ((unsigned int) (get_imm(inst2_addr, 0) & 0xffff));
 260   uint xh = (((get_imm(inst1_addr, 0)) & 0xffff) << 16);
 261 
 262   return CompressedOops::narrow_oop_cast(xl | xh);
 263 }
 264 #endif // _LP64
 265 
 266 // Returns true if successful.
 267 bool MacroAssembler::load_const_from_method_toc(Register dst, AddressLiteral& a,
 268                                                 Register toc, bool fixed_size) {
 269   int toc_offset = 0;
 270   // Use RelocationHolder::none for the constant pool entry, otherwise
 271   // we will end up with a failing NativeCall::verify(x) where x is
 272   // the address of the constant pool entry.
 273   // FIXME: We should insert relocation information for oops at the constant
 274   // pool entries instead of inserting it at the loads; patching of a constant
 275   // pool entry should be less expensive.
 276   address const_address = address_constant((address)a.value(), RelocationHolder::none);
 277   if (const_address == NULL) { return false; } // allocation failure
 278   // Relocate at the pc of the load.
 279   relocate(a.rspec());
 280   toc_offset = (int)(const_address - code()->consts()->start());
 281   ld_largeoffset_unchecked(dst, toc_offset, toc, fixed_size);
 282   return true;
 283 }
 284 
 285 bool MacroAssembler::is_load_const_from_method_toc_at(address a) {
 286   const address inst1_addr = a;
 287   const int inst1 = *(int *)inst1_addr;
 288 
 289    // The relocation points to the ld or the addis.
 290    return (is_ld(inst1)) ||
 291           (is_addis(inst1) && inv_ra_field(inst1) != 0);
 292 }
 293 
 294 int MacroAssembler::get_offset_of_load_const_from_method_toc_at(address a) {
 295   assert(is_load_const_from_method_toc_at(a), "must be load_const_from_method_toc");
 296 
 297   const address inst1_addr = a;
 298   const int inst1 = *(int *)inst1_addr;
 299 
 300   if (is_ld(inst1)) {
 301     return inv_d1_field(inst1);
 302   } else if (is_addis(inst1)) {
 303     const int dst = inv_rt_field(inst1);
 304 
 305     // Now, find the succeeding ld which reads and writes to dst.
 306     address inst2_addr = inst1_addr + BytesPerInstWord;
 307     int inst2 = 0;
 308     while (true) {
 309       inst2 = *(int *) inst2_addr;
 310       if (is_ld(inst2) && inv_ra_field(inst2) == dst && inv_rt_field(inst2) == dst) {
 311         // Stop, found the ld which reads and writes dst.
 312         break;
 313       }
 314       inst2_addr += BytesPerInstWord;
 315     }
 316     return (inv_d1_field(inst1) << 16) + inv_d1_field(inst2);
 317   }
 318   ShouldNotReachHere();
 319   return 0;
 320 }
 321 
 322 // Get the constant from a `load_const' sequence.
 323 long MacroAssembler::get_const(address a) {
 324   assert(is_load_const_at(a), "not a load of a constant");
 325   const int *p = (const int*) a;
 326   unsigned long x = (((unsigned long) (get_imm(a,0) & 0xffff)) << 48);
 327   if (is_ori(*(p+1))) {
 328     x |= (((unsigned long) (get_imm(a,1) & 0xffff)) << 32);
 329     x |= (((unsigned long) (get_imm(a,3) & 0xffff)) << 16);
 330     x |= (((unsigned long) (get_imm(a,4) & 0xffff)));
 331   } else if (is_lis(*(p+1))) {
 332     x |= (((unsigned long) (get_imm(a,2) & 0xffff)) << 32);
 333     x |= (((unsigned long) (get_imm(a,1) & 0xffff)) << 16);
 334     x |= (((unsigned long) (get_imm(a,3) & 0xffff)));
 335   } else {
 336     ShouldNotReachHere();
 337     return (long) 0;
 338   }
 339   return (long) x;
 340 }
 341 
 342 // Patch the 64 bit constant of a `load_const' sequence. This is a low
 343 // level procedure. It neither flushes the instruction cache nor is it
 344 // mt safe.
 345 void MacroAssembler::patch_const(address a, long x) {
 346   assert(is_load_const_at(a), "not a load of a constant");
 347   int *p = (int*) a;
 348   if (is_ori(*(p+1))) {
 349     set_imm(0 + p, (x >> 48) & 0xffff);
 350     set_imm(1 + p, (x >> 32) & 0xffff);
 351     set_imm(3 + p, (x >> 16) & 0xffff);
 352     set_imm(4 + p, x & 0xffff);
 353   } else if (is_lis(*(p+1))) {
 354     set_imm(0 + p, (x >> 48) & 0xffff);
 355     set_imm(2 + p, (x >> 32) & 0xffff);
 356     set_imm(1 + p, (x >> 16) & 0xffff);
 357     set_imm(3 + p, x & 0xffff);
 358   } else {
 359     ShouldNotReachHere();
 360   }
 361 }
 362 
 363 AddressLiteral MacroAssembler::allocate_metadata_address(Metadata* obj) {
 364   assert(oop_recorder() != NULL, "this assembler needs a Recorder");
 365   int index = oop_recorder()->allocate_metadata_index(obj);
 366   RelocationHolder rspec = metadata_Relocation::spec(index);
 367   return AddressLiteral((address)obj, rspec);
 368 }
 369 
 370 AddressLiteral MacroAssembler::constant_metadata_address(Metadata* obj) {
 371   assert(oop_recorder() != NULL, "this assembler needs a Recorder");
 372   int index = oop_recorder()->find_index(obj);
 373   RelocationHolder rspec = metadata_Relocation::spec(index);
 374   return AddressLiteral((address)obj, rspec);
 375 }
 376 
 377 AddressLiteral MacroAssembler::allocate_oop_address(jobject obj) {
 378   assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
 379   int oop_index = oop_recorder()->allocate_oop_index(obj);
 380   return AddressLiteral(address(obj), oop_Relocation::spec(oop_index));
 381 }
 382 
 383 AddressLiteral MacroAssembler::constant_oop_address(jobject obj) {
 384   assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
 385   int oop_index = oop_recorder()->find_index(obj);
 386   return AddressLiteral(address(obj), oop_Relocation::spec(oop_index));
 387 }
 388 
 389 #ifndef PRODUCT
 390 void MacroAssembler::pd_print_patched_instruction(address branch) {
 391   Unimplemented(); // TODO: PPC port
 392 }
 393 #endif // ndef PRODUCT
 394 
 395 // Conditional far branch for destinations encodable in 24+2 bits.
 396 void MacroAssembler::bc_far(int boint, int biint, Label& dest, int optimize) {
 397 
 398   // If requested by flag optimize, relocate the bc_far as a
 399   // runtime_call and prepare for optimizing it when the code gets
 400   // relocated.
 401   if (optimize == bc_far_optimize_on_relocate) {
 402     relocate(relocInfo::runtime_call_type);
 403   }
 404 
 405   // variant 2:
 406   //
 407   //    b!cxx SKIP
 408   //    bxx   DEST
 409   //  SKIP:
 410   //
 411 
 412   const int opposite_boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(boint)),
 413                                                 opposite_bcond(inv_boint_bcond(boint)));
 414 
 415   // We emit two branches.
 416   // First, a conditional branch which jumps around the far branch.
 417   const address not_taken_pc = pc() + 2 * BytesPerInstWord;
 418   const address bc_pc        = pc();
 419   bc(opposite_boint, biint, not_taken_pc);
 420 
 421   const int bc_instr = *(int*)bc_pc;
 422   assert(not_taken_pc == (address)inv_bd_field(bc_instr, (intptr_t)bc_pc), "postcondition");
 423   assert(opposite_boint == inv_bo_field(bc_instr), "postcondition");
 424   assert(boint == add_bhint_to_boint(opposite_bhint(inv_boint_bhint(inv_bo_field(bc_instr))),
 425                                      opposite_bcond(inv_boint_bcond(inv_bo_field(bc_instr)))),
 426          "postcondition");
 427   assert(biint == inv_bi_field(bc_instr), "postcondition");
 428 
 429   // Second, an unconditional far branch which jumps to dest.
 430   // Note: target(dest) remembers the current pc (see CodeSection::target)
 431   //       and returns the current pc if the label is not bound yet; when
 432   //       the label gets bound, the unconditional far branch will be patched.
 433   const address target_pc = target(dest);
 434   const address b_pc  = pc();
 435   b(target_pc);
 436 
 437   assert(not_taken_pc == pc(),                     "postcondition");
 438   assert(dest.is_bound() || target_pc == b_pc, "postcondition");
 439 }
 440 
 441 // 1 or 2 instructions
 442 void MacroAssembler::bc_far_optimized(int boint, int biint, Label& dest) {
 443   if (dest.is_bound() && is_within_range_of_bcxx(target(dest), pc())) {
 444     bc(boint, biint, dest);
 445   } else {
 446     bc_far(boint, biint, dest, MacroAssembler::bc_far_optimize_on_relocate);
 447   }
 448 }
 449 
 450 bool MacroAssembler::is_bc_far_at(address instruction_addr) {
 451   return is_bc_far_variant1_at(instruction_addr) ||
 452          is_bc_far_variant2_at(instruction_addr) ||
 453          is_bc_far_variant3_at(instruction_addr);
 454 }
 455 
 456 address MacroAssembler::get_dest_of_bc_far_at(address instruction_addr) {
 457   if (is_bc_far_variant1_at(instruction_addr)) {
 458     const address instruction_1_addr = instruction_addr;
 459     const int instruction_1 = *(int*)instruction_1_addr;
 460     return (address)inv_bd_field(instruction_1, (intptr_t)instruction_1_addr);
 461   } else if (is_bc_far_variant2_at(instruction_addr)) {
 462     const address instruction_2_addr = instruction_addr + 4;
 463     return bxx_destination(instruction_2_addr);
 464   } else if (is_bc_far_variant3_at(instruction_addr)) {
 465     return instruction_addr + 8;
 466   }
 467   // variant 4 ???
 468   ShouldNotReachHere();
 469   return NULL;
 470 }
 471 void MacroAssembler::set_dest_of_bc_far_at(address instruction_addr, address dest) {
 472 
 473   if (is_bc_far_variant3_at(instruction_addr)) {
 474     // variant 3, far cond branch to the next instruction, already patched to nops:
 475     //
 476     //    nop
 477     //    endgroup
 478     //  SKIP/DEST:
 479     //
 480     return;
 481   }
 482 
 483   // first, extract boint and biint from the current branch
 484   int boint = 0;
 485   int biint = 0;
 486 
 487   ResourceMark rm;
 488   const int code_size = 2 * BytesPerInstWord;
 489   CodeBuffer buf(instruction_addr, code_size);
 490   MacroAssembler masm(&buf);
 491   if (is_bc_far_variant2_at(instruction_addr) && dest == instruction_addr + 8) {
 492     // Far branch to next instruction: Optimize it by patching nops (produce variant 3).
 493     masm.nop();
 494     masm.endgroup();
 495   } else {
 496     if (is_bc_far_variant1_at(instruction_addr)) {
 497       // variant 1, the 1st instruction contains the destination address:
 498       //
 499       //    bcxx  DEST
 500       //    nop
 501       //
 502       const int instruction_1 = *(int*)(instruction_addr);
 503       boint = inv_bo_field(instruction_1);
 504       biint = inv_bi_field(instruction_1);
 505     } else if (is_bc_far_variant2_at(instruction_addr)) {
 506       // variant 2, the 2nd instruction contains the destination address:
 507       //
 508       //    b!cxx SKIP
 509       //    bxx   DEST
 510       //  SKIP:
 511       //
 512       const int instruction_1 = *(int*)(instruction_addr);
 513       boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(inv_bo_field(instruction_1))),
 514           opposite_bcond(inv_boint_bcond(inv_bo_field(instruction_1))));
 515       biint = inv_bi_field(instruction_1);
 516     } else {
 517       // variant 4???
 518       ShouldNotReachHere();
 519     }
 520 
 521     // second, set the new branch destination and optimize the code
 522     if (dest != instruction_addr + 4 && // the bc_far is still unbound!
 523         masm.is_within_range_of_bcxx(dest, instruction_addr)) {
 524       // variant 1:
 525       //
 526       //    bcxx  DEST
 527       //    nop
 528       //
 529       masm.bc(boint, biint, dest);
 530       masm.nop();
 531     } else {
 532       // variant 2:
 533       //
 534       //    b!cxx SKIP
 535       //    bxx   DEST
 536       //  SKIP:
 537       //
 538       const int opposite_boint = add_bhint_to_boint(opposite_bhint(inv_boint_bhint(boint)),
 539                                                     opposite_bcond(inv_boint_bcond(boint)));
 540       const address not_taken_pc = masm.pc() + 2 * BytesPerInstWord;
 541       masm.bc(opposite_boint, biint, not_taken_pc);
 542       masm.b(dest);
 543     }
 544   }
 545   ICache::ppc64_flush_icache_bytes(instruction_addr, code_size);
 546 }
 547 
 548 // Emit a NOT mt-safe patchable 64 bit absolute call/jump.
 549 void MacroAssembler::bxx64_patchable(address dest, relocInfo::relocType rt, bool link) {
 550   // get current pc
 551   uint64_t start_pc = (uint64_t) pc();
 552 
 553   const address pc_of_bl = (address) (start_pc + (6*BytesPerInstWord)); // bl is last
 554   const address pc_of_b  = (address) (start_pc + (0*BytesPerInstWord)); // b is first
 555 
 556   // relocate here
 557   if (rt != relocInfo::none) {
 558     relocate(rt);
 559   }
 560 
 561   if ( ReoptimizeCallSequences &&
 562        (( link && is_within_range_of_b(dest, pc_of_bl)) ||
 563         (!link && is_within_range_of_b(dest, pc_of_b)))) {
 564     // variant 2:
 565     // Emit an optimized, pc-relative call/jump.
 566 
 567     if (link) {
 568       // some padding
 569       nop();
 570       nop();
 571       nop();
 572       nop();
 573       nop();
 574       nop();
 575 
 576       // do the call
 577       assert(pc() == pc_of_bl, "just checking");
 578       bl(dest, relocInfo::none);
 579     } else {
 580       // do the jump
 581       assert(pc() == pc_of_b, "just checking");
 582       b(dest, relocInfo::none);
 583 
 584       // some padding
 585       nop();
 586       nop();
 587       nop();
 588       nop();
 589       nop();
 590       nop();
 591     }
 592 
 593     // Assert that we can identify the emitted call/jump.
 594     assert(is_bxx64_patchable_variant2_at((address)start_pc, link),
 595            "can't identify emitted call");
 596   } else {
 597     // variant 1:
 598     mr(R0, R11);  // spill R11 -> R0.
 599 
 600     // Load the destination address into CTR,
 601     // calculate destination relative to global toc.
 602     calculate_address_from_global_toc(R11, dest, true, true, false);
 603 
 604     mtctr(R11);
 605     mr(R11, R0);  // spill R11 <- R0.
 606     nop();
 607 
 608     // do the call/jump
 609     if (link) {
 610       bctrl();
 611     } else{
 612       bctr();
 613     }
 614     // Assert that we can identify the emitted call/jump.
 615     assert(is_bxx64_patchable_variant1b_at((address)start_pc, link),
 616            "can't identify emitted call");
 617   }
 618 
 619   // Assert that we can identify the emitted call/jump.
 620   assert(is_bxx64_patchable_at((address)start_pc, link),
 621          "can't identify emitted call");
 622   assert(get_dest_of_bxx64_patchable_at((address)start_pc, link) == dest,
 623          "wrong encoding of dest address");
 624 }
 625 
 626 // Identify a bxx64_patchable instruction.
 627 bool MacroAssembler::is_bxx64_patchable_at(address instruction_addr, bool link) {
 628   return is_bxx64_patchable_variant1b_at(instruction_addr, link)
 629     //|| is_bxx64_patchable_variant1_at(instruction_addr, link)
 630       || is_bxx64_patchable_variant2_at(instruction_addr, link);
 631 }
 632 
 633 // Does the call64_patchable instruction use a pc-relative encoding of
 634 // the call destination?
 635 bool MacroAssembler::is_bxx64_patchable_pcrelative_at(address instruction_addr, bool link) {
 636   // variant 2 is pc-relative
 637   return is_bxx64_patchable_variant2_at(instruction_addr, link);
 638 }
 639 
 640 // Identify variant 1.
 641 bool MacroAssembler::is_bxx64_patchable_variant1_at(address instruction_addr, bool link) {
 642   unsigned int* instr = (unsigned int*) instruction_addr;
 643   return (link ? is_bctrl(instr[6]) : is_bctr(instr[6])) // bctr[l]
 644       && is_mtctr(instr[5]) // mtctr
 645     && is_load_const_at(instruction_addr);
 646 }
 647 
 648 // Identify variant 1b: load destination relative to global toc.
 649 bool MacroAssembler::is_bxx64_patchable_variant1b_at(address instruction_addr, bool link) {
 650   unsigned int* instr = (unsigned int*) instruction_addr;
 651   return (link ? is_bctrl(instr[6]) : is_bctr(instr[6])) // bctr[l]
 652     && is_mtctr(instr[3]) // mtctr
 653     && is_calculate_address_from_global_toc_at(instruction_addr + 2*BytesPerInstWord, instruction_addr);
 654 }
 655 
 656 // Identify variant 2.
 657 bool MacroAssembler::is_bxx64_patchable_variant2_at(address instruction_addr, bool link) {
 658   unsigned int* instr = (unsigned int*) instruction_addr;
 659   if (link) {
 660     return is_bl (instr[6])  // bl dest is last
 661       && is_nop(instr[0])  // nop
 662       && is_nop(instr[1])  // nop
 663       && is_nop(instr[2])  // nop
 664       && is_nop(instr[3])  // nop
 665       && is_nop(instr[4])  // nop
 666       && is_nop(instr[5]); // nop
 667   } else {
 668     return is_b  (instr[0])  // b  dest is first
 669       && is_nop(instr[1])  // nop
 670       && is_nop(instr[2])  // nop
 671       && is_nop(instr[3])  // nop
 672       && is_nop(instr[4])  // nop
 673       && is_nop(instr[5])  // nop
 674       && is_nop(instr[6]); // nop
 675   }
 676 }
 677 
 678 // Set dest address of a bxx64_patchable instruction.
 679 void MacroAssembler::set_dest_of_bxx64_patchable_at(address instruction_addr, address dest, bool link) {
 680   ResourceMark rm;
 681   int code_size = MacroAssembler::bxx64_patchable_size;
 682   CodeBuffer buf(instruction_addr, code_size);
 683   MacroAssembler masm(&buf);
 684   masm.bxx64_patchable(dest, relocInfo::none, link);
 685   ICache::ppc64_flush_icache_bytes(instruction_addr, code_size);
 686 }
 687 
 688 // Get dest address of a bxx64_patchable instruction.
 689 address MacroAssembler::get_dest_of_bxx64_patchable_at(address instruction_addr, bool link) {
 690   if (is_bxx64_patchable_variant1_at(instruction_addr, link)) {
 691     return (address) (unsigned long) get_const(instruction_addr);
 692   } else if (is_bxx64_patchable_variant2_at(instruction_addr, link)) {
 693     unsigned int* instr = (unsigned int*) instruction_addr;
 694     if (link) {
 695       const int instr_idx = 6; // bl is last
 696       int branchoffset = branch_destination(instr[instr_idx], 0);
 697       return instruction_addr + branchoffset + instr_idx*BytesPerInstWord;
 698     } else {
 699       const int instr_idx = 0; // b is first
 700       int branchoffset = branch_destination(instr[instr_idx], 0);
 701       return instruction_addr + branchoffset + instr_idx*BytesPerInstWord;
 702     }
 703   // Load dest relative to global toc.
 704   } else if (is_bxx64_patchable_variant1b_at(instruction_addr, link)) {
 705     return get_address_of_calculate_address_from_global_toc_at(instruction_addr + 2*BytesPerInstWord,
 706                                                                instruction_addr);
 707   } else {
 708     ShouldNotReachHere();
 709     return NULL;
 710   }
 711 }
 712 
 713 void MacroAssembler::clobber_volatile_gprs(Register excluded_register) {
 714   const int magic_number = 0x42;
 715 
 716   // Preserve stack pointer register (R1_SP) and system thread id register (R13);
 717   // although they're technically volatile
 718   for (int i = 2; i < 13; i++) {
 719     Register reg = as_Register(i);
 720     if (reg == excluded_register) {
 721       continue;
 722     }
 723 
 724     li(reg, magic_number);
 725   }
 726 }
 727 
 728 void MacroAssembler::clobber_carg_stack_slots(Register tmp) {
 729   const int magic_number = 0x43;
 730 
 731   li(tmp, magic_number);
 732   for (int m = 0; m <= 7; m++) {
 733     std(tmp, frame::abi_minframe_size + m * 8, R1_SP);
 734   }
 735 }
 736 
 737 // Uses ordering which corresponds to ABI:
 738 //    _savegpr0_14:  std  r14,-144(r1)
 739 //    _savegpr0_15:  std  r15,-136(r1)
 740 //    _savegpr0_16:  std  r16,-128(r1)
 741 void MacroAssembler::save_nonvolatile_gprs(Register dst, int offset) {
 742   std(R14, offset, dst);   offset += 8;
 743   std(R15, offset, dst);   offset += 8;
 744   std(R16, offset, dst);   offset += 8;
 745   std(R17, offset, dst);   offset += 8;
 746   std(R18, offset, dst);   offset += 8;
 747   std(R19, offset, dst);   offset += 8;
 748   std(R20, offset, dst);   offset += 8;
 749   std(R21, offset, dst);   offset += 8;
 750   std(R22, offset, dst);   offset += 8;
 751   std(R23, offset, dst);   offset += 8;
 752   std(R24, offset, dst);   offset += 8;
 753   std(R25, offset, dst);   offset += 8;
 754   std(R26, offset, dst);   offset += 8;
 755   std(R27, offset, dst);   offset += 8;
 756   std(R28, offset, dst);   offset += 8;
 757   std(R29, offset, dst);   offset += 8;
 758   std(R30, offset, dst);   offset += 8;
 759   std(R31, offset, dst);   offset += 8;
 760 
 761   stfd(F14, offset, dst);   offset += 8;
 762   stfd(F15, offset, dst);   offset += 8;
 763   stfd(F16, offset, dst);   offset += 8;
 764   stfd(F17, offset, dst);   offset += 8;
 765   stfd(F18, offset, dst);   offset += 8;
 766   stfd(F19, offset, dst);   offset += 8;
 767   stfd(F20, offset, dst);   offset += 8;
 768   stfd(F21, offset, dst);   offset += 8;
 769   stfd(F22, offset, dst);   offset += 8;
 770   stfd(F23, offset, dst);   offset += 8;
 771   stfd(F24, offset, dst);   offset += 8;
 772   stfd(F25, offset, dst);   offset += 8;
 773   stfd(F26, offset, dst);   offset += 8;
 774   stfd(F27, offset, dst);   offset += 8;
 775   stfd(F28, offset, dst);   offset += 8;
 776   stfd(F29, offset, dst);   offset += 8;
 777   stfd(F30, offset, dst);   offset += 8;
 778   stfd(F31, offset, dst);
 779 }
 780 
 781 // Uses ordering which corresponds to ABI:
 782 //    _restgpr0_14:  ld   r14,-144(r1)
 783 //    _restgpr0_15:  ld   r15,-136(r1)
 784 //    _restgpr0_16:  ld   r16,-128(r1)
 785 void MacroAssembler::restore_nonvolatile_gprs(Register src, int offset) {
 786   ld(R14, offset, src);   offset += 8;
 787   ld(R15, offset, src);   offset += 8;
 788   ld(R16, offset, src);   offset += 8;
 789   ld(R17, offset, src);   offset += 8;
 790   ld(R18, offset, src);   offset += 8;
 791   ld(R19, offset, src);   offset += 8;
 792   ld(R20, offset, src);   offset += 8;
 793   ld(R21, offset, src);   offset += 8;
 794   ld(R22, offset, src);   offset += 8;
 795   ld(R23, offset, src);   offset += 8;
 796   ld(R24, offset, src);   offset += 8;
 797   ld(R25, offset, src);   offset += 8;
 798   ld(R26, offset, src);   offset += 8;
 799   ld(R27, offset, src);   offset += 8;
 800   ld(R28, offset, src);   offset += 8;
 801   ld(R29, offset, src);   offset += 8;
 802   ld(R30, offset, src);   offset += 8;
 803   ld(R31, offset, src);   offset += 8;
 804 
 805   // FP registers
 806   lfd(F14, offset, src);   offset += 8;
 807   lfd(F15, offset, src);   offset += 8;
 808   lfd(F16, offset, src);   offset += 8;
 809   lfd(F17, offset, src);   offset += 8;
 810   lfd(F18, offset, src);   offset += 8;
 811   lfd(F19, offset, src);   offset += 8;
 812   lfd(F20, offset, src);   offset += 8;
 813   lfd(F21, offset, src);   offset += 8;
 814   lfd(F22, offset, src);   offset += 8;
 815   lfd(F23, offset, src);   offset += 8;
 816   lfd(F24, offset, src);   offset += 8;
 817   lfd(F25, offset, src);   offset += 8;
 818   lfd(F26, offset, src);   offset += 8;
 819   lfd(F27, offset, src);   offset += 8;
 820   lfd(F28, offset, src);   offset += 8;
 821   lfd(F29, offset, src);   offset += 8;
 822   lfd(F30, offset, src);   offset += 8;
 823   lfd(F31, offset, src);
 824 }
 825 
 826 // For verify_oops.
 827 void MacroAssembler::save_volatile_gprs(Register dst, int offset, bool include_fp_regs, bool include_R3_RET_reg) {
 828   std(R2,  offset, dst);   offset += 8;
 829   if (include_R3_RET_reg) {
 830     std(R3, offset, dst);  offset += 8;
 831   }
 832   std(R4,  offset, dst);   offset += 8;
 833   std(R5,  offset, dst);   offset += 8;
 834   std(R6,  offset, dst);   offset += 8;
 835   std(R7,  offset, dst);   offset += 8;
 836   std(R8,  offset, dst);   offset += 8;
 837   std(R9,  offset, dst);   offset += 8;
 838   std(R10, offset, dst);   offset += 8;
 839   std(R11, offset, dst);   offset += 8;
 840   std(R12, offset, dst);   offset += 8;
 841 
 842   if (include_fp_regs) {
 843     stfd(F0, offset, dst);   offset += 8;
 844     stfd(F1, offset, dst);   offset += 8;
 845     stfd(F2, offset, dst);   offset += 8;
 846     stfd(F3, offset, dst);   offset += 8;
 847     stfd(F4, offset, dst);   offset += 8;
 848     stfd(F5, offset, dst);   offset += 8;
 849     stfd(F6, offset, dst);   offset += 8;
 850     stfd(F7, offset, dst);   offset += 8;
 851     stfd(F8, offset, dst);   offset += 8;
 852     stfd(F9, offset, dst);   offset += 8;
 853     stfd(F10, offset, dst);  offset += 8;
 854     stfd(F11, offset, dst);  offset += 8;
 855     stfd(F12, offset, dst);  offset += 8;
 856     stfd(F13, offset, dst);
 857   }
 858 }
 859 
 860 // For verify_oops.
 861 void MacroAssembler::restore_volatile_gprs(Register src, int offset, bool include_fp_regs, bool include_R3_RET_reg) {
 862   ld(R2,  offset, src);   offset += 8;
 863   if (include_R3_RET_reg) {
 864     ld(R3,  offset, src);   offset += 8;
 865   }
 866   ld(R4,  offset, src);   offset += 8;
 867   ld(R5,  offset, src);   offset += 8;
 868   ld(R6,  offset, src);   offset += 8;
 869   ld(R7,  offset, src);   offset += 8;
 870   ld(R8,  offset, src);   offset += 8;
 871   ld(R9,  offset, src);   offset += 8;
 872   ld(R10, offset, src);   offset += 8;
 873   ld(R11, offset, src);   offset += 8;
 874   ld(R12, offset, src);   offset += 8;
 875 
 876   if (include_fp_regs) {
 877     lfd(F0, offset, src);   offset += 8;
 878     lfd(F1, offset, src);   offset += 8;
 879     lfd(F2, offset, src);   offset += 8;
 880     lfd(F3, offset, src);   offset += 8;
 881     lfd(F4, offset, src);   offset += 8;
 882     lfd(F5, offset, src);   offset += 8;
 883     lfd(F6, offset, src);   offset += 8;
 884     lfd(F7, offset, src);   offset += 8;
 885     lfd(F8, offset, src);   offset += 8;
 886     lfd(F9, offset, src);   offset += 8;
 887     lfd(F10, offset, src);  offset += 8;
 888     lfd(F11, offset, src);  offset += 8;
 889     lfd(F12, offset, src);  offset += 8;
 890     lfd(F13, offset, src);
 891   }
 892 }
 893 
 894 void MacroAssembler::save_LR_CR(Register tmp) {
 895   mfcr(tmp);
 896   std(tmp, _abi0(cr), R1_SP);
 897   mflr(tmp);
 898   std(tmp, _abi0(lr), R1_SP);
 899   // Tmp must contain lr on exit! (see return_addr and prolog in ppc64.ad)
 900 }
 901 
 902 void MacroAssembler::restore_LR_CR(Register tmp) {
 903   assert(tmp != R1_SP, "must be distinct");
 904   ld(tmp, _abi0(lr), R1_SP);
 905   mtlr(tmp);
 906   ld(tmp, _abi0(cr), R1_SP);
 907   mtcr(tmp);
 908 }
 909 
 910 address MacroAssembler::get_PC_trash_LR(Register result) {
 911   Label L;
 912   bl(L);
 913   bind(L);
 914   address lr_pc = pc();
 915   mflr(result);
 916   return lr_pc;
 917 }
 918 
 919 void MacroAssembler::resize_frame(Register offset, Register tmp) {
 920 #ifdef ASSERT
 921   assert_different_registers(offset, tmp, R1_SP);
 922   andi_(tmp, offset, frame::alignment_in_bytes-1);
 923   asm_assert_eq("resize_frame: unaligned");
 924 #endif
 925 
 926   // tmp <- *(SP)
 927   ld(tmp, _abi0(callers_sp), R1_SP);
 928   // addr <- SP + offset;
 929   // *(addr) <- tmp;
 930   // SP <- addr
 931   stdux(tmp, R1_SP, offset);
 932 }
 933 
 934 void MacroAssembler::resize_frame(int offset, Register tmp) {
 935   assert(is_simm(offset, 16), "too big an offset");
 936   assert_different_registers(tmp, R1_SP);
 937   assert((offset & (frame::alignment_in_bytes-1))==0, "resize_frame: unaligned");
 938   // tmp <- *(SP)
 939   ld(tmp, _abi0(callers_sp), R1_SP);
 940   // addr <- SP + offset;
 941   // *(addr) <- tmp;
 942   // SP <- addr
 943   stdu(tmp, offset, R1_SP);
 944 }
 945 
 946 void MacroAssembler::resize_frame_absolute(Register addr, Register tmp1, Register tmp2) {
 947   // (addr == tmp1) || (addr == tmp2) is allowed here!
 948   assert(tmp1 != tmp2, "must be distinct");
 949 
 950   // compute offset w.r.t. current stack pointer
 951   // tmp_1 <- addr - SP (!)
 952   subf(tmp1, R1_SP, addr);
 953 
 954   // atomically update SP keeping back link.
 955   resize_frame(tmp1/* offset */, tmp2/* tmp */);
 956 }
 957 
 958 void MacroAssembler::push_frame(Register bytes, Register tmp) {
 959 #ifdef ASSERT
 960   assert(bytes != R0, "r0 not allowed here");
 961   andi_(R0, bytes, frame::alignment_in_bytes-1);
 962   asm_assert_eq("push_frame(Reg, Reg): unaligned");
 963 #endif
 964   neg(tmp, bytes);
 965   stdux(R1_SP, R1_SP, tmp);
 966 }
 967 
 968 // Push a frame of size `bytes'.
 969 void MacroAssembler::push_frame(unsigned int bytes, Register tmp) {
 970   long offset = align_addr(bytes, frame::alignment_in_bytes);
 971   if (is_simm(-offset, 16)) {
 972     stdu(R1_SP, -offset, R1_SP);
 973   } else {
 974     load_const_optimized(tmp, -offset);
 975     stdux(R1_SP, R1_SP, tmp);
 976   }
 977 }
 978 
 979 // Push a frame of size `bytes' plus abi_reg_args on top.
 980 void MacroAssembler::push_frame_reg_args(unsigned int bytes, Register tmp) {
 981   push_frame(bytes + frame::abi_reg_args_size, tmp);
 982 }
 983 
 984 // Setup up a new C frame with a spill area for non-volatile GPRs and
 985 // additional space for local variables.
 986 void MacroAssembler::push_frame_reg_args_nonvolatiles(unsigned int bytes,
 987                                                       Register tmp) {
 988   push_frame(bytes + frame::abi_reg_args_size + frame::spill_nonvolatiles_size, tmp);
 989 }
 990 
 991 // Pop current C frame.
 992 void MacroAssembler::pop_frame() {
 993   ld(R1_SP, _abi0(callers_sp), R1_SP);
 994 }
 995 
 996 #if defined(ABI_ELFv2)
 997 address MacroAssembler::branch_to(Register r_function_entry, bool and_link) {
 998   // TODO(asmundak): make sure the caller uses R12 as function descriptor
 999   // most of the times.
1000   if (R12 != r_function_entry) {
1001     mr(R12, r_function_entry);
1002   }
1003   mtctr(R12);
1004   // Do a call or a branch.
1005   if (and_link) {
1006     bctrl();
1007   } else {
1008     bctr();
1009   }
1010   _last_calls_return_pc = pc();
1011 
1012   return _last_calls_return_pc;
1013 }
1014 
1015 // Call a C function via a function descriptor and use full C
1016 // calling conventions. Updates and returns _last_calls_return_pc.
1017 address MacroAssembler::call_c(Register r_function_entry) {
1018   return branch_to(r_function_entry, /*and_link=*/true);
1019 }
1020 
1021 // For tail calls: only branch, don't link, so callee returns to caller of this function.
1022 address MacroAssembler::call_c_and_return_to_caller(Register r_function_entry) {
1023   return branch_to(r_function_entry, /*and_link=*/false);
1024 }
1025 
1026 address MacroAssembler::call_c(address function_entry, relocInfo::relocType rt) {
1027   load_const(R12, function_entry, R0);
1028   return branch_to(R12,  /*and_link=*/true);
1029 }
1030 
1031 #else
1032 // Generic version of a call to C function via a function descriptor
1033 // with variable support for C calling conventions (TOC, ENV, etc.).
1034 // Updates and returns _last_calls_return_pc.
1035 address MacroAssembler::branch_to(Register function_descriptor, bool and_link, bool save_toc_before_call,
1036                                   bool restore_toc_after_call, bool load_toc_of_callee, bool load_env_of_callee) {
1037   // we emit standard ptrgl glue code here
1038   assert((function_descriptor != R0), "function_descriptor cannot be R0");
1039 
1040   // retrieve necessary entries from the function descriptor
1041   ld(R0, in_bytes(FunctionDescriptor::entry_offset()), function_descriptor);
1042   mtctr(R0);
1043 
1044   if (load_toc_of_callee) {
1045     ld(R2_TOC, in_bytes(FunctionDescriptor::toc_offset()), function_descriptor);
1046   }
1047   if (load_env_of_callee) {
1048     ld(R11, in_bytes(FunctionDescriptor::env_offset()), function_descriptor);
1049   } else if (load_toc_of_callee) {
1050     li(R11, 0);
1051   }
1052 
1053   // do a call or a branch
1054   if (and_link) {
1055     bctrl();
1056   } else {
1057     bctr();
1058   }
1059   _last_calls_return_pc = pc();
1060 
1061   return _last_calls_return_pc;
1062 }
1063 
1064 // Call a C function via a function descriptor and use full C calling
1065 // conventions.
1066 // We don't use the TOC in generated code, so there is no need to save
1067 // and restore its value.
1068 address MacroAssembler::call_c(Register fd) {
1069   return branch_to(fd, /*and_link=*/true,
1070                        /*save toc=*/false,
1071                        /*restore toc=*/false,
1072                        /*load toc=*/true,
1073                        /*load env=*/true);
1074 }
1075 
1076 address MacroAssembler::call_c_and_return_to_caller(Register fd) {
1077   return branch_to(fd, /*and_link=*/false,
1078                        /*save toc=*/false,
1079                        /*restore toc=*/false,
1080                        /*load toc=*/true,
1081                        /*load env=*/true);
1082 }
1083 
1084 address MacroAssembler::call_c(const FunctionDescriptor* fd, relocInfo::relocType rt) {
1085   if (rt != relocInfo::none) {
1086     // this call needs to be relocatable
1087     if (!ReoptimizeCallSequences
1088         || (rt != relocInfo::runtime_call_type && rt != relocInfo::none)
1089         || fd == NULL   // support code-size estimation
1090         || !fd->is_friend_function()
1091         || fd->entry() == NULL) {
1092       // it's not a friend function as defined by class FunctionDescriptor,
1093       // so do a full call-c here.
1094       load_const(R11, (address)fd, R0);
1095 
1096       bool has_env = (fd != NULL && fd->env() != NULL);
1097       return branch_to(R11, /*and_link=*/true,
1098                             /*save toc=*/false,
1099                             /*restore toc=*/false,
1100                             /*load toc=*/true,
1101                             /*load env=*/has_env);
1102     } else {
1103       // It's a friend function. Load the entry point and don't care about
1104       // toc and env. Use an optimizable call instruction, but ensure the
1105       // same code-size as in the case of a non-friend function.
1106       nop();
1107       nop();
1108       nop();
1109       bl64_patchable(fd->entry(), rt);
1110       _last_calls_return_pc = pc();
1111       return _last_calls_return_pc;
1112     }
1113   } else {
1114     // This call does not need to be relocatable, do more aggressive
1115     // optimizations.
1116     if (!ReoptimizeCallSequences
1117       || !fd->is_friend_function()) {
1118       // It's not a friend function as defined by class FunctionDescriptor,
1119       // so do a full call-c here.
1120       load_const(R11, (address)fd, R0);
1121       return branch_to(R11, /*and_link=*/true,
1122                             /*save toc=*/false,
1123                             /*restore toc=*/false,
1124                             /*load toc=*/true,
1125                             /*load env=*/true);
1126     } else {
1127       // it's a friend function, load the entry point and don't care about
1128       // toc and env.
1129       address dest = fd->entry();
1130       if (is_within_range_of_b(dest, pc())) {
1131         bl(dest);
1132       } else {
1133         bl64_patchable(dest, rt);
1134       }
1135       _last_calls_return_pc = pc();
1136       return _last_calls_return_pc;
1137     }
1138   }
1139 }
1140 
1141 // Call a C function.  All constants needed reside in TOC.
1142 //
1143 // Read the address to call from the TOC.
1144 // Read env from TOC, if fd specifies an env.
1145 // Read new TOC from TOC.
1146 address MacroAssembler::call_c_using_toc(const FunctionDescriptor* fd,
1147                                          relocInfo::relocType rt, Register toc) {
1148   if (!ReoptimizeCallSequences
1149     || (rt != relocInfo::runtime_call_type && rt != relocInfo::none)
1150     || !fd->is_friend_function()) {
1151     // It's not a friend function as defined by class FunctionDescriptor,
1152     // so do a full call-c here.
1153     assert(fd->entry() != NULL, "function must be linked");
1154 
1155     AddressLiteral fd_entry(fd->entry());
1156     bool success = load_const_from_method_toc(R11, fd_entry, toc, /*fixed_size*/ true);
1157     mtctr(R11);
1158     if (fd->env() == NULL) {
1159       li(R11, 0);
1160       nop();
1161     } else {
1162       AddressLiteral fd_env(fd->env());
1163       success = success && load_const_from_method_toc(R11, fd_env, toc, /*fixed_size*/ true);
1164     }
1165     AddressLiteral fd_toc(fd->toc());
1166     // Set R2_TOC (load from toc)
1167     success = success && load_const_from_method_toc(R2_TOC, fd_toc, toc, /*fixed_size*/ true);
1168     bctrl();
1169     _last_calls_return_pc = pc();
1170     if (!success) { return NULL; }
1171   } else {
1172     // It's a friend function, load the entry point and don't care about
1173     // toc and env. Use an optimizable call instruction, but ensure the
1174     // same code-size as in the case of a non-friend function.
1175     nop();
1176     bl64_patchable(fd->entry(), rt);
1177     _last_calls_return_pc = pc();
1178   }
1179   return _last_calls_return_pc;
1180 }
1181 #endif // ABI_ELFv2
1182 
1183 void MacroAssembler::call_VM_base(Register oop_result,
1184                                   Register last_java_sp,
1185                                   address  entry_point,
1186                                   bool     check_exceptions) {
1187   BLOCK_COMMENT("call_VM {");
1188   // Determine last_java_sp register.
1189   if (!last_java_sp->is_valid()) {
1190     last_java_sp = R1_SP;
1191   }
1192   set_top_ijava_frame_at_SP_as_last_Java_frame(last_java_sp, R11_scratch1);
1193 
1194   // ARG1 must hold thread address.
1195   mr(R3_ARG1, R16_thread);
1196 #if defined(ABI_ELFv2)
1197   address return_pc = call_c(entry_point, relocInfo::none);
1198 #else
1199   address return_pc = call_c((FunctionDescriptor*)entry_point, relocInfo::none);
1200 #endif
1201 
1202   reset_last_Java_frame();
1203 
1204   // Check for pending exceptions.
1205   if (check_exceptions) {
1206     // We don't check for exceptions here.
1207     ShouldNotReachHere();
1208   }
1209 
1210   // Get oop result if there is one and reset the value in the thread.
1211   if (oop_result->is_valid()) {
1212     get_vm_result(oop_result);
1213   }
1214 
1215   _last_calls_return_pc = return_pc;
1216   BLOCK_COMMENT("} call_VM");
1217 }
1218 
1219 void MacroAssembler::call_VM_leaf_base(address entry_point) {
1220   BLOCK_COMMENT("call_VM_leaf {");
1221 #if defined(ABI_ELFv2)
1222   call_c(entry_point, relocInfo::none);
1223 #else
1224   call_c(CAST_FROM_FN_PTR(FunctionDescriptor*, entry_point), relocInfo::none);
1225 #endif
1226   BLOCK_COMMENT("} call_VM_leaf");
1227 }
1228 
1229 void MacroAssembler::call_VM(Register oop_result, address entry_point, bool check_exceptions) {
1230   call_VM_base(oop_result, noreg, entry_point, check_exceptions);
1231 }
1232 
1233 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1,
1234                              bool check_exceptions) {
1235   // R3_ARG1 is reserved for the thread.
1236   mr_if_needed(R4_ARG2, arg_1);
1237   call_VM(oop_result, entry_point, check_exceptions);
1238 }
1239 
1240 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2,
1241                              bool check_exceptions) {
1242   // R3_ARG1 is reserved for the thread
1243   mr_if_needed(R4_ARG2, arg_1);
1244   assert(arg_2 != R4_ARG2, "smashed argument");
1245   mr_if_needed(R5_ARG3, arg_2);
1246   call_VM(oop_result, entry_point, check_exceptions);
1247 }
1248 
1249 void MacroAssembler::call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, Register arg_3,
1250                              bool check_exceptions) {
1251   // R3_ARG1 is reserved for the thread
1252   mr_if_needed(R4_ARG2, arg_1);
1253   assert(arg_2 != R4_ARG2, "smashed argument");
1254   mr_if_needed(R5_ARG3, arg_2);
1255   mr_if_needed(R6_ARG4, arg_3);
1256   call_VM(oop_result, entry_point, check_exceptions);
1257 }
1258 
1259 void MacroAssembler::call_VM_leaf(address entry_point) {
1260   call_VM_leaf_base(entry_point);
1261 }
1262 
1263 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1) {
1264   mr_if_needed(R3_ARG1, arg_1);
1265   call_VM_leaf(entry_point);
1266 }
1267 
1268 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2) {
1269   mr_if_needed(R3_ARG1, arg_1);
1270   assert(arg_2 != R3_ARG1, "smashed argument");
1271   mr_if_needed(R4_ARG2, arg_2);
1272   call_VM_leaf(entry_point);
1273 }
1274 
1275 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3) {
1276   mr_if_needed(R3_ARG1, arg_1);
1277   assert(arg_2 != R3_ARG1, "smashed argument");
1278   mr_if_needed(R4_ARG2, arg_2);
1279   assert(arg_3 != R3_ARG1 && arg_3 != R4_ARG2, "smashed argument");
1280   mr_if_needed(R5_ARG3, arg_3);
1281   call_VM_leaf(entry_point);
1282 }
1283 
1284 // Check whether instruction is a read access to the polling page
1285 // which was emitted by load_from_polling_page(..).
1286 bool MacroAssembler::is_load_from_polling_page(int instruction, void* ucontext,
1287                                                address* polling_address_ptr) {
1288   if (!is_ld(instruction))
1289     return false; // It's not a ld. Fail.
1290 
1291   int rt = inv_rt_field(instruction);
1292   int ra = inv_ra_field(instruction);
1293   int ds = inv_ds_field(instruction);
1294   if (!(ds == 0 && ra != 0 && rt == 0)) {
1295     return false; // It's not a ld(r0, X, ra). Fail.
1296   }
1297 
1298   if (!ucontext) {
1299     // Set polling address.
1300     if (polling_address_ptr != NULL) {
1301       *polling_address_ptr = NULL;
1302     }
1303     return true; // No ucontext given. Can't check value of ra. Assume true.
1304   }
1305 
1306 #ifdef LINUX
1307   // Ucontext given. Check that register ra contains the address of
1308   // the safepoing polling page.
1309   ucontext_t* uc = (ucontext_t*) ucontext;
1310   // Set polling address.
1311   address addr = (address)uc->uc_mcontext.regs->gpr[ra] + (ssize_t)ds;
1312   if (polling_address_ptr != NULL) {
1313     *polling_address_ptr = addr;
1314   }
1315   return SafepointMechanism::is_poll_address(addr);
1316 #else
1317   // Not on Linux, ucontext must be NULL.
1318   ShouldNotReachHere();
1319   return false;
1320 #endif
1321 }
1322 
1323 void MacroAssembler::bang_stack_with_offset(int offset) {
1324   // When increasing the stack, the old stack pointer will be written
1325   // to the new top of stack according to the PPC64 abi.
1326   // Therefore, stack banging is not necessary when increasing
1327   // the stack by <= os::vm_page_size() bytes.
1328   // When increasing the stack by a larger amount, this method is
1329   // called repeatedly to bang the intermediate pages.
1330 
1331   // Stack grows down, caller passes positive offset.
1332   assert(offset > 0, "must bang with positive offset");
1333 
1334   long stdoffset = -offset;
1335 
1336   if (is_simm(stdoffset, 16)) {
1337     // Signed 16 bit offset, a simple std is ok.
1338     if (UseLoadInstructionsForStackBangingPPC64) {
1339       ld(R0, (int)(signed short)stdoffset, R1_SP);
1340     } else {
1341       std(R0,(int)(signed short)stdoffset, R1_SP);
1342     }
1343   } else if (is_simm(stdoffset, 31)) {
1344     const int hi = MacroAssembler::largeoffset_si16_si16_hi(stdoffset);
1345     const int lo = MacroAssembler::largeoffset_si16_si16_lo(stdoffset);
1346 
1347     Register tmp = R11;
1348     addis(tmp, R1_SP, hi);
1349     if (UseLoadInstructionsForStackBangingPPC64) {
1350       ld(R0,  lo, tmp);
1351     } else {
1352       std(R0, lo, tmp);
1353     }
1354   } else {
1355     ShouldNotReachHere();
1356   }
1357 }
1358 
1359 // If instruction is a stack bang of the form
1360 //    std    R0,    x(Ry),       (see bang_stack_with_offset())
1361 //    stdu   R1_SP, x(R1_SP),    (see push_frame(), resize_frame())
1362 // or stdux  R1_SP, Rx, R1_SP    (see push_frame(), resize_frame())
1363 // return the banged address. Otherwise, return 0.
1364 address MacroAssembler::get_stack_bang_address(int instruction, void *ucontext) {
1365 #ifdef LINUX
1366   ucontext_t* uc = (ucontext_t*) ucontext;
1367   int rs = inv_rs_field(instruction);
1368   int ra = inv_ra_field(instruction);
1369   if (   (is_ld(instruction)   && rs == 0 &&  UseLoadInstructionsForStackBangingPPC64)
1370       || (is_std(instruction)  && rs == 0 && !UseLoadInstructionsForStackBangingPPC64)
1371       || (is_stdu(instruction) && rs == 1)) {
1372     int ds = inv_ds_field(instruction);
1373     // return banged address
1374     return ds+(address)uc->uc_mcontext.regs->gpr[ra];
1375   } else if (is_stdux(instruction) && rs == 1) {
1376     int rb = inv_rb_field(instruction);
1377     address sp = (address)uc->uc_mcontext.regs->gpr[1];
1378     long rb_val = (long)uc->uc_mcontext.regs->gpr[rb];
1379     return ra != 1 || rb_val >= 0 ? NULL         // not a stack bang
1380                                   : sp + rb_val; // banged address
1381   }
1382   return NULL; // not a stack bang
1383 #else
1384   // workaround not needed on !LINUX :-)
1385   ShouldNotCallThis();
1386   return NULL;
1387 #endif
1388 }
1389 
1390 void MacroAssembler::reserved_stack_check(Register return_pc) {
1391   // Test if reserved zone needs to be enabled.
1392   Label no_reserved_zone_enabling;
1393 
1394   ld_ptr(R0, JavaThread::reserved_stack_activation_offset(), R16_thread);
1395   cmpld(CCR0, R1_SP, R0);
1396   blt_predict_taken(CCR0, no_reserved_zone_enabling);
1397 
1398   // Enable reserved zone again, throw stack overflow exception.
1399   push_frame_reg_args(0, R0);
1400   call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone), R16_thread);
1401   pop_frame();
1402   mtlr(return_pc);
1403   load_const_optimized(R0, StubRoutines::throw_delayed_StackOverflowError_entry());
1404   mtctr(R0);
1405   bctr();
1406 
1407   should_not_reach_here();
1408 
1409   bind(no_reserved_zone_enabling);
1410 }
1411 
1412 void MacroAssembler::getandsetd(Register dest_current_value, Register exchange_value, Register addr_base,
1413                                 bool cmpxchgx_hint) {
1414   Label retry;
1415   bind(retry);
1416   ldarx(dest_current_value, addr_base, cmpxchgx_hint);
1417   stdcx_(exchange_value, addr_base);
1418   if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1419     bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0.
1420   } else {
1421     bne(                  CCR0, retry); // StXcx_ sets CCR0.
1422   }
1423 }
1424 
1425 void MacroAssembler::getandaddd(Register dest_current_value, Register inc_value, Register addr_base,
1426                                 Register tmp, bool cmpxchgx_hint) {
1427   Label retry;
1428   bind(retry);
1429   ldarx(dest_current_value, addr_base, cmpxchgx_hint);
1430   add(tmp, dest_current_value, inc_value);
1431   stdcx_(tmp, addr_base);
1432   if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1433     bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0.
1434   } else {
1435     bne(                  CCR0, retry); // StXcx_ sets CCR0.
1436   }
1437 }
1438 
1439 // Word/sub-word atomic helper functions
1440 
1441 // Temps and addr_base are killed if size < 4 and processor does not support respective instructions.
1442 // Only signed types are supported with size < 4.
1443 // Atomic add always kills tmp1.
1444 void MacroAssembler::atomic_get_and_modify_generic(Register dest_current_value, Register exchange_value,
1445                                                    Register addr_base, Register tmp1, Register tmp2, Register tmp3,
1446                                                    bool cmpxchgx_hint, bool is_add, int size) {
1447   // Sub-word instructions are available since Power 8.
1448   // For older processors, instruction_type != size holds, and we
1449   // emulate the sub-word instructions by constructing a 4-byte value
1450   // that leaves the other bytes unchanged.
1451   const int instruction_type = VM_Version::has_lqarx() ? size : 4;
1452 
1453   Label retry;
1454   Register shift_amount = noreg,
1455            val32 = dest_current_value,
1456            modval = is_add ? tmp1 : exchange_value;
1457 
1458   if (instruction_type != size) {
1459     assert_different_registers(tmp1, tmp2, tmp3, dest_current_value, exchange_value, addr_base);
1460     modval = tmp1;
1461     shift_amount = tmp2;
1462     val32 = tmp3;
1463     // Need some preparation: Compute shift amount, align address. Note: shorts must be 2 byte aligned.
1464 #ifdef VM_LITTLE_ENDIAN
1465     rldic(shift_amount, addr_base, 3, 64-5); // (dest & 3) * 8;
1466     clrrdi(addr_base, addr_base, 2);
1467 #else
1468     xori(shift_amount, addr_base, (size == 1) ? 3 : 2);
1469     clrrdi(addr_base, addr_base, 2);
1470     rldic(shift_amount, shift_amount, 3, 64-5); // byte: ((3-dest) & 3) * 8; short: ((1-dest/2) & 1) * 16;
1471 #endif
1472   }
1473 
1474   // atomic emulation loop
1475   bind(retry);
1476 
1477   switch (instruction_type) {
1478     case 4: lwarx(val32, addr_base, cmpxchgx_hint); break;
1479     case 2: lharx(val32, addr_base, cmpxchgx_hint); break;
1480     case 1: lbarx(val32, addr_base, cmpxchgx_hint); break;
1481     default: ShouldNotReachHere();
1482   }
1483 
1484   if (instruction_type != size) {
1485     srw(dest_current_value, val32, shift_amount);
1486   }
1487 
1488   if (is_add) { add(modval, dest_current_value, exchange_value); }
1489 
1490   if (instruction_type != size) {
1491     // Transform exchange value such that the replacement can be done by one xor instruction.
1492     xorr(modval, dest_current_value, is_add ? modval : exchange_value);
1493     clrldi(modval, modval, (size == 1) ? 56 : 48);
1494     slw(modval, modval, shift_amount);
1495     xorr(modval, val32, modval);
1496   }
1497 
1498   switch (instruction_type) {
1499     case 4: stwcx_(modval, addr_base); break;
1500     case 2: sthcx_(modval, addr_base); break;
1501     case 1: stbcx_(modval, addr_base); break;
1502     default: ShouldNotReachHere();
1503   }
1504 
1505   if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1506     bne_predict_not_taken(CCR0, retry); // StXcx_ sets CCR0.
1507   } else {
1508     bne(                  CCR0, retry); // StXcx_ sets CCR0.
1509   }
1510 
1511   // l?arx zero-extends, but Java wants byte/short values sign-extended.
1512   if (size == 1) {
1513     extsb(dest_current_value, dest_current_value);
1514   } else if (size == 2) {
1515     extsh(dest_current_value, dest_current_value);
1516   };
1517 }
1518 
1519 // Temps, addr_base and exchange_value are killed if size < 4 and processor does not support respective instructions.
1520 // Only signed types are supported with size < 4.
1521 void MacroAssembler::cmpxchg_loop_body(ConditionRegister flag, Register dest_current_value,
1522                                        Register compare_value, Register exchange_value,
1523                                        Register addr_base, Register tmp1, Register tmp2,
1524                                        Label &retry, Label &failed, bool cmpxchgx_hint, int size) {
1525   // Sub-word instructions are available since Power 8.
1526   // For older processors, instruction_type != size holds, and we
1527   // emulate the sub-word instructions by constructing a 4-byte value
1528   // that leaves the other bytes unchanged.
1529   const int instruction_type = VM_Version::has_lqarx() ? size : 4;
1530 
1531   Register shift_amount = noreg,
1532            val32 = dest_current_value,
1533            modval = exchange_value;
1534 
1535   if (instruction_type != size) {
1536     assert_different_registers(tmp1, tmp2, dest_current_value, compare_value, exchange_value, addr_base);
1537     shift_amount = tmp1;
1538     val32 = tmp2;
1539     modval = tmp2;
1540     // Need some preparation: Compute shift amount, align address. Note: shorts must be 2 byte aligned.
1541 #ifdef VM_LITTLE_ENDIAN
1542     rldic(shift_amount, addr_base, 3, 64-5); // (dest & 3) * 8;
1543     clrrdi(addr_base, addr_base, 2);
1544 #else
1545     xori(shift_amount, addr_base, (size == 1) ? 3 : 2);
1546     clrrdi(addr_base, addr_base, 2);
1547     rldic(shift_amount, shift_amount, 3, 64-5); // byte: ((3-dest) & 3) * 8; short: ((1-dest/2) & 1) * 16;
1548 #endif
1549     // Transform exchange value such that the replacement can be done by one xor instruction.
1550     xorr(exchange_value, compare_value, exchange_value);
1551     clrldi(exchange_value, exchange_value, (size == 1) ? 56 : 48);
1552     slw(exchange_value, exchange_value, shift_amount);
1553   }
1554 
1555   // atomic emulation loop
1556   bind(retry);
1557 
1558   switch (instruction_type) {
1559     case 4: lwarx(val32, addr_base, cmpxchgx_hint); break;
1560     case 2: lharx(val32, addr_base, cmpxchgx_hint); break;
1561     case 1: lbarx(val32, addr_base, cmpxchgx_hint); break;
1562     default: ShouldNotReachHere();
1563   }
1564 
1565   if (instruction_type != size) {
1566     srw(dest_current_value, val32, shift_amount);
1567   }
1568   if (size == 1) {
1569     extsb(dest_current_value, dest_current_value);
1570   } else if (size == 2) {
1571     extsh(dest_current_value, dest_current_value);
1572   };
1573 
1574   cmpw(flag, dest_current_value, compare_value);
1575   if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1576     bne_predict_not_taken(flag, failed);
1577   } else {
1578     bne(                  flag, failed);
1579   }
1580   // branch to done  => (flag == ne), (dest_current_value != compare_value)
1581   // fall through    => (flag == eq), (dest_current_value == compare_value)
1582 
1583   if (instruction_type != size) {
1584     xorr(modval, val32, exchange_value);
1585   }
1586 
1587   switch (instruction_type) {
1588     case 4: stwcx_(modval, addr_base); break;
1589     case 2: sthcx_(modval, addr_base); break;
1590     case 1: stbcx_(modval, addr_base); break;
1591     default: ShouldNotReachHere();
1592   }
1593 }
1594 
1595 // CmpxchgX sets condition register to cmpX(current, compare).
1596 void MacroAssembler::cmpxchg_generic(ConditionRegister flag, Register dest_current_value,
1597                                      Register compare_value, Register exchange_value,
1598                                      Register addr_base, Register tmp1, Register tmp2,
1599                                      int semantics, bool cmpxchgx_hint,
1600                                      Register int_flag_success, bool contention_hint, bool weak, int size) {
1601   Label retry;
1602   Label failed;
1603   Label done;
1604 
1605   // Save one branch if result is returned via register and
1606   // result register is different from the other ones.
1607   bool use_result_reg    = (int_flag_success != noreg);
1608   bool preset_result_reg = (int_flag_success != dest_current_value && int_flag_success != compare_value &&
1609                             int_flag_success != exchange_value && int_flag_success != addr_base &&
1610                             int_flag_success != tmp1 && int_flag_success != tmp2);
1611   assert(!weak || flag == CCR0, "weak only supported with CCR0");
1612   assert(size == 1 || size == 2 || size == 4, "unsupported");
1613 
1614   if (use_result_reg && preset_result_reg) {
1615     li(int_flag_success, 0); // preset (assume cas failed)
1616   }
1617 
1618   // Add simple guard in order to reduce risk of starving under high contention (recommended by IBM).
1619   if (contention_hint) { // Don't try to reserve if cmp fails.
1620     switch (size) {
1621       case 1: lbz(dest_current_value, 0, addr_base); extsb(dest_current_value, dest_current_value); break;
1622       case 2: lha(dest_current_value, 0, addr_base); break;
1623       case 4: lwz(dest_current_value, 0, addr_base); break;
1624       default: ShouldNotReachHere();
1625     }
1626     cmpw(flag, dest_current_value, compare_value);
1627     bne(flag, failed);
1628   }
1629 
1630   // release/fence semantics
1631   if (semantics & MemBarRel) {
1632     release();
1633   }
1634 
1635   cmpxchg_loop_body(flag, dest_current_value, compare_value, exchange_value, addr_base, tmp1, tmp2,
1636                     retry, failed, cmpxchgx_hint, size);
1637   if (!weak || use_result_reg) {
1638     if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1639       bne_predict_not_taken(CCR0, weak ? failed : retry); // StXcx_ sets CCR0.
1640     } else {
1641       bne(                  CCR0, weak ? failed : retry); // StXcx_ sets CCR0.
1642     }
1643   }
1644   // fall through    => (flag == eq), (dest_current_value == compare_value), (swapped)
1645 
1646   // Result in register (must do this at the end because int_flag_success can be the
1647   // same register as one above).
1648   if (use_result_reg) {
1649     li(int_flag_success, 1);
1650   }
1651 
1652   if (semantics & MemBarFenceAfter) {
1653     fence();
1654   } else if (semantics & MemBarAcq) {
1655     isync();
1656   }
1657 
1658   if (use_result_reg && !preset_result_reg) {
1659     b(done);
1660   }
1661 
1662   bind(failed);
1663   if (use_result_reg && !preset_result_reg) {
1664     li(int_flag_success, 0);
1665   }
1666 
1667   bind(done);
1668   // (flag == ne) => (dest_current_value != compare_value), (!swapped)
1669   // (flag == eq) => (dest_current_value == compare_value), ( swapped)
1670 }
1671 
1672 // Performs atomic compare exchange:
1673 //   if (compare_value == *addr_base)
1674 //     *addr_base = exchange_value
1675 //     int_flag_success = 1;
1676 //   else
1677 //     int_flag_success = 0;
1678 //
1679 // ConditionRegister flag       = cmp(compare_value, *addr_base)
1680 // Register dest_current_value  = *addr_base
1681 // Register compare_value       Used to compare with value in memory
1682 // Register exchange_value      Written to memory if compare_value == *addr_base
1683 // Register addr_base           The memory location to compareXChange
1684 // Register int_flag_success    Set to 1 if exchange_value was written to *addr_base
1685 //
1686 // To avoid the costly compare exchange the value is tested beforehand.
1687 // Several special cases exist to avoid that unnecessary information is generated.
1688 //
1689 void MacroAssembler::cmpxchgd(ConditionRegister flag,
1690                               Register dest_current_value, RegisterOrConstant compare_value, Register exchange_value,
1691                               Register addr_base, int semantics, bool cmpxchgx_hint,
1692                               Register int_flag_success, Label* failed_ext, bool contention_hint, bool weak) {
1693   Label retry;
1694   Label failed_int;
1695   Label& failed = (failed_ext != NULL) ? *failed_ext : failed_int;
1696   Label done;
1697 
1698   // Save one branch if result is returned via register and result register is different from the other ones.
1699   bool use_result_reg    = (int_flag_success!=noreg);
1700   bool preset_result_reg = (int_flag_success!=dest_current_value && int_flag_success!=compare_value.register_or_noreg() &&
1701                             int_flag_success!=exchange_value && int_flag_success!=addr_base);
1702   assert(!weak || flag == CCR0, "weak only supported with CCR0");
1703   assert(int_flag_success == noreg || failed_ext == NULL, "cannot have both");
1704 
1705   if (use_result_reg && preset_result_reg) {
1706     li(int_flag_success, 0); // preset (assume cas failed)
1707   }
1708 
1709   // Add simple guard in order to reduce risk of starving under high contention (recommended by IBM).
1710   if (contention_hint) { // Don't try to reserve if cmp fails.
1711     ld(dest_current_value, 0, addr_base);
1712     cmpd(flag, compare_value, dest_current_value);
1713     bne(flag, failed);
1714   }
1715 
1716   // release/fence semantics
1717   if (semantics & MemBarRel) {
1718     release();
1719   }
1720 
1721   // atomic emulation loop
1722   bind(retry);
1723 
1724   ldarx(dest_current_value, addr_base, cmpxchgx_hint);
1725   cmpd(flag, compare_value, dest_current_value);
1726   if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1727     bne_predict_not_taken(flag, failed);
1728   } else {
1729     bne(                  flag, failed);
1730   }
1731 
1732   stdcx_(exchange_value, addr_base);
1733   if (!weak || use_result_reg || failed_ext) {
1734     if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
1735       bne_predict_not_taken(CCR0, weak ? failed : retry); // stXcx_ sets CCR0
1736     } else {
1737       bne(                  CCR0, weak ? failed : retry); // stXcx_ sets CCR0
1738     }
1739   }
1740 
1741   // result in register (must do this at the end because int_flag_success can be the same register as one above)
1742   if (use_result_reg) {
1743     li(int_flag_success, 1);
1744   }
1745 
1746   if (semantics & MemBarFenceAfter) {
1747     fence();
1748   } else if (semantics & MemBarAcq) {
1749     isync();
1750   }
1751 
1752   if (use_result_reg && !preset_result_reg) {
1753     b(done);
1754   }
1755 
1756   bind(failed_int);
1757   if (use_result_reg && !preset_result_reg) {
1758     li(int_flag_success, 0);
1759   }
1760 
1761   bind(done);
1762   // (flag == ne) => (dest_current_value != compare_value), (!swapped)
1763   // (flag == eq) => (dest_current_value == compare_value), ( swapped)
1764 }
1765 
1766 // Look up the method for a megamorphic invokeinterface call.
1767 // The target method is determined by <intf_klass, itable_index>.
1768 // The receiver klass is in recv_klass.
1769 // On success, the result will be in method_result, and execution falls through.
1770 // On failure, execution transfers to the given label.
1771 void MacroAssembler::lookup_interface_method(Register recv_klass,
1772                                              Register intf_klass,
1773                                              RegisterOrConstant itable_index,
1774                                              Register method_result,
1775                                              Register scan_temp,
1776                                              Register temp2,
1777                                              Label& L_no_such_interface,
1778                                              bool return_method) {
1779   assert_different_registers(recv_klass, intf_klass, method_result, scan_temp);
1780 
1781   // Compute start of first itableOffsetEntry (which is at the end of the vtable).
1782   int vtable_base = in_bytes(Klass::vtable_start_offset());
1783   int itentry_off = itableMethodEntry::method_offset_in_bytes();
1784   int logMEsize   = exact_log2(itableMethodEntry::size() * wordSize);
1785   int scan_step   = itableOffsetEntry::size() * wordSize;
1786   int log_vte_size= exact_log2(vtableEntry::size_in_bytes());
1787 
1788   lwz(scan_temp, in_bytes(Klass::vtable_length_offset()), recv_klass);
1789   // %%% We should store the aligned, prescaled offset in the klassoop.
1790   // Then the next several instructions would fold away.
1791 
1792   sldi(scan_temp, scan_temp, log_vte_size);
1793   addi(scan_temp, scan_temp, vtable_base);
1794   add(scan_temp, recv_klass, scan_temp);
1795 
1796   // Adjust recv_klass by scaled itable_index, so we can free itable_index.
1797   if (return_method) {
1798     if (itable_index.is_register()) {
1799       Register itable_offset = itable_index.as_register();
1800       sldi(method_result, itable_offset, logMEsize);
1801       if (itentry_off) { addi(method_result, method_result, itentry_off); }
1802       add(method_result, method_result, recv_klass);
1803     } else {
1804       long itable_offset = (long)itable_index.as_constant();
1805       // static address, no relocation
1806       add_const_optimized(method_result, recv_klass, (itable_offset << logMEsize) + itentry_off, temp2);
1807     }
1808   }
1809 
1810   // for (scan = klass->itable(); scan->interface() != NULL; scan += scan_step) {
1811   //   if (scan->interface() == intf) {
1812   //     result = (klass + scan->offset() + itable_index);
1813   //   }
1814   // }
1815   Label search, found_method;
1816 
1817   for (int peel = 1; peel >= 0; peel--) {
1818     // %%%% Could load both offset and interface in one ldx, if they were
1819     // in the opposite order. This would save a load.
1820     ld(temp2, itableOffsetEntry::interface_offset_in_bytes(), scan_temp);
1821 
1822     // Check that this entry is non-null. A null entry means that
1823     // the receiver class doesn't implement the interface, and wasn't the
1824     // same as when the caller was compiled.
1825     cmpd(CCR0, temp2, intf_klass);
1826 
1827     if (peel) {
1828       beq(CCR0, found_method);
1829     } else {
1830       bne(CCR0, search);
1831       // (invert the test to fall through to found_method...)
1832     }
1833 
1834     if (!peel) break;
1835 
1836     bind(search);
1837 
1838     cmpdi(CCR0, temp2, 0);
1839     beq(CCR0, L_no_such_interface);
1840     addi(scan_temp, scan_temp, scan_step);
1841   }
1842 
1843   bind(found_method);
1844 
1845   // Got a hit.
1846   if (return_method) {
1847     int ito_offset = itableOffsetEntry::offset_offset_in_bytes();
1848     lwz(scan_temp, ito_offset, scan_temp);
1849     ldx(method_result, scan_temp, method_result);
1850   }
1851 }
1852 
1853 // virtual method calling
1854 void MacroAssembler::lookup_virtual_method(Register recv_klass,
1855                                            RegisterOrConstant vtable_index,
1856                                            Register method_result) {
1857 
1858   assert_different_registers(recv_klass, method_result, vtable_index.register_or_noreg());
1859 
1860   const int base = in_bytes(Klass::vtable_start_offset());
1861   assert(vtableEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
1862 
1863   if (vtable_index.is_register()) {
1864     sldi(vtable_index.as_register(), vtable_index.as_register(), LogBytesPerWord);
1865     add(recv_klass, vtable_index.as_register(), recv_klass);
1866   } else {
1867     addi(recv_klass, recv_klass, vtable_index.as_constant() << LogBytesPerWord);
1868   }
1869   ld(R19_method, base + vtableEntry::method_offset_in_bytes(), recv_klass);
1870 }
1871 
1872 /////////////////////////////////////////// subtype checking ////////////////////////////////////////////
1873 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
1874                                                    Register super_klass,
1875                                                    Register temp1_reg,
1876                                                    Register temp2_reg,
1877                                                    Label* L_success,
1878                                                    Label* L_failure,
1879                                                    Label* L_slow_path,
1880                                                    RegisterOrConstant super_check_offset) {
1881 
1882   const Register check_cache_offset = temp1_reg;
1883   const Register cached_super       = temp2_reg;
1884 
1885   assert_different_registers(sub_klass, super_klass, check_cache_offset, cached_super);
1886 
1887   int sco_offset = in_bytes(Klass::super_check_offset_offset());
1888   int sc_offset  = in_bytes(Klass::secondary_super_cache_offset());
1889 
1890   bool must_load_sco = (super_check_offset.constant_or_zero() == -1);
1891   bool need_slow_path = (must_load_sco || super_check_offset.constant_or_zero() == sco_offset);
1892 
1893   Label L_fallthrough;
1894   int label_nulls = 0;
1895   if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
1896   if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
1897   if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
1898   assert(label_nulls <= 1 ||
1899          (L_slow_path == &L_fallthrough && label_nulls <= 2 && !need_slow_path),
1900          "at most one NULL in the batch, usually");
1901 
1902   // If the pointers are equal, we are done (e.g., String[] elements).
1903   // This self-check enables sharing of secondary supertype arrays among
1904   // non-primary types such as array-of-interface. Otherwise, each such
1905   // type would need its own customized SSA.
1906   // We move this check to the front of the fast path because many
1907   // type checks are in fact trivially successful in this manner,
1908   // so we get a nicely predicted branch right at the start of the check.
1909   cmpd(CCR0, sub_klass, super_klass);
1910   beq(CCR0, *L_success);
1911 
1912   // Check the supertype display:
1913   if (must_load_sco) {
1914     // The super check offset is always positive...
1915     lwz(check_cache_offset, sco_offset, super_klass);
1916     super_check_offset = RegisterOrConstant(check_cache_offset);
1917     // super_check_offset is register.
1918     assert_different_registers(sub_klass, super_klass, cached_super, super_check_offset.as_register());
1919   }
1920   // The loaded value is the offset from KlassOopDesc.
1921 
1922   ld(cached_super, super_check_offset, sub_klass);
1923   cmpd(CCR0, cached_super, super_klass);
1924 
1925   // This check has worked decisively for primary supers.
1926   // Secondary supers are sought in the super_cache ('super_cache_addr').
1927   // (Secondary supers are interfaces and very deeply nested subtypes.)
1928   // This works in the same check above because of a tricky aliasing
1929   // between the super_cache and the primary super display elements.
1930   // (The 'super_check_addr' can address either, as the case requires.)
1931   // Note that the cache is updated below if it does not help us find
1932   // what we need immediately.
1933   // So if it was a primary super, we can just fail immediately.
1934   // Otherwise, it's the slow path for us (no success at this point).
1935 
1936 #define FINAL_JUMP(label) if (&(label) != &L_fallthrough) { b(label); }
1937 
1938   if (super_check_offset.is_register()) {
1939     beq(CCR0, *L_success);
1940     cmpwi(CCR0, super_check_offset.as_register(), sc_offset);
1941     if (L_failure == &L_fallthrough) {
1942       beq(CCR0, *L_slow_path);
1943     } else {
1944       bne(CCR0, *L_failure);
1945       FINAL_JUMP(*L_slow_path);
1946     }
1947   } else {
1948     if (super_check_offset.as_constant() == sc_offset) {
1949       // Need a slow path; fast failure is impossible.
1950       if (L_slow_path == &L_fallthrough) {
1951         beq(CCR0, *L_success);
1952       } else {
1953         bne(CCR0, *L_slow_path);
1954         FINAL_JUMP(*L_success);
1955       }
1956     } else {
1957       // No slow path; it's a fast decision.
1958       if (L_failure == &L_fallthrough) {
1959         beq(CCR0, *L_success);
1960       } else {
1961         bne(CCR0, *L_failure);
1962         FINAL_JUMP(*L_success);
1963       }
1964     }
1965   }
1966 
1967   bind(L_fallthrough);
1968 #undef FINAL_JUMP
1969 }
1970 
1971 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
1972                                                    Register super_klass,
1973                                                    Register temp1_reg,
1974                                                    Register temp2_reg,
1975                                                    Label* L_success,
1976                                                    Register result_reg) {
1977   const Register array_ptr = temp1_reg; // current value from cache array
1978   const Register temp      = temp2_reg;
1979 
1980   assert_different_registers(sub_klass, super_klass, array_ptr, temp);
1981 
1982   int source_offset = in_bytes(Klass::secondary_supers_offset());
1983   int target_offset = in_bytes(Klass::secondary_super_cache_offset());
1984 
1985   int length_offset = Array<Klass*>::length_offset_in_bytes();
1986   int base_offset   = Array<Klass*>::base_offset_in_bytes();
1987 
1988   Label hit, loop, failure, fallthru;
1989 
1990   ld(array_ptr, source_offset, sub_klass);
1991 
1992   // TODO: PPC port: assert(4 == arrayOopDesc::length_length_in_bytes(), "precondition violated.");
1993   lwz(temp, length_offset, array_ptr);
1994   cmpwi(CCR0, temp, 0);
1995   beq(CCR0, result_reg!=noreg ? failure : fallthru); // length 0
1996 
1997   mtctr(temp); // load ctr
1998 
1999   bind(loop);
2000   // Oops in table are NO MORE compressed.
2001   ld(temp, base_offset, array_ptr);
2002   cmpd(CCR0, temp, super_klass);
2003   beq(CCR0, hit);
2004   addi(array_ptr, array_ptr, BytesPerWord);
2005   bdnz(loop);
2006 
2007   bind(failure);
2008   if (result_reg!=noreg) li(result_reg, 1); // load non-zero result (indicates a miss)
2009   b(fallthru);
2010 
2011   bind(hit);
2012   std(super_klass, target_offset, sub_klass); // save result to cache
2013   if (result_reg != noreg) { li(result_reg, 0); } // load zero result (indicates a hit)
2014   if (L_success != NULL) { b(*L_success); }
2015   else if (result_reg == noreg) { blr(); } // return with CR0.eq if neither label nor result reg provided
2016 
2017   bind(fallthru);
2018 }
2019 
2020 // Try fast path, then go to slow one if not successful
2021 void MacroAssembler::check_klass_subtype(Register sub_klass,
2022                          Register super_klass,
2023                          Register temp1_reg,
2024                          Register temp2_reg,
2025                          Label& L_success) {
2026   Label L_failure;
2027   check_klass_subtype_fast_path(sub_klass, super_klass, temp1_reg, temp2_reg, &L_success, &L_failure);
2028   check_klass_subtype_slow_path(sub_klass, super_klass, temp1_reg, temp2_reg, &L_success);
2029   bind(L_failure); // Fallthru if not successful.
2030 }
2031 
2032 void MacroAssembler::clinit_barrier(Register klass, Register thread, Label* L_fast_path, Label* L_slow_path) {
2033   assert(L_fast_path != NULL || L_slow_path != NULL, "at least one is required");
2034 
2035   Label L_fallthrough;
2036   if (L_fast_path == NULL) {
2037     L_fast_path = &L_fallthrough;
2038   } else if (L_slow_path == NULL) {
2039     L_slow_path = &L_fallthrough;
2040   }
2041 
2042   // Fast path check: class is fully initialized
2043   lbz(R0, in_bytes(InstanceKlass::init_state_offset()), klass);
2044   cmpwi(CCR0, R0, InstanceKlass::fully_initialized);
2045   beq(CCR0, *L_fast_path);
2046 
2047   // Fast path check: current thread is initializer thread
2048   ld(R0, in_bytes(InstanceKlass::init_thread_offset()), klass);
2049   cmpd(CCR0, thread, R0);
2050   if (L_slow_path == &L_fallthrough) {
2051     beq(CCR0, *L_fast_path);
2052   } else if (L_fast_path == &L_fallthrough) {
2053     bne(CCR0, *L_slow_path);
2054   } else {
2055     Unimplemented();
2056   }
2057 
2058   bind(L_fallthrough);
2059 }
2060 
2061 RegisterOrConstant MacroAssembler::argument_offset(RegisterOrConstant arg_slot,
2062                                                    Register temp_reg,
2063                                                    int extra_slot_offset) {
2064   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
2065   int stackElementSize = Interpreter::stackElementSize;
2066   int offset = extra_slot_offset * stackElementSize;
2067   if (arg_slot.is_constant()) {
2068     offset += arg_slot.as_constant() * stackElementSize;
2069     return offset;
2070   } else {
2071     assert(temp_reg != noreg, "must specify");
2072     sldi(temp_reg, arg_slot.as_register(), exact_log2(stackElementSize));
2073     if (offset != 0)
2074       addi(temp_reg, temp_reg, offset);
2075     return temp_reg;
2076   }
2077 }
2078 
2079 void MacroAssembler::tlab_allocate(
2080   Register obj,                      // result: pointer to object after successful allocation
2081   Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
2082   int      con_size_in_bytes,        // object size in bytes if   known at compile time
2083   Register t1,                       // temp register
2084   Label&   slow_case                 // continuation point if fast allocation fails
2085 ) {
2086   // make sure arguments make sense
2087   assert_different_registers(obj, var_size_in_bytes, t1);
2088   assert(0 <= con_size_in_bytes && is_simm16(con_size_in_bytes), "illegal object size");
2089   assert((con_size_in_bytes & MinObjAlignmentInBytesMask) == 0, "object size is not multiple of alignment");
2090 
2091   const Register new_top = t1;
2092   //verify_tlab(); not implemented
2093 
2094   ld(obj, in_bytes(JavaThread::tlab_top_offset()), R16_thread);
2095   ld(R0, in_bytes(JavaThread::tlab_end_offset()), R16_thread);
2096   if (var_size_in_bytes == noreg) {
2097     addi(new_top, obj, con_size_in_bytes);
2098   } else {
2099     add(new_top, obj, var_size_in_bytes);
2100   }
2101   cmpld(CCR0, new_top, R0);
2102   bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::greater), slow_case);
2103 
2104 #ifdef ASSERT
2105   // make sure new free pointer is properly aligned
2106   {
2107     Label L;
2108     andi_(R0, new_top, MinObjAlignmentInBytesMask);
2109     beq(CCR0, L);
2110     stop("updated TLAB free is not properly aligned");
2111     bind(L);
2112   }
2113 #endif // ASSERT
2114 
2115   // update the tlab top pointer
2116   std(new_top, in_bytes(JavaThread::tlab_top_offset()), R16_thread);
2117   //verify_tlab(); not implemented
2118 }
2119 void MacroAssembler::incr_allocated_bytes(RegisterOrConstant size_in_bytes, Register t1, Register t2) {
2120   unimplemented("incr_allocated_bytes");
2121 }
2122 
2123 address MacroAssembler::emit_trampoline_stub(int destination_toc_offset,
2124                                              int insts_call_instruction_offset, Register Rtoc) {
2125   // Start the stub.
2126   address stub = start_a_stub(64);
2127   if (stub == NULL) { return NULL; } // CodeCache full: bail out
2128 
2129   // Create a trampoline stub relocation which relates this trampoline stub
2130   // with the call instruction at insts_call_instruction_offset in the
2131   // instructions code-section.
2132   relocate(trampoline_stub_Relocation::spec(code()->insts()->start() + insts_call_instruction_offset));
2133   const int stub_start_offset = offset();
2134 
2135   // For java_to_interp stubs we use R11_scratch1 as scratch register
2136   // and in call trampoline stubs we use R12_scratch2. This way we
2137   // can distinguish them (see is_NativeCallTrampolineStub_at()).
2138   Register reg_scratch = R12_scratch2;
2139 
2140   // Now, create the trampoline stub's code:
2141   // - load the TOC
2142   // - load the call target from the constant pool
2143   // - call
2144   if (Rtoc == noreg) {
2145     calculate_address_from_global_toc(reg_scratch, method_toc());
2146     Rtoc = reg_scratch;
2147   }
2148 
2149   ld_largeoffset_unchecked(reg_scratch, destination_toc_offset, Rtoc, false);
2150   mtctr(reg_scratch);
2151   bctr();
2152 
2153   const address stub_start_addr = addr_at(stub_start_offset);
2154 
2155   // Assert that the encoded destination_toc_offset can be identified and that it is correct.
2156   assert(destination_toc_offset == NativeCallTrampolineStub_at(stub_start_addr)->destination_toc_offset(),
2157          "encoded offset into the constant pool must match");
2158   // Trampoline_stub_size should be good.
2159   assert((uint)(offset() - stub_start_offset) <= trampoline_stub_size, "should be good size");
2160   assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline");
2161 
2162   // End the stub.
2163   end_a_stub();
2164   return stub;
2165 }
2166 
2167 // TM on PPC64.
2168 void MacroAssembler::atomic_inc_ptr(Register addr, Register result, int simm16) {
2169   Label retry;
2170   bind(retry);
2171   ldarx(result, addr, /*hint*/ false);
2172   addi(result, result, simm16);
2173   stdcx_(result, addr);
2174   if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
2175     bne_predict_not_taken(CCR0, retry); // stXcx_ sets CCR0
2176   } else {
2177     bne(                  CCR0, retry); // stXcx_ sets CCR0
2178   }
2179 }
2180 
2181 void MacroAssembler::atomic_ori_int(Register addr, Register result, int uimm16) {
2182   Label retry;
2183   bind(retry);
2184   lwarx(result, addr, /*hint*/ false);
2185   ori(result, result, uimm16);
2186   stwcx_(result, addr);
2187   if (UseStaticBranchPredictionInCompareAndSwapPPC64) {
2188     bne_predict_not_taken(CCR0, retry); // stXcx_ sets CCR0
2189   } else {
2190     bne(                  CCR0, retry); // stXcx_ sets CCR0
2191   }
2192 }
2193 
2194 #if INCLUDE_RTM_OPT
2195 
2196 // Update rtm_counters based on abort status
2197 // input: abort_status
2198 //        rtm_counters_Reg (RTMLockingCounters*)
2199 void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters_Reg) {
2200   // Mapping to keep PreciseRTMLockingStatistics similar to x86.
2201   // x86 ppc (! means inverted, ? means not the same)
2202   //  0   31  Set if abort caused by XABORT instruction.
2203   //  1  ! 7  If set, the transaction may succeed on a retry. This bit is always clear if bit 0 is set.
2204   //  2   13  Set if another logical processor conflicted with a memory address that was part of the transaction that aborted.
2205   //  3   10  Set if an internal buffer overflowed.
2206   //  4  ?12  Set if a debug breakpoint was hit.
2207   //  5  ?32  Set if an abort occurred during execution of a nested transaction.
2208   const int failure_bit[] = {tm_tabort, // Signal handler will set this too.
2209                              tm_failure_persistent,
2210                              tm_non_trans_cf,
2211                              tm_trans_cf,
2212                              tm_footprint_of,
2213                              tm_failure_code,
2214                              tm_transaction_level};
2215 
2216   const int num_failure_bits = sizeof(failure_bit) / sizeof(int);
2217   const int num_counters = RTMLockingCounters::ABORT_STATUS_LIMIT;
2218 
2219   const int bit2counter_map[][num_counters] =
2220   // 0 = no map; 1 = mapped, no inverted logic; -1 = mapped, inverted logic
2221   // Inverted logic means that if a bit is set don't count it, or vice-versa.
2222   // Care must be taken when mapping bits to counters as bits for a given
2223   // counter must be mutually exclusive. Otherwise, the counter will be
2224   // incremented more than once.
2225   // counters:
2226   // 0        1        2         3         4         5
2227   // abort  , persist, conflict, overflow, debug   , nested         bits:
2228   {{ 1      , 0      , 0       , 0       , 0       , 0      },   // abort
2229    { 0      , -1     , 0       , 0       , 0       , 0      },   // failure_persistent
2230    { 0      , 0      , 1       , 0       , 0       , 0      },   // non_trans_cf
2231    { 0      , 0      , 1       , 0       , 0       , 0      },   // trans_cf
2232    { 0      , 0      , 0       , 1       , 0       , 0      },   // footprint_of
2233    { 0      , 0      , 0       , 0       , -1      , 0      },   // failure_code = 0xD4
2234    { 0      , 0      , 0       , 0       , 0       , 1      }};  // transaction_level > 1
2235   // ...
2236 
2237   // Move abort_status value to R0 and use abort_status register as a
2238   // temporary register because R0 as third operand in ld/std is treated
2239   // as base address zero (value). Likewise, R0 as second operand in addi
2240   // is problematic because it amounts to li.
2241   const Register temp_Reg = abort_status;
2242   const Register abort_status_R0 = R0;
2243   mr(abort_status_R0, abort_status);
2244 
2245   // Increment total abort counter.
2246   int counters_offs = RTMLockingCounters::abort_count_offset();
2247   ld(temp_Reg, counters_offs, rtm_counters_Reg);
2248   addi(temp_Reg, temp_Reg, 1);
2249   std(temp_Reg, counters_offs, rtm_counters_Reg);
2250 
2251   // Increment specific abort counters.
2252   if (PrintPreciseRTMLockingStatistics) {
2253 
2254     // #0 counter offset.
2255     int abortX_offs = RTMLockingCounters::abortX_count_offset();
2256 
2257     for (int nbit = 0; nbit < num_failure_bits; nbit++) {
2258       for (int ncounter = 0; ncounter < num_counters; ncounter++) {
2259         if (bit2counter_map[nbit][ncounter] != 0) {
2260           Label check_abort;
2261           int abort_counter_offs = abortX_offs + (ncounter << 3);
2262 
2263           if (failure_bit[nbit] == tm_transaction_level) {
2264             // Don't check outer transaction, TL = 1 (bit 63). Hence only
2265             // 11 bits in the TL field are checked to find out if failure
2266             // occurred in a nested transaction. This check also matches
2267             // the case when nesting_of = 1 (nesting overflow).
2268             rldicr_(temp_Reg, abort_status_R0, failure_bit[nbit], 10);
2269           } else if (failure_bit[nbit] == tm_failure_code) {
2270             // Check failure code for trap or illegal caught in TM.
2271             // Bits 0:7 are tested as bit 7 (persistent) is copied from
2272             // tabort or treclaim source operand.
2273             // On Linux: trap or illegal is TM_CAUSE_SIGNAL (0xD4).
2274             rldicl(temp_Reg, abort_status_R0, 8, 56);
2275             cmpdi(CCR0, temp_Reg, 0xD4);
2276           } else {
2277             rldicr_(temp_Reg, abort_status_R0, failure_bit[nbit], 0);
2278           }
2279 
2280           if (bit2counter_map[nbit][ncounter] == 1) {
2281             beq(CCR0, check_abort);
2282           } else {
2283             bne(CCR0, check_abort);
2284           }
2285 
2286           // We don't increment atomically.
2287           ld(temp_Reg, abort_counter_offs, rtm_counters_Reg);
2288           addi(temp_Reg, temp_Reg, 1);
2289           std(temp_Reg, abort_counter_offs, rtm_counters_Reg);
2290 
2291           bind(check_abort);
2292         }
2293       }
2294     }
2295   }
2296   // Restore abort_status.
2297   mr(abort_status, abort_status_R0);
2298 }
2299 
2300 // Branch if (random & (count-1) != 0), count is 2^n
2301 // tmp and CR0 are killed
2302 void MacroAssembler::branch_on_random_using_tb(Register tmp, int count, Label& brLabel) {
2303   mftb(tmp);
2304   andi_(tmp, tmp, count-1);
2305   bne(CCR0, brLabel);
2306 }
2307 
2308 // Perform abort ratio calculation, set no_rtm bit if high ratio.
2309 // input:  rtm_counters_Reg (RTMLockingCounters* address) - KILLED
2310 void MacroAssembler::rtm_abort_ratio_calculation(Register rtm_counters_Reg,
2311                                                  RTMLockingCounters* rtm_counters,
2312                                                  Metadata* method_data) {
2313   Label L_done, L_check_always_rtm1, L_check_always_rtm2;
2314 
2315   if (RTMLockingCalculationDelay > 0) {
2316     // Delay calculation.
2317     ld(rtm_counters_Reg, (RegisterOrConstant)(intptr_t)RTMLockingCounters::rtm_calculation_flag_addr());
2318     cmpdi(CCR0, rtm_counters_Reg, 0);
2319     beq(CCR0, L_done);
2320     load_const_optimized(rtm_counters_Reg, (address)rtm_counters, R0); // reload
2321   }
2322   // Abort ratio calculation only if abort_count > RTMAbortThreshold.
2323   //   Aborted transactions = abort_count * 100
2324   //   All transactions = total_count *  RTMTotalCountIncrRate
2325   //   Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio)
2326   ld(R0, RTMLockingCounters::abort_count_offset(), rtm_counters_Reg);
2327   if (is_simm(RTMAbortThreshold, 16)) {   // cmpdi can handle 16bit immediate only.
2328     cmpdi(CCR0, R0, RTMAbortThreshold);
2329     blt(CCR0, L_check_always_rtm2);  // reload of rtm_counters_Reg not necessary
2330   } else {
2331     load_const_optimized(rtm_counters_Reg, RTMAbortThreshold);
2332     cmpd(CCR0, R0, rtm_counters_Reg);
2333     blt(CCR0, L_check_always_rtm1);  // reload of rtm_counters_Reg required
2334   }
2335   mulli(R0, R0, 100);
2336 
2337   const Register tmpReg = rtm_counters_Reg;
2338   ld(tmpReg, RTMLockingCounters::total_count_offset(), rtm_counters_Reg);
2339   mulli(tmpReg, tmpReg, RTMTotalCountIncrRate); // allowable range: int16
2340   mulli(tmpReg, tmpReg, RTMAbortRatio);         // allowable range: int16
2341   cmpd(CCR0, R0, tmpReg);
2342   blt(CCR0, L_check_always_rtm1); // jump to reload
2343   if (method_data != NULL) {
2344     // Set rtm_state to "no rtm" in MDO.
2345     // Not using a metadata relocation. Method and Class Loader are kept alive anyway.
2346     // (See nmethod::metadata_do and CodeBuffer::finalize_oop_references.)
2347     load_const(R0, (address)method_data + MethodData::rtm_state_offset_in_bytes(), tmpReg);
2348     atomic_ori_int(R0, tmpReg, NoRTM);
2349   }
2350   b(L_done);
2351 
2352   bind(L_check_always_rtm1);
2353   load_const_optimized(rtm_counters_Reg, (address)rtm_counters, R0); // reload
2354   bind(L_check_always_rtm2);
2355   ld(tmpReg, RTMLockingCounters::total_count_offset(), rtm_counters_Reg);
2356   int64_t thresholdValue = RTMLockingThreshold / RTMTotalCountIncrRate;
2357   if (is_simm(thresholdValue, 16)) {   // cmpdi can handle 16bit immediate only.
2358     cmpdi(CCR0, tmpReg, thresholdValue);
2359   } else {
2360     load_const_optimized(R0, thresholdValue);
2361     cmpd(CCR0, tmpReg, R0);
2362   }
2363   blt(CCR0, L_done);
2364   if (method_data != NULL) {
2365     // Set rtm_state to "always rtm" in MDO.
2366     // Not using a metadata relocation. See above.
2367     load_const(R0, (address)method_data + MethodData::rtm_state_offset_in_bytes(), tmpReg);
2368     atomic_ori_int(R0, tmpReg, UseRTM);
2369   }
2370   bind(L_done);
2371 }
2372 
2373 // Update counters and perform abort ratio calculation.
2374 // input: abort_status_Reg
2375 void MacroAssembler::rtm_profiling(Register abort_status_Reg, Register temp_Reg,
2376                                    RTMLockingCounters* rtm_counters,
2377                                    Metadata* method_data,
2378                                    bool profile_rtm) {
2379 
2380   assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
2381   // Update rtm counters based on state at abort.
2382   // Reads abort_status_Reg, updates flags.
2383   assert_different_registers(abort_status_Reg, temp_Reg);
2384   load_const_optimized(temp_Reg, (address)rtm_counters, R0);
2385   rtm_counters_update(abort_status_Reg, temp_Reg);
2386   if (profile_rtm) {
2387     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
2388     rtm_abort_ratio_calculation(temp_Reg, rtm_counters, method_data);
2389   }
2390 }
2391 
2392 // Retry on abort if abort's status indicates non-persistent failure.
2393 // inputs: retry_count_Reg
2394 //       : abort_status_Reg
2395 // output: retry_count_Reg decremented by 1
2396 void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg,
2397                                              Label& retryLabel, Label* checkRetry) {
2398   Label doneRetry;
2399 
2400   // Don't retry if failure is persistent.
2401   // The persistent bit is set when a (A) Disallowed operation is performed in
2402   // transactional state, like for instance trying to write the TFHAR after a
2403   // transaction is started; or when there is (B) a Nesting Overflow (too many
2404   // nested transactions); or when (C) the Footprint overflows (too many
2405   // addresses touched in TM state so there is no more space in the footprint
2406   // area to track them); or in case of (D) a Self-Induced Conflict, i.e. a
2407   // store is performed to a given address in TM state, then once in suspended
2408   // state the same address is accessed. Failure (A) is very unlikely to occur
2409   // in the JVM. Failure (D) will never occur because Suspended state is never
2410   // used in the JVM. Thus mostly (B) a Nesting Overflow or (C) a Footprint
2411   // Overflow will set the persistent bit.
2412   rldicr_(R0, abort_status_Reg, tm_failure_persistent, 0);
2413   bne(CCR0, doneRetry);
2414 
2415   // Don't retry if transaction was deliberately aborted, i.e. caused by a
2416   // tabort instruction.
2417   rldicr_(R0, abort_status_Reg, tm_tabort, 0);
2418   bne(CCR0, doneRetry);
2419 
2420   // Retry if transaction aborted due to a conflict with another thread.
2421   if (checkRetry) { bind(*checkRetry); }
2422   addic_(retry_count_Reg, retry_count_Reg, -1);
2423   blt(CCR0, doneRetry);
2424   b(retryLabel);
2425   bind(doneRetry);
2426 }
2427 
2428 // Spin and retry if lock is busy.
2429 // inputs: owner_addr_Reg (monitor address)
2430 //       : retry_count_Reg
2431 // output: retry_count_Reg decremented by 1
2432 // CTR is killed
2433 void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register owner_addr_Reg, Label& retryLabel) {
2434   Label SpinLoop, doneRetry, doRetry;
2435   addic_(retry_count_Reg, retry_count_Reg, -1);
2436   blt(CCR0, doneRetry);
2437 
2438   if (RTMSpinLoopCount > 1) {
2439     li(R0, RTMSpinLoopCount);
2440     mtctr(R0);
2441   }
2442 
2443   // low thread priority
2444   smt_prio_low();
2445   bind(SpinLoop);
2446 
2447   if (RTMSpinLoopCount > 1) {
2448     bdz(doRetry);
2449     ld(R0, 0, owner_addr_Reg);
2450     cmpdi(CCR0, R0, 0);
2451     bne(CCR0, SpinLoop);
2452   }
2453 
2454   bind(doRetry);
2455 
2456   // restore thread priority to default in userspace
2457 #ifdef LINUX
2458   smt_prio_medium_low();
2459 #else
2460   smt_prio_medium();
2461 #endif
2462 
2463   b(retryLabel);
2464 
2465   bind(doneRetry);
2466 }
2467 
2468 // Use RTM for normal stack locks.
2469 // Input: objReg (object to lock)
2470 void MacroAssembler::rtm_stack_locking(ConditionRegister flag,
2471                                        Register obj, Register mark_word, Register tmp,
2472                                        Register retry_on_abort_count_Reg,
2473                                        RTMLockingCounters* stack_rtm_counters,
2474                                        Metadata* method_data, bool profile_rtm,
2475                                        Label& DONE_LABEL, Label& IsInflated) {
2476   assert(UseRTMForStackLocks, "why call this otherwise?");
2477   Label L_rtm_retry, L_decrement_retry, L_on_abort;
2478 
2479   if (RTMRetryCount > 0) {
2480     load_const_optimized(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
2481     bind(L_rtm_retry);
2482   }
2483   andi_(R0, mark_word, markWord::monitor_value);  // inflated vs stack-locked|neutral
2484   bne(CCR0, IsInflated);
2485 
2486   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
2487     Label L_noincrement;
2488     if (RTMTotalCountIncrRate > 1) {
2489       branch_on_random_using_tb(tmp, RTMTotalCountIncrRate, L_noincrement);
2490     }
2491     assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM");
2492     load_const_optimized(tmp, (address)stack_rtm_counters->total_count_addr(), R0);
2493     //atomic_inc_ptr(tmp, /*temp, will be reloaded*/mark_word); We don't increment atomically
2494     ldx(mark_word, tmp);
2495     addi(mark_word, mark_word, 1);
2496     stdx(mark_word, tmp);
2497     bind(L_noincrement);
2498   }
2499   tbegin_();
2500   beq(CCR0, L_on_abort);
2501   ld(mark_word, oopDesc::mark_offset_in_bytes(), obj);   // Reload in transaction, conflicts need to be tracked.
2502   andi(R0, mark_word, markWord::lock_mask_in_place);     // look at 2 lock bits
2503   cmpwi(flag, R0, markWord::unlocked_value);             // bits = 01 unlocked
2504   beq(flag, DONE_LABEL);                                 // all done if unlocked
2505 
2506   if (UseRTMXendForLockBusy) {
2507     tend_();
2508     b(L_decrement_retry);
2509   } else {
2510     tabort_();
2511   }
2512   bind(L_on_abort);
2513   const Register abort_status_Reg = tmp;
2514   mftexasr(abort_status_Reg);
2515   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
2516     rtm_profiling(abort_status_Reg, /*temp*/mark_word, stack_rtm_counters, method_data, profile_rtm);
2517   }
2518   ld(mark_word, oopDesc::mark_offset_in_bytes(), obj); // reload
2519   if (RTMRetryCount > 0) {
2520     // Retry on lock abort if abort status is not permanent.
2521     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry, &L_decrement_retry);
2522   } else {
2523     bind(L_decrement_retry);
2524   }
2525 }
2526 
2527 // Use RTM for inflating locks
2528 // inputs: obj       (object to lock)
2529 //         mark_word (current header - KILLED)
2530 //         boxReg    (on-stack box address (displaced header location) - KILLED)
2531 void MacroAssembler::rtm_inflated_locking(ConditionRegister flag,
2532                                           Register obj, Register mark_word, Register boxReg,
2533                                           Register retry_on_busy_count_Reg, Register retry_on_abort_count_Reg,
2534                                           RTMLockingCounters* rtm_counters,
2535                                           Metadata* method_data, bool profile_rtm,
2536                                           Label& DONE_LABEL) {
2537   assert(UseRTMLocking, "why call this otherwise?");
2538   Label L_rtm_retry, L_decrement_retry, L_on_abort;
2539   // Clean monitor_value bit to get valid pointer.
2540   int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markWord::monitor_value;
2541 
2542   // Store non-null, using boxReg instead of (intptr_t)markWord::unused_mark().
2543   std(boxReg, BasicLock::displaced_header_offset_in_bytes(), boxReg);
2544   const Register tmpReg = boxReg;
2545   const Register owner_addr_Reg = mark_word;
2546   addi(owner_addr_Reg, mark_word, owner_offset);
2547 
2548   if (RTMRetryCount > 0) {
2549     load_const_optimized(retry_on_busy_count_Reg, RTMRetryCount);  // Retry on lock busy.
2550     load_const_optimized(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort.
2551     bind(L_rtm_retry);
2552   }
2553   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
2554     Label L_noincrement;
2555     if (RTMTotalCountIncrRate > 1) {
2556       branch_on_random_using_tb(R0, RTMTotalCountIncrRate, L_noincrement);
2557     }
2558     assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
2559     load_const(R0, (address)rtm_counters->total_count_addr(), tmpReg);
2560     //atomic_inc_ptr(R0, tmpReg); We don't increment atomically
2561     ldx(tmpReg, R0);
2562     addi(tmpReg, tmpReg, 1);
2563     stdx(tmpReg, R0);
2564     bind(L_noincrement);
2565   }
2566   tbegin_();
2567   beq(CCR0, L_on_abort);
2568   // We don't reload mark word. Will only be reset at safepoint.
2569   ld(R0, 0, owner_addr_Reg); // Load in transaction, conflicts need to be tracked.
2570   cmpdi(flag, R0, 0);
2571   beq(flag, DONE_LABEL);
2572 
2573   if (UseRTMXendForLockBusy) {
2574     tend_();
2575     b(L_decrement_retry);
2576   } else {
2577     tabort_();
2578   }
2579   bind(L_on_abort);
2580   const Register abort_status_Reg = tmpReg;
2581   mftexasr(abort_status_Reg);
2582   if (PrintPreciseRTMLockingStatistics || profile_rtm) {
2583     rtm_profiling(abort_status_Reg, /*temp*/ owner_addr_Reg, rtm_counters, method_data, profile_rtm);
2584     // Restore owner_addr_Reg
2585     ld(mark_word, oopDesc::mark_offset_in_bytes(), obj);
2586 #ifdef ASSERT
2587     andi_(R0, mark_word, markWord::monitor_value);
2588     asm_assert_ne("must be inflated"); // Deflating only allowed at safepoint.
2589 #endif
2590     addi(owner_addr_Reg, mark_word, owner_offset);
2591   }
2592   if (RTMRetryCount > 0) {
2593     // Retry on lock abort if abort status is not permanent.
2594     rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
2595   }
2596 
2597   // Appears unlocked - try to swing _owner from null to non-null.
2598   cmpxchgd(flag, /*current val*/ R0, (intptr_t)0, /*new val*/ R16_thread, owner_addr_Reg,
2599            MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2600            MacroAssembler::cmpxchgx_hint_acquire_lock(), noreg, &L_decrement_retry, true);
2601 
2602   if (RTMRetryCount > 0) {
2603     // success done else retry
2604     b(DONE_LABEL);
2605     bind(L_decrement_retry);
2606     // Spin and retry if lock is busy.
2607     rtm_retry_lock_on_busy(retry_on_busy_count_Reg, owner_addr_Reg, L_rtm_retry);
2608   } else {
2609     bind(L_decrement_retry);
2610   }
2611 }
2612 
2613 #endif //  INCLUDE_RTM_OPT
2614 
2615 // "The box" is the space on the stack where we copy the object mark.
2616 void MacroAssembler::compiler_fast_lock_object(ConditionRegister flag, Register oop, Register box,
2617                                                Register temp, Register displaced_header, Register current_header,
2618                                                RTMLockingCounters* rtm_counters,
2619                                                RTMLockingCounters* stack_rtm_counters,
2620                                                Metadata* method_data,
2621                                                bool use_rtm, bool profile_rtm) {
2622   assert_different_registers(oop, box, temp, displaced_header, current_header);
2623   assert(flag != CCR0, "bad condition register");
2624   Label cont;
2625   Label object_has_monitor;
2626   Label cas_failed;
2627 
2628   // Load markWord from object into displaced_header.
2629   ld(displaced_header, oopDesc::mark_offset_in_bytes(), oop);
2630 
2631   if (DiagnoseSyncOnValueBasedClasses != 0) {
2632     load_klass(temp, oop);
2633     lwz(temp, in_bytes(Klass::access_flags_offset()), temp);
2634     testbitdi(flag, R0, temp, exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS));
2635     bne(flag, cont);
2636   }
2637 
2638 #if INCLUDE_RTM_OPT
2639   if (UseRTMForStackLocks && use_rtm) {
2640     rtm_stack_locking(flag, oop, displaced_header, temp, /*temp*/ current_header,
2641                       stack_rtm_counters, method_data, profile_rtm,
2642                       cont, object_has_monitor);
2643   }
2644 #endif // INCLUDE_RTM_OPT
2645 
2646   // Handle existing monitor.
2647   // The object has an existing monitor iff (mark & monitor_value) != 0.
2648   andi_(temp, displaced_header, markWord::monitor_value);
2649   bne(CCR0, object_has_monitor);
2650 
2651   if (!UseHeavyMonitors) {
2652     // Set displaced_header to be (markWord of object | UNLOCK_VALUE).
2653     ori(displaced_header, displaced_header, markWord::unlocked_value);
2654 
2655     // Load Compare Value application register.
2656 
2657     // Initialize the box. (Must happen before we update the object mark!)
2658     std(displaced_header, BasicLock::displaced_header_offset_in_bytes(), box);
2659 
2660     // Must fence, otherwise, preceding store(s) may float below cmpxchg.
2661     // Compare object markWord with mark and if equal exchange scratch1 with object markWord.
2662     cmpxchgd(/*flag=*/flag,
2663              /*current_value=*/current_header,
2664              /*compare_value=*/displaced_header,
2665              /*exchange_value=*/box,
2666              /*where=*/oop,
2667              MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2668              MacroAssembler::cmpxchgx_hint_acquire_lock(),
2669              noreg,
2670              &cas_failed,
2671              /*check without membar and ldarx first*/true);
2672     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2673   } else {
2674     // Set NE to indicate 'failure' -> take slow-path.
2675     crandc(flag, Assembler::equal, flag, Assembler::equal);
2676   }
2677 
2678   // If the compare-and-exchange succeeded, then we found an unlocked
2679   // object and we have now locked it.
2680   b(cont);
2681 
2682   bind(cas_failed);
2683   // We did not see an unlocked object so try the fast recursive case.
2684 
2685   // Check if the owner is self by comparing the value in the markWord of object
2686   // (current_header) with the stack pointer.
2687   sub(current_header, current_header, R1_SP);
2688   load_const_optimized(temp, ~(os::vm_page_size()-1) | markWord::lock_mask_in_place);
2689 
2690   and_(R0/*==0?*/, current_header, temp);
2691   // If condition is true we are cont and hence we can store 0 as the
2692   // displaced header in the box, which indicates that it is a recursive lock.
2693   mcrf(flag,CCR0);
2694   std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), box);
2695 
2696   // Handle existing monitor.
2697   b(cont);
2698 
2699   bind(object_has_monitor);
2700   // The object's monitor m is unlocked iff m->owner == NULL,
2701   // otherwise m->owner may contain a thread or a stack address.
2702 
2703 #if INCLUDE_RTM_OPT
2704   // Use the same RTM locking code in 32- and 64-bit VM.
2705   if (use_rtm) {
2706     rtm_inflated_locking(flag, oop, displaced_header, box, temp, /*temp*/ current_header,
2707                          rtm_counters, method_data, profile_rtm, cont);
2708   } else {
2709 #endif // INCLUDE_RTM_OPT
2710 
2711   // Try to CAS m->owner from NULL to current thread.
2712   addi(temp, displaced_header, ObjectMonitor::owner_offset_in_bytes()-markWord::monitor_value);
2713   cmpxchgd(/*flag=*/flag,
2714            /*current_value=*/current_header,
2715            /*compare_value=*/(intptr_t)0,
2716            /*exchange_value=*/R16_thread,
2717            /*where=*/temp,
2718            MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2719            MacroAssembler::cmpxchgx_hint_acquire_lock());
2720 
2721   // Store a non-null value into the box.
2722   std(box, BasicLock::displaced_header_offset_in_bytes(), box);
2723   beq(flag, cont);
2724 
2725   // Check for recursive locking.
2726   cmpd(flag, current_header, R16_thread);
2727   bne(flag, cont);
2728 
2729   // Current thread already owns the lock. Just increment recursions.
2730   Register recursions = displaced_header;
2731   ld(recursions, ObjectMonitor::recursions_offset_in_bytes()-ObjectMonitor::owner_offset_in_bytes(), temp);
2732   addi(recursions, recursions, 1);
2733   std(recursions, ObjectMonitor::recursions_offset_in_bytes()-ObjectMonitor::owner_offset_in_bytes(), temp);
2734 
2735 #if INCLUDE_RTM_OPT
2736   } // use_rtm()
2737 #endif
2738 
2739   bind(cont);
2740   // flag == EQ indicates success
2741   // flag == NE indicates failure
2742 }
2743 
2744 void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
2745                                                  Register temp, Register displaced_header, Register current_header,
2746                                                  bool use_rtm) {
2747   assert_different_registers(oop, box, temp, displaced_header, current_header);
2748   assert(flag != CCR0, "bad condition register");
2749   Label cont, object_has_monitor, notRecursive;
2750 
2751 #if INCLUDE_RTM_OPT
2752   if (UseRTMForStackLocks && use_rtm) {
2753     Label L_regular_unlock;
2754     ld(current_header, oopDesc::mark_offset_in_bytes(), oop);   // fetch markword
2755     andi(R0, current_header, markWord::lock_mask_in_place);     // look at 2 lock bits
2756     cmpwi(flag, R0, markWord::unlocked_value);                  // bits = 01 unlocked
2757     bne(flag, L_regular_unlock);                                // else RegularLock
2758     tend_();                                                    // otherwise end...
2759     b(cont);                                                    // ... and we're done
2760     bind(L_regular_unlock);
2761   }
2762 #endif
2763 
2764   if (!UseHeavyMonitors) {
2765     // Find the lock address and load the displaced header from the stack.
2766     ld(displaced_header, BasicLock::displaced_header_offset_in_bytes(), box);
2767 
2768     // If the displaced header is 0, we have a recursive unlock.
2769     cmpdi(flag, displaced_header, 0);
2770     beq(flag, cont);
2771   }
2772 
2773   // Handle existing monitor.
2774   // The object has an existing monitor iff (mark & monitor_value) != 0.
2775   RTM_OPT_ONLY( if (!(UseRTMForStackLocks && use_rtm)) ) // skip load if already done
2776   ld(current_header, oopDesc::mark_offset_in_bytes(), oop);
2777   andi_(R0, current_header, markWord::monitor_value);
2778   bne(CCR0, object_has_monitor);
2779 
2780   if (!UseHeavyMonitors) {
2781     // Check if it is still a light weight lock, this is is true if we see
2782     // the stack address of the basicLock in the markWord of the object.
2783     // Cmpxchg sets flag to cmpd(current_header, box).
2784     cmpxchgd(/*flag=*/flag,
2785              /*current_value=*/current_header,
2786              /*compare_value=*/box,
2787              /*exchange_value=*/displaced_header,
2788              /*where=*/oop,
2789              MacroAssembler::MemBarRel,
2790              MacroAssembler::cmpxchgx_hint_release_lock(),
2791              noreg,
2792              &cont);
2793     assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2794   } else {
2795     // Set NE to indicate 'failure' -> take slow-path.
2796     crandc(flag, Assembler::equal, flag, Assembler::equal);
2797   }
2798 
2799   // Handle existing monitor.
2800   b(cont);
2801 
2802   bind(object_has_monitor);
2803   STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
2804   addi(current_header, current_header, -(int)markWord::monitor_value); // monitor
2805   ld(temp,             ObjectMonitor::owner_offset_in_bytes(), current_header);
2806 
2807     // It's inflated.
2808 #if INCLUDE_RTM_OPT
2809   if (use_rtm) {
2810     Label L_regular_inflated_unlock;
2811     // Clean monitor_value bit to get valid pointer
2812     cmpdi(flag, temp, 0);
2813     bne(flag, L_regular_inflated_unlock);
2814     tend_();
2815     b(cont);
2816     bind(L_regular_inflated_unlock);
2817   }
2818 #endif
2819 
2820   ld(displaced_header, ObjectMonitor::recursions_offset_in_bytes(), current_header);
2821 
2822   cmpd(flag, temp, R16_thread);
2823   bne(flag, cont);
2824 
2825   addic_(displaced_header, displaced_header, -1);
2826   blt(CCR0, notRecursive); // Not recursive if negative after decrement.
2827   std(displaced_header, ObjectMonitor::recursions_offset_in_bytes(), current_header);
2828   b(cont); // flag is already EQ here.
2829 
2830   bind(notRecursive);
2831   ld(temp,             ObjectMonitor::EntryList_offset_in_bytes(), current_header);
2832   ld(displaced_header, ObjectMonitor::cxq_offset_in_bytes(), current_header);
2833   orr(temp, temp, displaced_header); // Will be 0 if both are 0.
2834   cmpdi(flag, temp, 0);
2835   bne(flag, cont);
2836   release();
2837   std(temp, ObjectMonitor::owner_offset_in_bytes(), current_header);
2838 
2839   bind(cont);
2840   // flag == EQ indicates success
2841   // flag == NE indicates failure
2842 }
2843 
2844 void MacroAssembler::safepoint_poll(Label& slow_path, Register temp, bool at_return, bool in_nmethod) {
2845   ld(temp, in_bytes(JavaThread::polling_word_offset()), R16_thread);
2846 
2847   if (at_return) {
2848     if (in_nmethod) {
2849       if (UseSIGTRAP) {
2850         // Use Signal Handler.
2851         relocate(relocInfo::poll_return_type);
2852         td(traptoGreaterThanUnsigned, R1_SP, temp);
2853       } else {
2854         cmpld(CCR0, R1_SP, temp);
2855         // Stub may be out of range for short conditional branch.
2856         bc_far_optimized(Assembler::bcondCRbiIs1, bi0(CCR0, Assembler::greater), slow_path);
2857       }
2858     } else { // Not in nmethod.
2859       // Frame still on stack, need to get fp.
2860       Register fp = R0;
2861       ld(fp, _abi0(callers_sp), R1_SP);
2862       cmpld(CCR0, fp, temp);
2863       bgt(CCR0, slow_path);
2864     }
2865   } else { // Normal safepoint poll. Not at return.
2866     assert(!in_nmethod, "should use load_from_polling_page");
2867     andi_(temp, temp, SafepointMechanism::poll_bit());
2868     bne(CCR0, slow_path);
2869   }
2870 }
2871 
2872 void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp2,
2873                                      MacroAssembler::PreservationLevel preservation_level) {
2874   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2875   bs->resolve_jobject(this, value, tmp1, tmp2, preservation_level);
2876 }
2877 
2878 // Values for last_Java_pc, and last_Java_sp must comply to the rules
2879 // in frame_ppc.hpp.
2880 void MacroAssembler::set_last_Java_frame(Register last_Java_sp, Register last_Java_pc) {
2881   // Always set last_Java_pc and flags first because once last_Java_sp
2882   // is visible has_last_Java_frame is true and users will look at the
2883   // rest of the fields. (Note: flags should always be zero before we
2884   // get here so doesn't need to be set.)
2885 
2886   // Verify that last_Java_pc was zeroed on return to Java
2887   asm_assert_mem8_is_zero(in_bytes(JavaThread::last_Java_pc_offset()), R16_thread,
2888                           "last_Java_pc not zeroed before leaving Java");
2889 
2890   // When returning from calling out from Java mode the frame anchor's
2891   // last_Java_pc will always be set to NULL. It is set here so that
2892   // if we are doing a call to native (not VM) that we capture the
2893   // known pc and don't have to rely on the native call having a
2894   // standard frame linkage where we can find the pc.
2895   if (last_Java_pc != noreg)
2896     std(last_Java_pc, in_bytes(JavaThread::last_Java_pc_offset()), R16_thread);
2897 
2898   // Set last_Java_sp last.
2899   std(last_Java_sp, in_bytes(JavaThread::last_Java_sp_offset()), R16_thread);
2900 }
2901 
2902 void MacroAssembler::reset_last_Java_frame(void) {
2903   asm_assert_mem8_isnot_zero(in_bytes(JavaThread::last_Java_sp_offset()),
2904                              R16_thread, "SP was not set, still zero");
2905 
2906   BLOCK_COMMENT("reset_last_Java_frame {");
2907   li(R0, 0);
2908 
2909   // _last_Java_sp = 0
2910   std(R0, in_bytes(JavaThread::last_Java_sp_offset()), R16_thread);
2911 
2912   // _last_Java_pc = 0
2913   std(R0, in_bytes(JavaThread::last_Java_pc_offset()), R16_thread);
2914   BLOCK_COMMENT("} reset_last_Java_frame");
2915 }
2916 
2917 void MacroAssembler::set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1) {
2918   assert_different_registers(sp, tmp1);
2919 
2920   // sp points to a TOP_IJAVA_FRAME, retrieve frame's PC via
2921   // TOP_IJAVA_FRAME_ABI.
2922   // FIXME: assert that we really have a TOP_IJAVA_FRAME here!
2923   address entry = pc();
2924   load_const_optimized(tmp1, entry);
2925 
2926   set_last_Java_frame(/*sp=*/sp, /*pc=*/tmp1);
2927 }
2928 
2929 void MacroAssembler::get_vm_result(Register oop_result) {
2930   // Read:
2931   //   R16_thread
2932   //   R16_thread->in_bytes(JavaThread::vm_result_offset())
2933   //
2934   // Updated:
2935   //   oop_result
2936   //   R16_thread->in_bytes(JavaThread::vm_result_offset())
2937 
2938   verify_thread();
2939 
2940   ld(oop_result, in_bytes(JavaThread::vm_result_offset()), R16_thread);
2941   li(R0, 0);
2942   std(R0, in_bytes(JavaThread::vm_result_offset()), R16_thread);
2943 
2944   verify_oop(oop_result, FILE_AND_LINE);
2945 }
2946 
2947 void MacroAssembler::get_vm_result_2(Register metadata_result) {
2948   // Read:
2949   //   R16_thread
2950   //   R16_thread->in_bytes(JavaThread::vm_result_2_offset())
2951   //
2952   // Updated:
2953   //   metadata_result
2954   //   R16_thread->in_bytes(JavaThread::vm_result_2_offset())
2955 
2956   ld(metadata_result, in_bytes(JavaThread::vm_result_2_offset()), R16_thread);
2957   li(R0, 0);
2958   std(R0, in_bytes(JavaThread::vm_result_2_offset()), R16_thread);
2959 }
2960 
2961 Register MacroAssembler::encode_klass_not_null(Register dst, Register src) {
2962   Register current = (src != noreg) ? src : dst; // Klass is in dst if no src provided.
2963   if (CompressedKlassPointers::base() != 0) {
2964     // Use dst as temp if it is free.
2965     sub_const_optimized(dst, current, CompressedKlassPointers::base(), R0);
2966     current = dst;
2967   }
2968   if (CompressedKlassPointers::shift() != 0) {
2969     srdi(dst, current, CompressedKlassPointers::shift());
2970     current = dst;
2971   }
2972   return current;
2973 }
2974 
2975 void MacroAssembler::store_klass(Register dst_oop, Register klass, Register ck) {
2976   if (UseCompressedClassPointers) {
2977     Register compressedKlass = encode_klass_not_null(ck, klass);
2978     stw(compressedKlass, oopDesc::klass_offset_in_bytes(), dst_oop);
2979   } else {
2980     std(klass, oopDesc::klass_offset_in_bytes(), dst_oop);
2981   }
2982 }
2983 
2984 void MacroAssembler::store_klass_gap(Register dst_oop, Register val) {
2985   if (UseCompressedClassPointers) {
2986     if (val == noreg) {
2987       val = R0;
2988       li(val, 0);
2989     }
2990     stw(val, oopDesc::klass_gap_offset_in_bytes(), dst_oop); // klass gap if compressed
2991   }
2992 }
2993 
2994 int MacroAssembler::instr_size_for_decode_klass_not_null() {
2995   static int computed_size = -1;
2996 
2997   // Not yet computed?
2998   if (computed_size == -1) {
2999 
3000     if (!UseCompressedClassPointers) {
3001       computed_size = 0;
3002     } else {
3003       // Determine by scratch emit.
3004       ResourceMark rm;
3005       int code_size = 8 * BytesPerInstWord;
3006       CodeBuffer cb("decode_klass_not_null scratch buffer", code_size, 0);
3007       MacroAssembler* a = new MacroAssembler(&cb);
3008       a->decode_klass_not_null(R11_scratch1);
3009       computed_size = a->offset();
3010     }
3011   }
3012 
3013   return computed_size;
3014 }
3015 
3016 void MacroAssembler::decode_klass_not_null(Register dst, Register src) {
3017   assert(dst != R0, "Dst reg may not be R0, as R0 is used here.");
3018   if (src == noreg) src = dst;
3019   Register shifted_src = src;
3020   if (CompressedKlassPointers::shift() != 0 ||
3021       CompressedKlassPointers::base() == 0 && src != dst) {  // Move required.
3022     shifted_src = dst;
3023     sldi(shifted_src, src, CompressedKlassPointers::shift());
3024   }
3025   if (CompressedKlassPointers::base() != 0) {
3026     add_const_optimized(dst, shifted_src, CompressedKlassPointers::base(), R0);
3027   }
3028 }
3029 
3030 void MacroAssembler::load_klass(Register dst, Register src) {
3031   if (UseCompressedClassPointers) {
3032     lwz(dst, oopDesc::klass_offset_in_bytes(), src);
3033     // Attention: no null check here!
3034     decode_klass_not_null(dst, dst);
3035   } else {
3036     ld(dst, oopDesc::klass_offset_in_bytes(), src);
3037   }
3038 }
3039 
3040 // ((OopHandle)result).resolve();
3041 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2,
3042                                         MacroAssembler::PreservationLevel preservation_level) {
3043   access_load_at(T_OBJECT, IN_NATIVE, result, noreg, result, tmp1, tmp2, preservation_level);
3044 }
3045 
3046 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2,
3047                                          MacroAssembler::PreservationLevel preservation_level) {
3048   Label resolved;
3049 
3050   // A null weak handle resolves to null.
3051   cmpdi(CCR0, result, 0);
3052   beq(CCR0, resolved);
3053 
3054   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, result, noreg, result, tmp1, tmp2,
3055                  preservation_level);
3056   bind(resolved);
3057 }
3058 
3059 void MacroAssembler::load_method_holder(Register holder, Register method) {
3060   ld(holder, in_bytes(Method::const_offset()), method);
3061   ld(holder, in_bytes(ConstMethod::constants_offset()), holder);
3062   ld(holder, ConstantPool::pool_holder_offset_in_bytes(), holder);
3063 }
3064 
3065 // Clear Array
3066 // For very short arrays. tmp == R0 is allowed.
3067 void MacroAssembler::clear_memory_unrolled(Register base_ptr, int cnt_dwords, Register tmp, int offset) {
3068   if (cnt_dwords > 0) { li(tmp, 0); }
3069   for (int i = 0; i < cnt_dwords; ++i) { std(tmp, offset + i * 8, base_ptr); }
3070 }
3071 
3072 // Version for constant short array length. Kills base_ptr. tmp == R0 is allowed.
3073 void MacroAssembler::clear_memory_constlen(Register base_ptr, int cnt_dwords, Register tmp) {
3074   if (cnt_dwords < 8) {
3075     clear_memory_unrolled(base_ptr, cnt_dwords, tmp);
3076     return;
3077   }
3078 
3079   Label loop;
3080   const long loopcnt   = cnt_dwords >> 1,
3081              remainder = cnt_dwords & 1;
3082 
3083   li(tmp, loopcnt);
3084   mtctr(tmp);
3085   li(tmp, 0);
3086   bind(loop);
3087     std(tmp, 0, base_ptr);
3088     std(tmp, 8, base_ptr);
3089     addi(base_ptr, base_ptr, 16);
3090     bdnz(loop);
3091   if (remainder) { std(tmp, 0, base_ptr); }
3092 }
3093 
3094 // Kills both input registers. tmp == R0 is allowed.
3095 void MacroAssembler::clear_memory_doubleword(Register base_ptr, Register cnt_dwords, Register tmp, long const_cnt) {
3096   // Procedure for large arrays (uses data cache block zero instruction).
3097     Label startloop, fast, fastloop, small_rest, restloop, done;
3098     const int cl_size         = VM_Version::L1_data_cache_line_size(),
3099               cl_dwords       = cl_size >> 3,
3100               cl_dw_addr_bits = exact_log2(cl_dwords),
3101               dcbz_min        = 1,  // Min count of dcbz executions, needs to be >0.
3102               min_cnt         = ((dcbz_min + 1) << cl_dw_addr_bits) - 1;
3103 
3104   if (const_cnt >= 0) {
3105     // Constant case.
3106     if (const_cnt < min_cnt) {
3107       clear_memory_constlen(base_ptr, const_cnt, tmp);
3108       return;
3109     }
3110     load_const_optimized(cnt_dwords, const_cnt, tmp);
3111   } else {
3112     // cnt_dwords already loaded in register. Need to check size.
3113     cmpdi(CCR1, cnt_dwords, min_cnt); // Big enough? (ensure >= dcbz_min lines included).
3114     blt(CCR1, small_rest);
3115   }
3116     rldicl_(tmp, base_ptr, 64-3, 64-cl_dw_addr_bits); // Extract dword offset within first cache line.
3117     beq(CCR0, fast);                                  // Already 128byte aligned.
3118 
3119     subfic(tmp, tmp, cl_dwords);
3120     mtctr(tmp);                        // Set ctr to hit 128byte boundary (0<ctr<cl_dwords).
3121     subf(cnt_dwords, tmp, cnt_dwords); // rest.
3122     li(tmp, 0);
3123 
3124   bind(startloop);                     // Clear at the beginning to reach 128byte boundary.
3125     std(tmp, 0, base_ptr);             // Clear 8byte aligned block.
3126     addi(base_ptr, base_ptr, 8);
3127     bdnz(startloop);
3128 
3129   bind(fast);                                  // Clear 128byte blocks.
3130     srdi(tmp, cnt_dwords, cl_dw_addr_bits);    // Loop count for 128byte loop (>0).
3131     andi(cnt_dwords, cnt_dwords, cl_dwords-1); // Rest in dwords.
3132     mtctr(tmp);                                // Load counter.
3133 
3134   bind(fastloop);
3135     dcbz(base_ptr);                    // Clear 128byte aligned block.
3136     addi(base_ptr, base_ptr, cl_size);
3137     bdnz(fastloop);
3138 
3139   bind(small_rest);
3140     cmpdi(CCR0, cnt_dwords, 0);        // size 0?
3141     beq(CCR0, done);                   // rest == 0
3142     li(tmp, 0);
3143     mtctr(cnt_dwords);                 // Load counter.
3144 
3145   bind(restloop);                      // Clear rest.
3146     std(tmp, 0, base_ptr);             // Clear 8byte aligned block.
3147     addi(base_ptr, base_ptr, 8);
3148     bdnz(restloop);
3149 
3150   bind(done);
3151 }
3152 
3153 /////////////////////////////////////////// String intrinsics ////////////////////////////////////////////
3154 
3155 // Helpers for Intrinsic Emitters
3156 //
3157 // Revert the byte order of a 32bit value in a register
3158 //   src: 0x44556677
3159 //   dst: 0x77665544
3160 // Three steps to obtain the result:
3161 //  1) Rotate src (as doubleword) left 5 bytes. That puts the leftmost byte of the src word
3162 //     into the rightmost byte position. Afterwards, everything left of the rightmost byte is cleared.
3163 //     This value initializes dst.
3164 //  2) Rotate src (as word) left 3 bytes. That puts the rightmost byte of the src word into the leftmost
3165 //     byte position. Furthermore, byte 5 is rotated into byte 6 position where it is supposed to go.
3166 //     This value is mask inserted into dst with a [0..23] mask of 1s.
3167 //  3) Rotate src (as word) left 1 byte. That puts byte 6 into byte 5 position.
3168 //     This value is mask inserted into dst with a [8..15] mask of 1s.
3169 void MacroAssembler::load_reverse_32(Register dst, Register src) {
3170   assert_different_registers(dst, src);
3171 
3172   rldicl(dst, src, (4+1)*8, 56);       // Rotate byte 4 into position 7 (rightmost), clear all to the left.
3173   rlwimi(dst, src,     3*8,  0, 23);   // Insert byte 5 into position 6, 7 into 4, leave pos 7 alone.
3174   rlwimi(dst, src,     1*8,  8, 15);   // Insert byte 6 into position 5, leave the rest alone.
3175 }
3176 
3177 // Calculate the column addresses of the crc32 lookup table into distinct registers.
3178 // This loop-invariant calculation is moved out of the loop body, reducing the loop
3179 // body size from 20 to 16 instructions.
3180 // Returns the offset that was used to calculate the address of column tc3.
3181 // Due to register shortage, setting tc3 may overwrite table. With the return offset
3182 // at hand, the original table address can be easily reconstructed.
3183 int MacroAssembler::crc32_table_columns(Register table, Register tc0, Register tc1, Register tc2, Register tc3) {
3184   assert(!VM_Version::has_vpmsumb(), "Vector version should be used instead!");
3185 
3186   // Point to 4 byte folding tables (byte-reversed version for Big Endian)
3187   // Layout: See StubRoutines::ppc::generate_crc_constants.
3188 #ifdef VM_LITTLE_ENDIAN
3189   const int ix0 = 3 * CRC32_TABLE_SIZE;
3190   const int ix1 = 2 * CRC32_TABLE_SIZE;
3191   const int ix2 = 1 * CRC32_TABLE_SIZE;
3192   const int ix3 = 0 * CRC32_TABLE_SIZE;
3193 #else
3194   const int ix0 = 1 * CRC32_TABLE_SIZE;
3195   const int ix1 = 2 * CRC32_TABLE_SIZE;
3196   const int ix2 = 3 * CRC32_TABLE_SIZE;
3197   const int ix3 = 4 * CRC32_TABLE_SIZE;
3198 #endif
3199   assert_different_registers(table, tc0, tc1, tc2);
3200   assert(table == tc3, "must be!");
3201 
3202   addi(tc0, table, ix0);
3203   addi(tc1, table, ix1);
3204   addi(tc2, table, ix2);
3205   if (ix3 != 0) addi(tc3, table, ix3);
3206 
3207   return ix3;
3208 }
3209 
3210 /**
3211  * uint32_t crc;
3212  * table[crc & 0xFF] ^ (crc >> 8);
3213  */
3214 void MacroAssembler::fold_byte_crc32(Register crc, Register val, Register table, Register tmp) {
3215   assert_different_registers(crc, table, tmp);
3216   assert_different_registers(val, table);
3217 
3218   if (crc == val) {                   // Must rotate first to use the unmodified value.
3219     rlwinm(tmp, val, 2, 24-2, 31-2);  // Insert (rightmost) byte 7 of val, shifted left by 2, into byte 6..7 of tmp, clear the rest.
3220                                       // As we use a word (4-byte) instruction, we have to adapt the mask bit positions.
3221     srwi(crc, crc, 8);                // Unsigned shift, clear leftmost 8 bits.
3222   } else {
3223     srwi(crc, crc, 8);                // Unsigned shift, clear leftmost 8 bits.
3224     rlwinm(tmp, val, 2, 24-2, 31-2);  // Insert (rightmost) byte 7 of val, shifted left by 2, into byte 6..7 of tmp, clear the rest.
3225   }
3226   lwzx(tmp, table, tmp);
3227   xorr(crc, crc, tmp);
3228 }
3229 
3230 /**
3231  * Emits code to update CRC-32 with a byte value according to constants in table.
3232  *
3233  * @param [in,out]crc   Register containing the crc.
3234  * @param [in]val       Register containing the byte to fold into the CRC.
3235  * @param [in]table     Register containing the table of crc constants.
3236  *
3237  * uint32_t crc;
3238  * val = crc_table[(val ^ crc) & 0xFF];
3239  * crc = val ^ (crc >> 8);
3240  */
3241 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) {
3242   BLOCK_COMMENT("update_byte_crc32:");
3243   xorr(val, val, crc);
3244   fold_byte_crc32(crc, val, table, val);
3245 }
3246 
3247 /**
3248  * @param crc   register containing existing CRC (32-bit)
3249  * @param buf   register pointing to input byte buffer (byte*)
3250  * @param len   register containing number of bytes
3251  * @param table register pointing to CRC table
3252  */
3253 void MacroAssembler::update_byteLoop_crc32(Register crc, Register buf, Register len, Register table,
3254                                            Register data, bool loopAlignment) {
3255   assert_different_registers(crc, buf, len, table, data);
3256 
3257   Label L_mainLoop, L_done;
3258   const int mainLoop_stepping  = 1;
3259   const int mainLoop_alignment = loopAlignment ? 32 : 4; // (InputForNewCode > 4 ? InputForNewCode : 32) : 4;
3260 
3261   // Process all bytes in a single-byte loop.
3262   clrldi_(len, len, 32);                         // Enforce 32 bit. Anything to do?
3263   beq(CCR0, L_done);
3264 
3265   mtctr(len);
3266   align(mainLoop_alignment);
3267   BIND(L_mainLoop);
3268     lbz(data, 0, buf);                           // Byte from buffer, zero-extended.
3269     addi(buf, buf, mainLoop_stepping);           // Advance buffer position.
3270     update_byte_crc32(crc, data, table);
3271     bdnz(L_mainLoop);                            // Iterate.
3272 
3273   bind(L_done);
3274 }
3275 
3276 /**
3277  * Emits code to update CRC-32 with a 4-byte value according to constants in table
3278  * Implementation according to jdk/src/share/native/java/util/zip/zlib-1.2.8/crc32.c
3279  */
3280 // A note on the lookup table address(es):
3281 // The implementation uses 4 table columns (byte-reversed versions for Big Endian).
3282 // To save the effort of adding the column offset to the table address each time
3283 // a table element is looked up, it is possible to pass the pre-calculated
3284 // column addresses.
3285 // Uses R9..R12 as work register. Must be saved/restored by caller, if necessary.
3286 void MacroAssembler::update_1word_crc32(Register crc, Register buf, Register table, int bufDisp, int bufInc,
3287                                         Register t0,  Register t1,  Register t2,  Register t3,
3288                                         Register tc0, Register tc1, Register tc2, Register tc3) {
3289   assert_different_registers(crc, t3);
3290 
3291   // XOR crc with next four bytes of buffer.
3292   lwz(t3, bufDisp, buf);
3293   if (bufInc != 0) {
3294     addi(buf, buf, bufInc);
3295   }
3296   xorr(t3, t3, crc);
3297 
3298   // Chop crc into 4 single-byte pieces, shifted left 2 bits, to form the table indices.
3299   rlwinm(t0, t3,  2,         24-2, 31-2);  // ((t1 >>  0) & 0xff) << 2
3300   rlwinm(t1, t3,  32+(2- 8), 24-2, 31-2);  // ((t1 >>  8) & 0xff) << 2
3301   rlwinm(t2, t3,  32+(2-16), 24-2, 31-2);  // ((t1 >> 16) & 0xff) << 2
3302   rlwinm(t3, t3,  32+(2-24), 24-2, 31-2);  // ((t1 >> 24) & 0xff) << 2
3303 
3304   // Use the pre-calculated column addresses.
3305   // Load pre-calculated table values.
3306   lwzx(t0, tc0, t0);
3307   lwzx(t1, tc1, t1);
3308   lwzx(t2, tc2, t2);
3309   lwzx(t3, tc3, t3);
3310 
3311   // Calculate new crc from table values.
3312   xorr(t0,  t0, t1);
3313   xorr(t2,  t2, t3);
3314   xorr(crc, t0, t2);  // Now crc contains the final checksum value.
3315 }
3316 
3317 /**
3318  * @param crc   register containing existing CRC (32-bit)
3319  * @param buf   register pointing to input byte buffer (byte*)
3320  * @param len   register containing number of bytes
3321  * @param table register pointing to CRC table
3322  *
3323  * uses R9..R12 as work register. Must be saved/restored by caller!
3324  */
3325 void MacroAssembler::kernel_crc32_1word(Register crc, Register buf, Register len, Register table,
3326                                         Register t0,  Register t1,  Register t2,  Register t3,
3327                                         Register tc0, Register tc1, Register tc2, Register tc3,
3328                                         bool invertCRC) {
3329   assert_different_registers(crc, buf, len, table);
3330 
3331   Label L_mainLoop, L_tail;
3332   Register  tmp          = t0;
3333   Register  data         = t0;
3334   Register  tmp2         = t1;
3335   const int mainLoop_stepping  = 4;
3336   const int tailLoop_stepping  = 1;
3337   const int log_stepping       = exact_log2(mainLoop_stepping);
3338   const int mainLoop_alignment = 32; // InputForNewCode > 4 ? InputForNewCode : 32;
3339   const int complexThreshold   = 2*mainLoop_stepping;
3340 
3341   // Don't test for len <= 0 here. This pathological case should not occur anyway.
3342   // Optimizing for it by adding a test and a branch seems to be a waste of CPU cycles
3343   // for all well-behaved cases. The situation itself is detected and handled correctly
3344   // within update_byteLoop_crc32.
3345   assert(tailLoop_stepping == 1, "check tailLoop_stepping!");
3346 
3347   BLOCK_COMMENT("kernel_crc32_1word {");
3348 
3349   if (invertCRC) {
3350     nand(crc, crc, crc);                      // 1s complement of crc
3351   }
3352 
3353   // Check for short (<mainLoop_stepping) buffer.
3354   cmpdi(CCR0, len, complexThreshold);
3355   blt(CCR0, L_tail);
3356 
3357   // Pre-mainLoop alignment did show a slight (1%) positive effect on performance.
3358   // We leave the code in for reference. Maybe we need alignment when we exploit vector instructions.
3359   {
3360     // Align buf addr to mainLoop_stepping boundary.
3361     neg(tmp2, buf);                              // Calculate # preLoop iterations for alignment.
3362     rldicl(tmp2, tmp2, 0, 64-log_stepping);      // Rotate tmp2 0 bits, insert into tmp2, anding with mask with 1s from 62..63.
3363 
3364     if (complexThreshold > mainLoop_stepping) {
3365       sub(len, len, tmp2);                       // Remaining bytes for main loop (>=mainLoop_stepping is guaranteed).
3366     } else {
3367       sub(tmp, len, tmp2);                       // Remaining bytes for main loop.
3368       cmpdi(CCR0, tmp, mainLoop_stepping);
3369       blt(CCR0, L_tail);                         // For less than one mainloop_stepping left, do only tail processing
3370       mr(len, tmp);                              // remaining bytes for main loop (>=mainLoop_stepping is guaranteed).
3371     }
3372     update_byteLoop_crc32(crc, buf, tmp2, table, data, false);
3373   }
3374 
3375   srdi(tmp2, len, log_stepping);                 // #iterations for mainLoop
3376   andi(len, len, mainLoop_stepping-1);           // remaining bytes for tailLoop
3377   mtctr(tmp2);
3378 
3379 #ifdef VM_LITTLE_ENDIAN
3380   Register crc_rv = crc;
3381 #else
3382   Register crc_rv = tmp;                         // Load_reverse needs separate registers to work on.
3383                                                  // Occupies tmp, but frees up crc.
3384   load_reverse_32(crc_rv, crc);                  // Revert byte order because we are dealing with big-endian data.
3385   tmp = crc;
3386 #endif
3387 
3388   int reconstructTableOffset = crc32_table_columns(table, tc0, tc1, tc2, tc3);
3389 
3390   align(mainLoop_alignment);                     // Octoword-aligned loop address. Shows 2% improvement.
3391   BIND(L_mainLoop);
3392     update_1word_crc32(crc_rv, buf, table, 0, mainLoop_stepping, crc_rv, t1, t2, t3, tc0, tc1, tc2, tc3);
3393     bdnz(L_mainLoop);
3394 
3395 #ifndef VM_LITTLE_ENDIAN
3396   load_reverse_32(crc, crc_rv);                  // Revert byte order because we are dealing with big-endian data.
3397   tmp = crc_rv;                                  // Tmp uses it's original register again.
3398 #endif
3399 
3400   // Restore original table address for tailLoop.
3401   if (reconstructTableOffset != 0) {
3402     addi(table, table, -reconstructTableOffset);
3403   }
3404 
3405   // Process last few (<complexThreshold) bytes of buffer.
3406   BIND(L_tail);
3407   update_byteLoop_crc32(crc, buf, len, table, data, false);
3408 
3409   if (invertCRC) {
3410     nand(crc, crc, crc);                      // 1s complement of crc
3411   }
3412   BLOCK_COMMENT("} kernel_crc32_1word");
3413 }
3414 
3415 /**
3416  * @param crc             register containing existing CRC (32-bit)
3417  * @param buf             register pointing to input byte buffer (byte*)
3418  * @param len             register containing number of bytes
3419  * @param constants       register pointing to precomputed constants
3420  * @param t0-t6           temp registers
3421  */
3422 void MacroAssembler::kernel_crc32_vpmsum(Register crc, Register buf, Register len, Register constants,
3423                                          Register t0, Register t1, Register t2, Register t3,
3424                                          Register t4, Register t5, Register t6, bool invertCRC) {
3425   assert_different_registers(crc, buf, len, constants);
3426 
3427   Label L_tail;
3428 
3429   BLOCK_COMMENT("kernel_crc32_vpmsum {");
3430 
3431   if (invertCRC) {
3432     nand(crc, crc, crc);                      // 1s complement of crc
3433   }
3434 
3435   // Enforce 32 bit.
3436   clrldi(len, len, 32);
3437 
3438   // Align if we have enough bytes for the fast version.
3439   const int alignment = 16,
3440             threshold = 32;
3441   Register prealign = t0;
3442 
3443   neg(prealign, buf);
3444   addi(t1, len, -threshold);
3445   andi(prealign, prealign, alignment - 1);
3446   cmpw(CCR0, t1, prealign);
3447   blt(CCR0, L_tail); // len - prealign < threshold?
3448 
3449   subf(len, prealign, len);
3450   update_byteLoop_crc32(crc, buf, prealign, constants, t2, false);
3451 
3452   // Calculate from first aligned address as far as possible.
3453   addi(constants, constants, CRC32_TABLE_SIZE); // Point to vector constants.
3454   kernel_crc32_vpmsum_aligned(crc, buf, len, constants, t0, t1, t2, t3, t4, t5, t6);
3455   addi(constants, constants, -CRC32_TABLE_SIZE); // Point to table again.
3456 
3457   // Remaining bytes.
3458   BIND(L_tail);
3459   update_byteLoop_crc32(crc, buf, len, constants, t2, false);
3460 
3461   if (invertCRC) {
3462     nand(crc, crc, crc);                      // 1s complement of crc
3463   }
3464 
3465   BLOCK_COMMENT("} kernel_crc32_vpmsum");
3466 }
3467 
3468 /**
3469  * @param crc             register containing existing CRC (32-bit)
3470  * @param buf             register pointing to input byte buffer (byte*)
3471  * @param len             register containing number of bytes (will get updated to remaining bytes)
3472  * @param constants       register pointing to CRC table for 128-bit aligned memory
3473  * @param t0-t6           temp registers
3474  */
3475 void MacroAssembler::kernel_crc32_vpmsum_aligned(Register crc, Register buf, Register len, Register constants,
3476     Register t0, Register t1, Register t2, Register t3, Register t4, Register t5, Register t6) {
3477 
3478   // Save non-volatile vector registers (frameless).
3479   Register offset = t1;
3480   int offsetInt = 0;
3481   offsetInt -= 16; li(offset, offsetInt); stvx(VR20, offset, R1_SP);
3482   offsetInt -= 16; li(offset, offsetInt); stvx(VR21, offset, R1_SP);
3483   offsetInt -= 16; li(offset, offsetInt); stvx(VR22, offset, R1_SP);
3484   offsetInt -= 16; li(offset, offsetInt); stvx(VR23, offset, R1_SP);
3485   offsetInt -= 16; li(offset, offsetInt); stvx(VR24, offset, R1_SP);
3486   offsetInt -= 16; li(offset, offsetInt); stvx(VR25, offset, R1_SP);
3487 #ifndef VM_LITTLE_ENDIAN
3488   offsetInt -= 16; li(offset, offsetInt); stvx(VR26, offset, R1_SP);
3489 #endif
3490   offsetInt -= 8; std(R14, offsetInt, R1_SP);
3491   offsetInt -= 8; std(R15, offsetInt, R1_SP);
3492 
3493   // Implementation uses an inner loop which uses between 256 and 16 * unroll_factor
3494   // bytes per iteration. The basic scheme is:
3495   // lvx: load vector (Big Endian needs reversal)
3496   // vpmsumw: carry-less 32 bit multiplications with constant representing a large CRC shift
3497   // vxor: xor partial results together to get unroll_factor2 vectors
3498 
3499   // Outer loop performs the CRC shifts needed to combine the unroll_factor2 vectors.
3500 
3501   // Using 16 * unroll_factor / unroll_factor_2 bytes for constants.
3502   const int unroll_factor = CRC32_UNROLL_FACTOR,
3503             unroll_factor2 = CRC32_UNROLL_FACTOR2;
3504 
3505   const int outer_consts_size = (unroll_factor2 - 1) * 16,
3506             inner_consts_size = (unroll_factor / unroll_factor2) * 16;
3507 
3508   // Support registers.
3509   Register offs[] = { noreg, t0, t1, t2, t3, t4, t5, t6 };
3510   Register num_bytes = R14,
3511            loop_count = R15,
3512            cur_const = crc; // will live in VCRC
3513   // Constant array for outer loop: unroll_factor2 - 1 registers,
3514   // Constant array for inner loop: unroll_factor / unroll_factor2 registers.
3515   VectorRegister consts0[] = { VR16, VR17, VR18, VR19, VR20, VR21, VR22 },
3516                  consts1[] = { VR23, VR24 };
3517   // Data register arrays: 2 arrays with unroll_factor2 registers.
3518   VectorRegister data0[] = { VR0, VR1, VR2, VR3, VR4, VR5, VR6, VR7 },
3519                  data1[] = { VR8, VR9, VR10, VR11, VR12, VR13, VR14, VR15 };
3520 
3521   VectorRegister VCRC = data0[0];
3522   VectorRegister Vc = VR25;
3523   VectorRegister swap_bytes = VR26; // Only for Big Endian.
3524 
3525   // We have at least 1 iteration (ensured by caller).
3526   Label L_outer_loop, L_inner_loop, L_last;
3527 
3528   // If supported set DSCR pre-fetch to deepest.
3529   if (VM_Version::has_mfdscr()) {
3530     load_const_optimized(t0, VM_Version::_dscr_val | 7);
3531     mtdscr(t0);
3532   }
3533 
3534   mtvrwz(VCRC, crc); // crc lives in VCRC, now
3535 
3536   for (int i = 1; i < unroll_factor2; ++i) {
3537     li(offs[i], 16 * i);
3538   }
3539 
3540   // Load consts for outer loop
3541   lvx(consts0[0], constants);
3542   for (int i = 1; i < unroll_factor2 - 1; ++i) {
3543     lvx(consts0[i], offs[i], constants);
3544   }
3545 
3546   load_const_optimized(num_bytes, 16 * unroll_factor);
3547 
3548   // Reuse data registers outside of the loop.
3549   VectorRegister Vtmp = data1[0];
3550   VectorRegister Vtmp2 = data1[1];
3551   VectorRegister zeroes = data1[2];
3552 
3553   vspltisb(Vtmp, 0);
3554   vsldoi(VCRC, Vtmp, VCRC, 8); // 96 bit zeroes, 32 bit CRC.
3555 
3556   // Load vector for vpermxor (to xor both 64 bit parts together)
3557   lvsl(Vtmp, buf);   // 000102030405060708090a0b0c0d0e0f
3558   vspltisb(Vc, 4);
3559   vsl(Vc, Vtmp, Vc); // 00102030405060708090a0b0c0d0e0f0
3560   xxspltd(Vc->to_vsr(), Vc->to_vsr(), 0);
3561   vor(Vc, Vtmp, Vc); // 001122334455667708192a3b4c5d6e7f
3562 
3563 #ifdef VM_LITTLE_ENDIAN
3564 #define BE_swap_bytes(x)
3565 #else
3566   vspltisb(Vtmp2, 0xf);
3567   vxor(swap_bytes, Vtmp, Vtmp2);
3568 #define BE_swap_bytes(x) vperm(x, x, x, swap_bytes)
3569 #endif
3570 
3571   cmpd(CCR0, len, num_bytes);
3572   blt(CCR0, L_last);
3573 
3574   addi(cur_const, constants, outer_consts_size); // Point to consts for inner loop
3575   load_const_optimized(loop_count, unroll_factor / (2 * unroll_factor2) - 1); // One double-iteration peeled off.
3576 
3577   // ********** Main loop start **********
3578   align(32);
3579   bind(L_outer_loop);
3580 
3581   // Begin of unrolled first iteration (no xor).
3582   lvx(data1[0], buf);
3583   for (int i = 1; i < unroll_factor2 / 2; ++i) {
3584     lvx(data1[i], offs[i], buf);
3585   }
3586   vpermxor(VCRC, VCRC, VCRC, Vc); // xor both halves to 64 bit result.
3587   lvx(consts1[0], cur_const);
3588   mtctr(loop_count);
3589   for (int i = 0; i < unroll_factor2 / 2; ++i) {
3590     BE_swap_bytes(data1[i]);
3591     if (i == 0) { vxor(data1[0], data1[0], VCRC); } // xor in previous CRC.
3592     lvx(data1[i + unroll_factor2 / 2], offs[i + unroll_factor2 / 2], buf);
3593     vpmsumw(data0[i], data1[i], consts1[0]);
3594   }
3595   addi(buf, buf, 16 * unroll_factor2);
3596   subf(len, num_bytes, len);
3597   lvx(consts1[1], offs[1], cur_const);
3598   addi(cur_const, cur_const, 32);
3599   // Begin of unrolled second iteration (head).
3600   for (int i = 0; i < unroll_factor2 / 2; ++i) {
3601     BE_swap_bytes(data1[i + unroll_factor2 / 2]);
3602     if (i == 0) { lvx(data1[0], buf); } else { lvx(data1[i], offs[i], buf); }
3603     vpmsumw(data0[i + unroll_factor2 / 2], data1[i + unroll_factor2 / 2], consts1[0]);
3604   }
3605   for (int i = 0; i < unroll_factor2 / 2; ++i) {
3606     BE_swap_bytes(data1[i]);
3607     lvx(data1[i + unroll_factor2 / 2], offs[i + unroll_factor2 / 2], buf);
3608     vpmsumw(data1[i], data1[i], consts1[1]);
3609   }
3610   addi(buf, buf, 16 * unroll_factor2);
3611 
3612   // Generate most performance relevant code. Loads + half of the vpmsumw have been generated.
3613   // Double-iteration allows using the 2 constant registers alternatingly.
3614   align(32);
3615   bind(L_inner_loop);
3616   for (int j = 1; j < 3; ++j) { // j < unroll_factor / unroll_factor2 - 1 for complete unrolling.
3617     if (j & 1) {
3618       lvx(consts1[0], cur_const);
3619     } else {
3620       lvx(consts1[1], offs[1], cur_const);
3621       addi(cur_const, cur_const, 32);
3622     }
3623     for (int i = 0; i < unroll_factor2; ++i) {
3624       int idx = i + unroll_factor2 / 2, inc = 0; // For modulo-scheduled input.
3625       if (idx >= unroll_factor2) { idx -= unroll_factor2; inc = 1; }
3626       BE_swap_bytes(data1[idx]);
3627       vxor(data0[i], data0[i], data1[i]);
3628       if (i == 0) lvx(data1[0], buf); else lvx(data1[i], offs[i], buf);
3629       vpmsumw(data1[idx], data1[idx], consts1[(j + inc) & 1]);
3630     }
3631     addi(buf, buf, 16 * unroll_factor2);
3632   }
3633   bdnz(L_inner_loop);
3634 
3635   addi(cur_const, constants, outer_consts_size); // Reset
3636 
3637   // Tail of last iteration (no loads).
3638   for (int i = 0; i < unroll_factor2 / 2; ++i) {
3639     BE_swap_bytes(data1[i + unroll_factor2 / 2]);
3640     vxor(data0[i], data0[i], data1[i]);
3641     vpmsumw(data1[i + unroll_factor2 / 2], data1[i + unroll_factor2 / 2], consts1[1]);
3642   }
3643   for (int i = 0; i < unroll_factor2 / 2; ++i) {
3644     vpmsumw(data0[i], data0[i], consts0[unroll_factor2 - 2 - i]); // First half of fixup shifts.
3645     vxor(data0[i + unroll_factor2 / 2], data0[i + unroll_factor2 / 2], data1[i + unroll_factor2 / 2]);
3646   }
3647 
3648   // Last data register is ok, other ones need fixup shift.
3649   for (int i = unroll_factor2 / 2; i < unroll_factor2 - 1; ++i) {
3650     vpmsumw(data0[i], data0[i], consts0[unroll_factor2 - 2 - i]);
3651   }
3652 
3653   // Combine to 128 bit result vector VCRC = data0[0].
3654   for (int i = 1; i < unroll_factor2; i<<=1) {
3655     for (int j = 0; j <= unroll_factor2 - 2*i; j+=2*i) {
3656       vxor(data0[j], data0[j], data0[j+i]);
3657     }
3658   }
3659   cmpd(CCR0, len, num_bytes);
3660   bge(CCR0, L_outer_loop);
3661 
3662   // Last chance with lower num_bytes.
3663   bind(L_last);
3664   srdi(loop_count, len, exact_log2(16 * 2 * unroll_factor2)); // Use double-iterations.
3665   // Point behind last const for inner loop.
3666   add_const_optimized(cur_const, constants, outer_consts_size + inner_consts_size);
3667   sldi(R0, loop_count, exact_log2(16 * 2)); // Bytes of constants to be used.
3668   clrrdi(num_bytes, len, exact_log2(16 * 2 * unroll_factor2));
3669   subf(cur_const, R0, cur_const); // Point to constant to be used first.
3670 
3671   addic_(loop_count, loop_count, -1); // One double-iteration peeled off.
3672   bgt(CCR0, L_outer_loop);
3673   // ********** Main loop end **********
3674 
3675   // Restore DSCR pre-fetch value.
3676   if (VM_Version::has_mfdscr()) {
3677     load_const_optimized(t0, VM_Version::_dscr_val);
3678     mtdscr(t0);
3679   }
3680 
3681   // ********** Simple loop for remaining 16 byte blocks **********
3682   {
3683     Label L_loop, L_done;
3684 
3685     srdi_(t0, len, 4); // 16 bytes per iteration
3686     clrldi(len, len, 64-4);
3687     beq(CCR0, L_done);
3688 
3689     // Point to const (same as last const for inner loop).
3690     add_const_optimized(cur_const, constants, outer_consts_size + inner_consts_size - 16);
3691     mtctr(t0);
3692     lvx(Vtmp2, cur_const);
3693 
3694     align(32);
3695     bind(L_loop);
3696 
3697     lvx(Vtmp, buf);
3698     addi(buf, buf, 16);
3699     vpermxor(VCRC, VCRC, VCRC, Vc); // xor both halves to 64 bit result.
3700     BE_swap_bytes(Vtmp);
3701     vxor(VCRC, VCRC, Vtmp);
3702     vpmsumw(VCRC, VCRC, Vtmp2);
3703     bdnz(L_loop);
3704 
3705     bind(L_done);
3706   }
3707   // ********** Simple loop end **********
3708 #undef BE_swap_bytes
3709 
3710   // Point to Barrett constants
3711   add_const_optimized(cur_const, constants, outer_consts_size + inner_consts_size);
3712 
3713   vspltisb(zeroes, 0);
3714 
3715   // Combine to 64 bit result.
3716   vpermxor(VCRC, VCRC, VCRC, Vc); // xor both halves to 64 bit result.
3717 
3718   // Reduce to 32 bit CRC: Remainder by multiply-high.
3719   lvx(Vtmp, cur_const);
3720   vsldoi(Vtmp2, zeroes, VCRC, 12);  // Extract high 32 bit.
3721   vpmsumd(Vtmp2, Vtmp2, Vtmp);      // Multiply by inverse long poly.
3722   vsldoi(Vtmp2, zeroes, Vtmp2, 12); // Extract high 32 bit.
3723   vsldoi(Vtmp, zeroes, Vtmp, 8);
3724   vpmsumd(Vtmp2, Vtmp2, Vtmp);      // Multiply quotient by long poly.
3725   vxor(VCRC, VCRC, Vtmp2);          // Remainder fits into 32 bit.
3726 
3727   // Move result. len is already updated.
3728   vsldoi(VCRC, VCRC, zeroes, 8);
3729   mfvrd(crc, VCRC);
3730 
3731   // Restore non-volatile Vector registers (frameless).
3732   offsetInt = 0;
3733   offsetInt -= 16; li(offset, offsetInt); lvx(VR20, offset, R1_SP);
3734   offsetInt -= 16; li(offset, offsetInt); lvx(VR21, offset, R1_SP);
3735   offsetInt -= 16; li(offset, offsetInt); lvx(VR22, offset, R1_SP);
3736   offsetInt -= 16; li(offset, offsetInt); lvx(VR23, offset, R1_SP);
3737   offsetInt -= 16; li(offset, offsetInt); lvx(VR24, offset, R1_SP);
3738   offsetInt -= 16; li(offset, offsetInt); lvx(VR25, offset, R1_SP);
3739 #ifndef VM_LITTLE_ENDIAN
3740   offsetInt -= 16; li(offset, offsetInt); lvx(VR26, offset, R1_SP);
3741 #endif
3742   offsetInt -= 8;  ld(R14, offsetInt, R1_SP);
3743   offsetInt -= 8;  ld(R15, offsetInt, R1_SP);
3744 }
3745 
3746 void MacroAssembler::crc32(Register crc, Register buf, Register len, Register t0, Register t1, Register t2,
3747                            Register t3, Register t4, Register t5, Register t6, Register t7, bool is_crc32c) {
3748   load_const_optimized(t0, is_crc32c ? StubRoutines::crc32c_table_addr()
3749                                      : StubRoutines::crc_table_addr()   , R0);
3750 
3751   if (VM_Version::has_vpmsumb()) {
3752     kernel_crc32_vpmsum(crc, buf, len, t0, t1, t2, t3, t4, t5, t6, t7, !is_crc32c);
3753   } else {
3754     kernel_crc32_1word(crc, buf, len, t0, t1, t2, t3, t4, t5, t6, t7, t0, !is_crc32c);
3755   }
3756 }
3757 
3758 void MacroAssembler::kernel_crc32_singleByteReg(Register crc, Register val, Register table, bool invertCRC) {
3759   assert_different_registers(crc, val, table);
3760 
3761   BLOCK_COMMENT("kernel_crc32_singleByteReg:");
3762   if (invertCRC) {
3763     nand(crc, crc, crc);                // 1s complement of crc
3764   }
3765 
3766   update_byte_crc32(crc, val, table);
3767 
3768   if (invertCRC) {
3769     nand(crc, crc, crc);                // 1s complement of crc
3770   }
3771 }
3772 
3773 // dest_lo += src1 + src2
3774 // dest_hi += carry1 + carry2
3775 void MacroAssembler::add2_with_carry(Register dest_hi,
3776                                      Register dest_lo,
3777                                      Register src1, Register src2) {
3778   li(R0, 0);
3779   addc(dest_lo, dest_lo, src1);
3780   adde(dest_hi, dest_hi, R0);
3781   addc(dest_lo, dest_lo, src2);
3782   adde(dest_hi, dest_hi, R0);
3783 }
3784 
3785 // Multiply 64 bit by 64 bit first loop.
3786 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart,
3787                                            Register x_xstart,
3788                                            Register y, Register y_idx,
3789                                            Register z,
3790                                            Register carry,
3791                                            Register product_high, Register product,
3792                                            Register idx, Register kdx,
3793                                            Register tmp) {
3794   //  jlong carry, x[], y[], z[];
3795   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) {
3796   //    huge_128 product = y[idx] * x[xstart] + carry;
3797   //    z[kdx] = (jlong)product;
3798   //    carry  = (jlong)(product >>> 64);
3799   //  }
3800   //  z[xstart] = carry;
3801 
3802   Label L_first_loop, L_first_loop_exit;
3803   Label L_one_x, L_one_y, L_multiply;
3804 
3805   addic_(xstart, xstart, -1);
3806   blt(CCR0, L_one_x);   // Special case: length of x is 1.
3807 
3808   // Load next two integers of x.
3809   sldi(tmp, xstart, LogBytesPerInt);
3810   ldx(x_xstart, x, tmp);
3811 #ifdef VM_LITTLE_ENDIAN
3812   rldicl(x_xstart, x_xstart, 32, 0);
3813 #endif
3814 
3815   align(32, 16);
3816   bind(L_first_loop);
3817 
3818   cmpdi(CCR0, idx, 1);
3819   blt(CCR0, L_first_loop_exit);
3820   addi(idx, idx, -2);
3821   beq(CCR0, L_one_y);
3822 
3823   // Load next two integers of y.
3824   sldi(tmp, idx, LogBytesPerInt);
3825   ldx(y_idx, y, tmp);
3826 #ifdef VM_LITTLE_ENDIAN
3827   rldicl(y_idx, y_idx, 32, 0);
3828 #endif
3829 
3830 
3831   bind(L_multiply);
3832   multiply64(product_high, product, x_xstart, y_idx);
3833 
3834   li(tmp, 0);
3835   addc(product, product, carry);         // Add carry to result.
3836   adde(product_high, product_high, tmp); // Add carry of the last addition.
3837   addi(kdx, kdx, -2);
3838 
3839   // Store result.
3840 #ifdef VM_LITTLE_ENDIAN
3841   rldicl(product, product, 32, 0);
3842 #endif
3843   sldi(tmp, kdx, LogBytesPerInt);
3844   stdx(product, z, tmp);
3845   mr_if_needed(carry, product_high);
3846   b(L_first_loop);
3847 
3848 
3849   bind(L_one_y); // Load one 32 bit portion of y as (0,value).
3850 
3851   lwz(y_idx, 0, y);
3852   b(L_multiply);
3853 
3854 
3855   bind(L_one_x); // Load one 32 bit portion of x as (0,value).
3856 
3857   lwz(x_xstart, 0, x);
3858   b(L_first_loop);
3859 
3860   bind(L_first_loop_exit);
3861 }
3862 
3863 // Multiply 64 bit by 64 bit and add 128 bit.
3864 void MacroAssembler::multiply_add_128_x_128(Register x_xstart, Register y,
3865                                             Register z, Register yz_idx,
3866                                             Register idx, Register carry,
3867                                             Register product_high, Register product,
3868                                             Register tmp, int offset) {
3869 
3870   //  huge_128 product = (y[idx] * x_xstart) + z[kdx] + carry;
3871   //  z[kdx] = (jlong)product;
3872 
3873   sldi(tmp, idx, LogBytesPerInt);
3874   if (offset) {
3875     addi(tmp, tmp, offset);
3876   }
3877   ldx(yz_idx, y, tmp);
3878 #ifdef VM_LITTLE_ENDIAN
3879   rldicl(yz_idx, yz_idx, 32, 0);
3880 #endif
3881 
3882   multiply64(product_high, product, x_xstart, yz_idx);
3883   ldx(yz_idx, z, tmp);
3884 #ifdef VM_LITTLE_ENDIAN
3885   rldicl(yz_idx, yz_idx, 32, 0);
3886 #endif
3887 
3888   add2_with_carry(product_high, product, carry, yz_idx);
3889 
3890   sldi(tmp, idx, LogBytesPerInt);
3891   if (offset) {
3892     addi(tmp, tmp, offset);
3893   }
3894 #ifdef VM_LITTLE_ENDIAN
3895   rldicl(product, product, 32, 0);
3896 #endif
3897   stdx(product, z, tmp);
3898 }
3899 
3900 // Multiply 128 bit by 128 bit. Unrolled inner loop.
3901 void MacroAssembler::multiply_128_x_128_loop(Register x_xstart,
3902                                              Register y, Register z,
3903                                              Register yz_idx, Register idx, Register carry,
3904                                              Register product_high, Register product,
3905                                              Register carry2, Register tmp) {
3906 
3907   //  jlong carry, x[], y[], z[];
3908   //  int kdx = ystart+1;
3909   //  for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
3910   //    huge_128 product = (y[idx+1] * x_xstart) + z[kdx+idx+1] + carry;
3911   //    z[kdx+idx+1] = (jlong)product;
3912   //    jlong carry2 = (jlong)(product >>> 64);
3913   //    product = (y[idx] * x_xstart) + z[kdx+idx] + carry2;
3914   //    z[kdx+idx] = (jlong)product;
3915   //    carry = (jlong)(product >>> 64);
3916   //  }
3917   //  idx += 2;
3918   //  if (idx > 0) {
3919   //    product = (y[idx] * x_xstart) + z[kdx+idx] + carry;
3920   //    z[kdx+idx] = (jlong)product;
3921   //    carry = (jlong)(product >>> 64);
3922   //  }
3923 
3924   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
3925   const Register jdx = R0;
3926 
3927   // Scale the index.
3928   srdi_(jdx, idx, 2);
3929   beq(CCR0, L_third_loop_exit);
3930   mtctr(jdx);
3931 
3932   align(32, 16);
3933   bind(L_third_loop);
3934 
3935   addi(idx, idx, -4);
3936 
3937   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product_high, product, tmp, 8);
3938   mr_if_needed(carry2, product_high);
3939 
3940   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry2, product_high, product, tmp, 0);
3941   mr_if_needed(carry, product_high);
3942   bdnz(L_third_loop);
3943 
3944   bind(L_third_loop_exit);  // Handle any left-over operand parts.
3945 
3946   andi_(idx, idx, 0x3);
3947   beq(CCR0, L_post_third_loop_done);
3948 
3949   Label L_check_1;
3950 
3951   addic_(idx, idx, -2);
3952   blt(CCR0, L_check_1);
3953 
3954   multiply_add_128_x_128(x_xstart, y, z, yz_idx, idx, carry, product_high, product, tmp, 0);
3955   mr_if_needed(carry, product_high);
3956 
3957   bind(L_check_1);
3958 
3959   addi(idx, idx, 0x2);
3960   andi_(idx, idx, 0x1);
3961   addic_(idx, idx, -1);
3962   blt(CCR0, L_post_third_loop_done);
3963 
3964   sldi(tmp, idx, LogBytesPerInt);
3965   lwzx(yz_idx, y, tmp);
3966   multiply64(product_high, product, x_xstart, yz_idx);
3967   lwzx(yz_idx, z, tmp);
3968 
3969   add2_with_carry(product_high, product, yz_idx, carry);
3970 
3971   sldi(tmp, idx, LogBytesPerInt);
3972   stwx(product, z, tmp);
3973   srdi(product, product, 32);
3974 
3975   sldi(product_high, product_high, 32);
3976   orr(product, product, product_high);
3977   mr_if_needed(carry, product);
3978 
3979   bind(L_post_third_loop_done);
3980 }   // multiply_128_x_128_loop
3981 
3982 void MacroAssembler::muladd(Register out, Register in,
3983                             Register offset, Register len, Register k,
3984                             Register tmp1, Register tmp2, Register carry) {
3985 
3986   // Labels
3987   Label LOOP, SKIP;
3988 
3989   // Make sure length is positive.
3990   cmpdi  (CCR0,    len,     0);
3991 
3992   // Prepare variables
3993   subi   (offset,  offset,  4);
3994   li     (carry,   0);
3995   ble    (CCR0,    SKIP);
3996 
3997   mtctr  (len);
3998   subi   (len,     len,     1    );
3999   sldi   (len,     len,     2    );
4000 
4001   // Main loop
4002   bind(LOOP);
4003   lwzx   (tmp1,    len,     in   );
4004   lwzx   (tmp2,    offset,  out  );
4005   mulld  (tmp1,    tmp1,    k    );
4006   add    (tmp2,    carry,   tmp2 );
4007   add    (tmp2,    tmp1,    tmp2 );
4008   stwx   (tmp2,    offset,  out  );
4009   srdi   (carry,   tmp2,    32   );
4010   subi   (offset,  offset,  4    );
4011   subi   (len,     len,     4    );
4012   bdnz   (LOOP);
4013   bind(SKIP);
4014 }
4015 
4016 void MacroAssembler::multiply_to_len(Register x, Register xlen,
4017                                      Register y, Register ylen,
4018                                      Register z, Register zlen,
4019                                      Register tmp1, Register tmp2,
4020                                      Register tmp3, Register tmp4,
4021                                      Register tmp5, Register tmp6,
4022                                      Register tmp7, Register tmp8,
4023                                      Register tmp9, Register tmp10,
4024                                      Register tmp11, Register tmp12,
4025                                      Register tmp13) {
4026 
4027   ShortBranchVerifier sbv(this);
4028 
4029   assert_different_registers(x, xlen, y, ylen, z, zlen,
4030                              tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
4031   assert_different_registers(x, xlen, y, ylen, z, zlen,
4032                              tmp1, tmp2, tmp3, tmp4, tmp5, tmp7);
4033   assert_different_registers(x, xlen, y, ylen, z, zlen,
4034                              tmp1, tmp2, tmp3, tmp4, tmp5, tmp8);
4035 
4036   const Register idx = tmp1;
4037   const Register kdx = tmp2;
4038   const Register xstart = tmp3;
4039 
4040   const Register y_idx = tmp4;
4041   const Register carry = tmp5;
4042   const Register product = tmp6;
4043   const Register product_high = tmp7;
4044   const Register x_xstart = tmp8;
4045   const Register tmp = tmp9;
4046 
4047   // First Loop.
4048   //
4049   //  final static long LONG_MASK = 0xffffffffL;
4050   //  int xstart = xlen - 1;
4051   //  int ystart = ylen - 1;
4052   //  long carry = 0;
4053   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx-, kdx--) {
4054   //    long product = (y[idx] & LONG_MASK) * (x[xstart] & LONG_MASK) + carry;
4055   //    z[kdx] = (int)product;
4056   //    carry = product >>> 32;
4057   //  }
4058   //  z[xstart] = (int)carry;
4059 
4060   mr_if_needed(idx, ylen);        // idx = ylen
4061   mr_if_needed(kdx, zlen);        // kdx = xlen + ylen
4062   li(carry, 0);                   // carry = 0
4063 
4064   Label L_done;
4065 
4066   addic_(xstart, xlen, -1);
4067   blt(CCR0, L_done);
4068 
4069   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z,
4070                         carry, product_high, product, idx, kdx, tmp);
4071 
4072   Label L_second_loop;
4073 
4074   cmpdi(CCR0, kdx, 0);
4075   beq(CCR0, L_second_loop);
4076 
4077   Label L_carry;
4078 
4079   addic_(kdx, kdx, -1);
4080   beq(CCR0, L_carry);
4081 
4082   // Store lower 32 bits of carry.
4083   sldi(tmp, kdx, LogBytesPerInt);
4084   stwx(carry, z, tmp);
4085   srdi(carry, carry, 32);
4086   addi(kdx, kdx, -1);
4087 
4088 
4089   bind(L_carry);
4090 
4091   // Store upper 32 bits of carry.
4092   sldi(tmp, kdx, LogBytesPerInt);
4093   stwx(carry, z, tmp);
4094 
4095   // Second and third (nested) loops.
4096   //
4097   //  for (int i = xstart-1; i >= 0; i--) { // Second loop
4098   //    carry = 0;
4099   //    for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
4100   //      long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
4101   //                     (z[k] & LONG_MASK) + carry;
4102   //      z[k] = (int)product;
4103   //      carry = product >>> 32;
4104   //    }
4105   //    z[i] = (int)carry;
4106   //  }
4107   //
4108   //  i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = rdx
4109 
4110   bind(L_second_loop);
4111 
4112   li(carry, 0);                   // carry = 0;
4113 
4114   addic_(xstart, xstart, -1);     // i = xstart-1;
4115   blt(CCR0, L_done);
4116 
4117   Register zsave = tmp10;
4118 
4119   mr(zsave, z);
4120 
4121 
4122   Label L_last_x;
4123 
4124   sldi(tmp, xstart, LogBytesPerInt);
4125   add(z, z, tmp);                 // z = z + k - j
4126   addi(z, z, 4);
4127   addic_(xstart, xstart, -1);     // i = xstart-1;
4128   blt(CCR0, L_last_x);
4129 
4130   sldi(tmp, xstart, LogBytesPerInt);
4131   ldx(x_xstart, x, tmp);
4132 #ifdef VM_LITTLE_ENDIAN
4133   rldicl(x_xstart, x_xstart, 32, 0);
4134 #endif
4135 
4136 
4137   Label L_third_loop_prologue;
4138 
4139   bind(L_third_loop_prologue);
4140 
4141   Register xsave = tmp11;
4142   Register xlensave = tmp12;
4143   Register ylensave = tmp13;
4144 
4145   mr(xsave, x);
4146   mr(xlensave, xstart);
4147   mr(ylensave, ylen);
4148 
4149 
4150   multiply_128_x_128_loop(x_xstart, y, z, y_idx, ylen,
4151                           carry, product_high, product, x, tmp);
4152 
4153   mr(z, zsave);
4154   mr(x, xsave);
4155   mr(xlen, xlensave);   // This is the decrement of the loop counter!
4156   mr(ylen, ylensave);
4157 
4158   addi(tmp3, xlen, 1);
4159   sldi(tmp, tmp3, LogBytesPerInt);
4160   stwx(carry, z, tmp);
4161   addic_(tmp3, tmp3, -1);
4162   blt(CCR0, L_done);
4163 
4164   srdi(carry, carry, 32);
4165   sldi(tmp, tmp3, LogBytesPerInt);
4166   stwx(carry, z, tmp);
4167   b(L_second_loop);
4168 
4169   // Next infrequent code is moved outside loops.
4170   bind(L_last_x);
4171 
4172   lwz(x_xstart, 0, x);
4173   b(L_third_loop_prologue);
4174 
4175   bind(L_done);
4176 }   // multiply_to_len
4177 
4178 void MacroAssembler::asm_assert(bool check_equal, const char *msg) {
4179 #ifdef ASSERT
4180   Label ok;
4181   if (check_equal) {
4182     beq(CCR0, ok);
4183   } else {
4184     bne(CCR0, ok);
4185   }
4186   stop(msg);
4187   bind(ok);
4188 #endif
4189 }
4190 
4191 void MacroAssembler::asm_assert_mems_zero(bool check_equal, int size, int mem_offset,
4192                                           Register mem_base, const char* msg) {
4193 #ifdef ASSERT
4194   switch (size) {
4195     case 4:
4196       lwz(R0, mem_offset, mem_base);
4197       cmpwi(CCR0, R0, 0);
4198       break;
4199     case 8:
4200       ld(R0, mem_offset, mem_base);
4201       cmpdi(CCR0, R0, 0);
4202       break;
4203     default:
4204       ShouldNotReachHere();
4205   }
4206   asm_assert(check_equal, msg);
4207 #endif // ASSERT
4208 }
4209 
4210 void MacroAssembler::verify_thread() {
4211   if (VerifyThread) {
4212     unimplemented("'VerifyThread' currently not implemented on PPC");
4213   }
4214 }
4215 
4216 void MacroAssembler::verify_coop(Register coop, const char* msg) {
4217   if (!VerifyOops) { return; }
4218   if (UseCompressedOops) { decode_heap_oop(coop); }
4219   verify_oop(coop, msg);
4220   if (UseCompressedOops) { encode_heap_oop(coop, coop); }
4221 }
4222 
4223 // READ: oop. KILL: R0. Volatile floats perhaps.
4224 void MacroAssembler::verify_oop(Register oop, const char* msg) {
4225   if (!VerifyOops) {
4226     return;
4227   }
4228 
4229   address/* FunctionDescriptor** */fd = StubRoutines::verify_oop_subroutine_entry_address();
4230   const Register tmp = R11; // Will be preserved.
4231   const int nbytes_save = MacroAssembler::num_volatile_regs * 8;
4232 
4233   BLOCK_COMMENT("verify_oop {");
4234 
4235   save_volatile_gprs(R1_SP, -nbytes_save); // except R0
4236 
4237   mr_if_needed(R4_ARG2, oop);
4238   save_LR_CR(tmp); // save in old frame
4239   push_frame_reg_args(nbytes_save, tmp);
4240   // load FunctionDescriptor** / entry_address *
4241   load_const_optimized(tmp, fd, R0);
4242   // load FunctionDescriptor* / entry_address
4243   ld(tmp, 0, tmp);
4244   load_const_optimized(R3_ARG1, (address)msg, R0);
4245   // Call destination for its side effect.
4246   call_c(tmp);
4247 
4248   pop_frame();
4249   restore_LR_CR(tmp);
4250   restore_volatile_gprs(R1_SP, -nbytes_save); // except R0
4251 
4252   BLOCK_COMMENT("} verify_oop");
4253 }
4254 
4255 void MacroAssembler::verify_oop_addr(RegisterOrConstant offs, Register base, const char* msg) {
4256   if (!VerifyOops) {
4257     return;
4258   }
4259 
4260   address/* FunctionDescriptor** */fd = StubRoutines::verify_oop_subroutine_entry_address();
4261   const Register tmp = R11; // Will be preserved.
4262   const int nbytes_save = MacroAssembler::num_volatile_regs * 8;
4263   save_volatile_gprs(R1_SP, -nbytes_save); // except R0
4264 
4265   ld(R4_ARG2, offs, base);
4266   save_LR_CR(tmp); // save in old frame
4267   push_frame_reg_args(nbytes_save, tmp);
4268   // load FunctionDescriptor** / entry_address *
4269   load_const_optimized(tmp, fd, R0);
4270   // load FunctionDescriptor* / entry_address
4271   ld(tmp, 0, tmp);
4272   load_const_optimized(R3_ARG1, (address)msg, R0);
4273   // Call destination for its side effect.
4274   call_c(tmp);
4275 
4276   pop_frame();
4277   restore_LR_CR(tmp);
4278   restore_volatile_gprs(R1_SP, -nbytes_save); // except R0
4279 }
4280 
4281 // Call a C-function that prints output.
4282 void MacroAssembler::stop(int type, const char* msg) {
4283   bool msg_present = (msg != NULL);
4284 
4285 #ifndef PRODUCT
4286   block_comment(err_msg("stop(type %d): %s {", type, msg_present ? msg : "null"));
4287 #else
4288   block_comment("stop {");
4289 #endif
4290 
4291   if (msg_present) {
4292     type |= stop_msg_present;
4293   }
4294   tdi_unchecked(traptoUnconditional, 0/*reg 0*/, type);
4295   if (msg_present) {
4296     emit_int64((uintptr_t)msg);
4297   }
4298 
4299   block_comment("} stop;");
4300 }
4301 
4302 #ifndef PRODUCT
4303 // Write pattern 0x0101010101010101 in memory region [low-before, high+after].
4304 // Val, addr are temp registers.
4305 // If low == addr, addr is killed.
4306 // High is preserved.
4307 void MacroAssembler::zap_from_to(Register low, int before, Register high, int after, Register val, Register addr) {
4308   if (!ZapMemory) return;
4309 
4310   assert_different_registers(low, val);
4311 
4312   BLOCK_COMMENT("zap memory region {");
4313   load_const_optimized(val, 0x0101010101010101);
4314   int size = before + after;
4315   if (low == high && size < 5 && size > 0) {
4316     int offset = -before*BytesPerWord;
4317     for (int i = 0; i < size; ++i) {
4318       std(val, offset, low);
4319       offset += (1*BytesPerWord);
4320     }
4321   } else {
4322     addi(addr, low, -before*BytesPerWord);
4323     assert_different_registers(high, val);
4324     if (after) addi(high, high, after * BytesPerWord);
4325     Label loop;
4326     bind(loop);
4327     std(val, 0, addr);
4328     addi(addr, addr, 8);
4329     cmpd(CCR6, addr, high);
4330     ble(CCR6, loop);
4331     if (after) addi(high, high, -after * BytesPerWord);  // Correct back to old value.
4332   }
4333   BLOCK_COMMENT("} zap memory region");
4334 }
4335 
4336 #endif // !PRODUCT
4337 
4338 void SkipIfEqualZero::skip_to_label_if_equal_zero(MacroAssembler* masm, Register temp,
4339                                                   const bool* flag_addr, Label& label) {
4340   int simm16_offset = masm->load_const_optimized(temp, (address)flag_addr, R0, true);
4341   assert(sizeof(bool) == 1, "PowerPC ABI");
4342   masm->lbz(temp, simm16_offset, temp);
4343   masm->cmpwi(CCR0, temp, 0);
4344   masm->beq(CCR0, label);
4345 }
4346 
4347 SkipIfEqualZero::SkipIfEqualZero(MacroAssembler* masm, Register temp, const bool* flag_addr) : _masm(masm), _label() {
4348   skip_to_label_if_equal_zero(masm, temp, flag_addr, _label);
4349 }
4350 
4351 SkipIfEqualZero::~SkipIfEqualZero() {
4352   _masm->bind(_label);
4353 }
4354 
4355 void MacroAssembler::cache_wb(Address line) {
4356   assert(line.index() == noreg, "index should be noreg");
4357   assert(line.disp() == 0, "displacement should be 0");
4358   assert(VM_Version::supports_data_cache_line_flush(), "CPU or OS does not support flush to persistent memory");
4359   // Data Cache Store, not really a flush, so it works like a sync of cache
4360   // line and persistent mem, i.e. copying the cache line to persistent whilst
4361   // not invalidating the cache line.
4362   dcbst(line.base());
4363 }
4364 
4365 void MacroAssembler::cache_wbsync(bool is_presync) {
4366   assert(VM_Version::supports_data_cache_line_flush(), "CPU or OS does not support sync related to persistent memory");
4367   // We only need a post sync barrier. Post means _after_ a cache line flush or
4368   // store instruction, pre means a barrier emitted before such a instructions.
4369   if (!is_presync) {
4370     fence();
4371   }
4372 }