1 /*
  2  * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2012, 2023 SAP SE. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  *
 24  */
 25 
 26 #ifndef CPU_PPC_MACROASSEMBLER_PPC_HPP
 27 #define CPU_PPC_MACROASSEMBLER_PPC_HPP
 28 
 29 #include "asm/assembler.hpp"
 30 #include "oops/accessDecorators.hpp"
 31 #include "utilities/macros.hpp"
 32 
 33 // MacroAssembler extends Assembler by a few frequently used macros.
 34 
 35 class ciTypeArray;
 36 class OopMap;
 37 
 38 class MacroAssembler: public Assembler {
 39  public:
 40   MacroAssembler(CodeBuffer* code) : Assembler(code) {}
 41 
 42   // Indicates whether and, if so, which registers must be preserved when calling runtime code.
 43   enum PreservationLevel {
 44     PRESERVATION_NONE,
 45     PRESERVATION_FRAME_LR,
 46     PRESERVATION_FRAME_LR_GP_REGS,
 47     PRESERVATION_FRAME_LR_GP_FP_REGS
 48   };
 49 
 50   //
 51   // Optimized instruction emitters
 52   //
 53 
 54   inline static int largeoffset_si16_si16_hi(int si31) { return (si31 + (1<<15)) >> 16; }
 55   inline static int largeoffset_si16_si16_lo(int si31) { return si31 - (((si31 + (1<<15)) >> 16) << 16); }
 56 
 57   // load d = *[a+si31]
 58   // Emits several instructions if the offset is not encodable in one instruction.
 59   void ld_largeoffset_unchecked(Register d, int si31, Register a, int emit_filler_nop);
 60   void ld_largeoffset          (Register d, int si31, Register a, int emit_filler_nop);
 61   inline static bool is_ld_largeoffset(address a);
 62   inline static int get_ld_largeoffset_offset(address a);
 63 
 64   inline void round_to(Register r, int modulus);
 65 
 66   // Load/store with type given by parameter.
 67   void load_sized_value( Register dst, RegisterOrConstant offs, Register base, size_t size_in_bytes, bool is_signed);
 68   void store_sized_value(Register dst, RegisterOrConstant offs, Register base, size_t size_in_bytes);
 69 
 70   // Move register if destination register and target register are different
 71   inline void mr_if_needed(Register rd, Register rs);
 72   inline void fmr_if_needed(FloatRegister rd, FloatRegister rs);
 73   // This is dedicated for emitting scheduled mach nodes. For better
 74   // readability of the ad file I put it here.
 75   // Endgroups are not needed if
 76   //  - the scheduler is off
 77   //  - the scheduler found that there is a natural group end, in that
 78   //    case it reduced the size of the instruction used in the test
 79   //    yielding 'needed'.
 80   inline void endgroup_if_needed(bool needed);
 81 
 82   // Memory barriers.
 83   inline void membar(int bits);
 84   inline void release();
 85   inline void acquire();
 86   inline void fence();
 87 
 88   // nop padding
 89   void align(int modulus, int max = 252, int rem = 0);
 90 
 91   // Align prefix opcode to make sure it's not on the last word of a
 92   // 64-byte block.
 93   //
 94   // Note: do not call align_prefix() in a .ad file (e.g. ppc.ad).  Instead
 95   // add ins_alignment(2) to the instruct definition and implement the
 96   // compute_padding() method of the instruct node to use
 97   // compute_prefix_padding().  See loadConI32Node::compute_padding() in
 98   // ppc.ad for an example.
 99   void align_prefix();
100 
101   //
102   // Constants, loading constants, TOC support
103   //
104 
105   // Address of the global TOC.
106   inline static address global_toc();
107   // Offset of given address to the global TOC.
108   inline static int offset_to_global_toc(const address addr);
109 
110   // Address of TOC of the current method.
111   inline address method_toc();
112   // Offset of given address to TOC of the current method.
113   inline int offset_to_method_toc(const address addr);
114 
115   // Global TOC.
116   void calculate_address_from_global_toc(Register dst, address addr,
117                                          bool hi16 = true, bool lo16 = true,
118                                          bool add_relocation = true, bool emit_dummy_addr = false);
119   inline void calculate_address_from_global_toc_hi16only(Register dst, address addr) {
120     calculate_address_from_global_toc(dst, addr, true, false);
121   };
122   inline void calculate_address_from_global_toc_lo16only(Register dst, address addr) {
123     calculate_address_from_global_toc(dst, addr, false, true);
124   };
125 
126   inline static bool is_calculate_address_from_global_toc_at(address a, address bound);
127   // Returns address of first instruction in sequence.
128   static address patch_calculate_address_from_global_toc_at(address a, address bound, address addr);
129   static address get_address_of_calculate_address_from_global_toc_at(address a, address addr);
130 
131 #ifdef _LP64
132   // Patch narrow oop constant.
133   inline static bool is_set_narrow_oop(address a, address bound);
134   // Returns address of first instruction in sequence.
135   static address patch_set_narrow_oop(address a, address bound, narrowOop data);
136   static narrowOop get_narrow_oop(address a, address bound);
137 #endif
138 
139   inline static bool is_load_const_at(address a);
140 
141   // Emits an oop const to the constant pool, loads the constant, and
142   // sets a relocation info with address current_pc.
143   // Returns true if successful.
144   bool load_const_from_method_toc(Register dst, AddressLiteral& a, Register toc, bool fixed_size = false);
145 
146   static bool is_load_const_from_method_toc_at(address a);
147   static int get_offset_of_load_const_from_method_toc_at(address a);
148 
149   // Get the 64 bit constant from a `load_const' sequence.
150   static long get_const(address load_const);
151 
152   // Patch the 64 bit constant of a `load_const' sequence. This is a
153   // low level procedure. It neither flushes the instruction cache nor
154   // is it atomic.
155   static void patch_const(address load_const, long x);
156 
157   // Metadata in code that we have to keep track of.
158   AddressLiteral allocate_metadata_address(Metadata* obj); // allocate_index
159   AddressLiteral constant_metadata_address(Metadata* obj); // find_index
160   // Oops used directly in compiled code are stored in the constant pool,
161   // and loaded from there.
162   // Allocate new entry for oop in constant pool. Generate relocation.
163   AddressLiteral allocate_oop_address(jobject obj);
164   // Find oop obj in constant pool. Return relocation with it's index.
165   AddressLiteral constant_oop_address(jobject obj);
166 
167   // Find oop in constant pool and emit instructions to load it.
168   // Uses constant_oop_address.
169   inline void set_oop_constant(jobject obj, Register d);
170   // Same as load_address.
171   inline void set_oop         (AddressLiteral obj_addr, Register d);
172 
173   //
174   // branch, jump
175   //
176   // set dst to -1, 0, +1 as follows: if CCR0bi is "greater than", dst is set to 1,
177   // if CCR0bi is "equal", dst is set to 0, otherwise it's set to -1.
178   void inline set_cmp3(Register dst);
179   // set dst to (treat_unordered_like_less ? -1 : +1)
180   void inline set_cmpu3(Register dst, bool treat_unordered_like_less);
181 
182   inline void pd_patch_instruction(address branch, address target, const char* file, int line);
183   NOT_PRODUCT(static void pd_print_patched_instruction(address branch);)
184 
185   // Conditional far branch for destinations encodable in 24+2 bits.
186   // Same interface as bc, e.g. no inverse boint-field.
187   enum {
188     bc_far_optimize_not         = 0,
189     bc_far_optimize_on_relocate = 1
190   };
191   // optimize: flag for telling the conditional far branch to optimize
192   //           itself when relocated.
193   void bc_far(int boint, int biint, Label& dest, int optimize);
194   void bc_far_optimized(int boint, int biint, Label& dest); // 1 or 2 instructions
195   // Relocation of conditional far branches.
196   static bool    is_bc_far_at(address instruction_addr);
197   static address get_dest_of_bc_far_at(address instruction_addr);
198   static void    set_dest_of_bc_far_at(address instruction_addr, address dest);
199  private:
200   static bool inline is_bc_far_variant1_at(address instruction_addr);
201   static bool inline is_bc_far_variant2_at(address instruction_addr);
202   static bool inline is_bc_far_variant3_at(address instruction_addr);
203  public:
204 
205   // Convenience bc_far versions.
206   inline void blt_far(ConditionRegister crx, Label& L, int optimize);
207   inline void bgt_far(ConditionRegister crx, Label& L, int optimize);
208   inline void beq_far(ConditionRegister crx, Label& L, int optimize);
209   inline void bso_far(ConditionRegister crx, Label& L, int optimize);
210   inline void bge_far(ConditionRegister crx, Label& L, int optimize);
211   inline void ble_far(ConditionRegister crx, Label& L, int optimize);
212   inline void bne_far(ConditionRegister crx, Label& L, int optimize);
213   inline void bns_far(ConditionRegister crx, Label& L, int optimize);
214 
215   // Emit, identify and patch a NOT mt-safe patchable 64 bit absolute call/jump.
216  private:
217   enum {
218     bxx64_patchable_instruction_count = (2/*load_codecache_const*/ + 3/*5load_const*/ + 1/*mtctr*/ + 1/*bctrl*/),
219     bxx64_patchable_size              = bxx64_patchable_instruction_count * BytesPerInstWord,
220     bxx64_patchable_ret_addr_offset   = bxx64_patchable_size
221   };
222   void bxx64_patchable(address target, relocInfo::relocType rt, bool link);
223   static bool is_bxx64_patchable_at(            address instruction_addr, bool link);
224   // Does the instruction use a pc-relative encoding of the destination?
225   static bool is_bxx64_patchable_pcrelative_at( address instruction_addr, bool link);
226   static bool is_bxx64_patchable_variant1_at(   address instruction_addr, bool link);
227   // Load destination relative to global toc.
228   static bool is_bxx64_patchable_variant1b_at(  address instruction_addr, bool link);
229   static bool is_bxx64_patchable_variant2_at(   address instruction_addr, bool link);
230   static void set_dest_of_bxx64_patchable_at(   address instruction_addr, address target, bool link);
231   static address get_dest_of_bxx64_patchable_at(address instruction_addr, bool link);
232 
233  public:
234   // call
235   enum {
236     bl64_patchable_instruction_count = bxx64_patchable_instruction_count,
237     bl64_patchable_size              = bxx64_patchable_size,
238     bl64_patchable_ret_addr_offset   = bxx64_patchable_ret_addr_offset
239   };
240   inline void bl64_patchable(address target, relocInfo::relocType rt) {
241     bxx64_patchable(target, rt, /*link=*/true);
242   }
243   inline static bool is_bl64_patchable_at(address instruction_addr) {
244     return is_bxx64_patchable_at(instruction_addr, /*link=*/true);
245   }
246   inline static bool is_bl64_patchable_pcrelative_at(address instruction_addr) {
247     return is_bxx64_patchable_pcrelative_at(instruction_addr, /*link=*/true);
248   }
249   inline static void set_dest_of_bl64_patchable_at(address instruction_addr, address target) {
250     set_dest_of_bxx64_patchable_at(instruction_addr, target, /*link=*/true);
251   }
252   inline static address get_dest_of_bl64_patchable_at(address instruction_addr) {
253     return get_dest_of_bxx64_patchable_at(instruction_addr, /*link=*/true);
254   }
255   // jump
256   enum {
257     b64_patchable_instruction_count = bxx64_patchable_instruction_count,
258     b64_patchable_size              = bxx64_patchable_size,
259   };
260   inline void b64_patchable(address target, relocInfo::relocType rt) {
261     bxx64_patchable(target, rt, /*link=*/false);
262   }
263   inline static bool is_b64_patchable_at(address instruction_addr) {
264     return is_bxx64_patchable_at(instruction_addr, /*link=*/false);
265   }
266   inline static bool is_b64_patchable_pcrelative_at(address instruction_addr) {
267     return is_bxx64_patchable_pcrelative_at(instruction_addr, /*link=*/false);
268   }
269   inline static void set_dest_of_b64_patchable_at(address instruction_addr, address target) {
270     set_dest_of_bxx64_patchable_at(instruction_addr, target, /*link=*/false);
271   }
272   inline static address get_dest_of_b64_patchable_at(address instruction_addr) {
273     return get_dest_of_bxx64_patchable_at(instruction_addr, /*link=*/false);
274   }
275 
276   //
277   // Support for frame handling
278   //
279 
280   // some ABI-related functions
281 
282   // Clobbers all volatile, (non-floating-point) general-purpose registers for debugging purposes.
283   // This is especially useful for making calls to the JRT in places in which this hasn't been done before;
284   // e.g. with the introduction of LRBs (load reference barriers) for concurrent garbage collection.
285   void clobber_volatile_gprs(Register excluded_register = noreg);
286   void clobber_carg_stack_slots(Register tmp);
287 
288   void save_nonvolatile_gprs(   Register dst_base, int offset);
289   void restore_nonvolatile_gprs(Register src_base, int offset);
290 
291   enum {
292     num_volatile_gp_regs = 11,
293     num_volatile_fp_regs = 14,
294     num_volatile_regs = num_volatile_gp_regs + num_volatile_fp_regs
295   };
296 
297   void save_volatile_gprs(   Register dst_base, int offset,
298                              bool include_fp_regs = true, bool include_R3_RET_reg = true);
299   void restore_volatile_gprs(Register src_base, int offset,
300                              bool include_fp_regs = true, bool include_R3_RET_reg = true);
301   void save_LR_CR(   Register tmp);     // tmp contains LR on return.
302   void restore_LR_CR(Register tmp);
303 
304   // Get current PC using bl-next-instruction trick.
305   address get_PC_trash_LR(Register result);
306 
307   // Resize current frame either relatively wrt to current SP or absolute.
308   void resize_frame(Register offset, Register tmp);
309   void resize_frame(int      offset, Register tmp);
310   void resize_frame_absolute(Register addr, Register tmp1, Register tmp2);
311 
312   // Push a frame of size bytes.
313   void push_frame(Register bytes, Register tmp);
314 
315   // Push a frame of size `bytes'. No abi space provided.
316   void push_frame(unsigned int bytes, Register tmp);
317 
318   // Push a frame of size `bytes' plus native_abi_reg_args on top.
319   void push_frame_reg_args(unsigned int bytes, Register tmp);
320 
321   // Setup up a new C frame with a spill area for non-volatile GPRs and additional
322   // space for local variables
323   void push_frame_reg_args_nonvolatiles(unsigned int bytes, Register tmp);
324 
325   // pop current C frame
326   void pop_frame();
327 
328   //
329   // Calls
330   //
331 
332  private:
333   address _last_calls_return_pc;
334 
335 #if defined(ABI_ELFv2)
336   // Generic version of a call to C function.
337   // Updates and returns _last_calls_return_pc.
338   address branch_to(Register function_entry, bool and_link);
339 #else
340   // Generic version of a call to C function via a function descriptor
341   // with variable support for C calling conventions (TOC, ENV, etc.).
342   // updates and returns _last_calls_return_pc.
343   address branch_to(Register function_descriptor, bool and_link, bool save_toc_before_call,
344                     bool restore_toc_after_call, bool load_toc_of_callee, bool load_env_of_callee);
345 #endif
346 
347  public:
348 
349   // Get the pc where the last call will return to. returns _last_calls_return_pc.
350   inline address last_calls_return_pc();
351 
352 #if defined(ABI_ELFv2)
353   // Call a C function via a function descriptor and use full C
354   // calling conventions. Updates and returns _last_calls_return_pc.
355   address call_c(Register function_entry);
356   // For tail calls: only branch, don't link, so callee returns to caller of this function.
357   address call_c_and_return_to_caller(Register function_entry);
358   address call_c(address function_entry, relocInfo::relocType rt);
359 #else
360   // Call a C function via a function descriptor and use full C
361   // calling conventions. Updates and returns _last_calls_return_pc.
362   address call_c(Register function_descriptor);
363   // For tail calls: only branch, don't link, so callee returns to caller of this function.
364   address call_c_and_return_to_caller(Register function_descriptor);
365   address call_c(const FunctionDescriptor* function_descriptor, relocInfo::relocType rt);
366   address call_c_using_toc(const FunctionDescriptor* function_descriptor, relocInfo::relocType rt,
367                            Register toc);
368 #endif
369 
370  protected:
371 
372   // It is imperative that all calls into the VM are handled via the
373   // call_VM macros. They make sure that the stack linkage is setup
374   // correctly. call_VM's correspond to ENTRY/ENTRY_X entry points
375   // while call_VM_leaf's correspond to LEAF entry points.
376   //
377   // This is the base routine called by the different versions of
378   // call_VM. The interpreter may customize this version by overriding
379   // it for its purposes (e.g., to save/restore additional registers
380   // when doing a VM call).
381   //
382   // If no last_java_sp is specified (noreg) then SP will be used instead.
383   virtual void call_VM_base(
384      // where an oop-result ends up if any; use noreg otherwise
385     Register        oop_result,
386     // to set up last_Java_frame in stubs; use noreg otherwise
387     Register        last_java_sp,
388     // the entry point
389     address         entry_point,
390     // flag which indicates if exception should be checked
391     bool            check_exception = true
392   );
393 
394   // Support for VM calls. This is the base routine called by the
395   // different versions of call_VM_leaf. The interpreter may customize
396   // this version by overriding it for its purposes (e.g., to
397   // save/restore additional registers when doing a VM call).
398   void call_VM_leaf_base(address entry_point);
399 
400  public:
401   // Call into the VM.
402   // Passes the thread pointer (in R3_ARG1) as a prepended argument.
403   // Makes sure oop return values are visible to the GC.
404   void call_VM(Register oop_result, address entry_point, bool check_exceptions = true);
405   void call_VM(Register oop_result, address entry_point, Register arg_1, bool check_exceptions = true);
406   void call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true);
407   void call_VM(Register oop_result, address entry_point, Register arg_1, Register arg_2, Register arg3, bool check_exceptions = true);
408   void call_VM_leaf(address entry_point);
409   void call_VM_leaf(address entry_point, Register arg_1);
410   void call_VM_leaf(address entry_point, Register arg_1, Register arg_2);
411   void call_VM_leaf(address entry_point, Register arg_1, Register arg_2, Register arg_3);
412 
413   // Call a stub function via a function descriptor, but don't save
414   // TOC before call, don't setup TOC and ENV for call, and don't
415   // restore TOC after call. Updates and returns _last_calls_return_pc.
416   inline address call_stub(Register function_entry);
417   inline void call_stub_and_return_to(Register function_entry, Register return_pc);
418 
419   void post_call_nop();
420 
421   //
422   // Java utilities
423   //
424 
425   // Read from the polling page, its address is already in a register.
426   inline void load_from_polling_page(Register polling_page_address, int offset = 0);
427   // Check whether instruction is a read access to the polling page
428   // which was emitted by load_from_polling_page(..).
429   static bool is_load_from_polling_page(int instruction, void* ucontext/*may be nullptr*/,
430                                         address* polling_address_ptr = nullptr);
431 
432   // Support for null-checks
433   //
434   // Generates code that causes a null OS exception if the content of reg is null.
435   // If the accessed location is M[reg + offset] and the offset is known, provide the
436   // offset. No explicit code generation is needed if the offset is within a certain
437   // range (0 <= offset <= page_size).
438 
439   // Stack overflow checking
440   void bang_stack_with_offset(int offset);
441 
442   // If instruction is a stack bang of the form ld, stdu, or
443   // stdux, return the banged address. Otherwise, return 0.
444   static address get_stack_bang_address(int instruction, void* ucontext);
445 
446   // Check for reserved stack access in method being exited. If the reserved
447   // stack area was accessed, protect it again and throw StackOverflowError.
448   void reserved_stack_check(Register return_pc);
449 
450   // Atomics
451   // CmpxchgX sets condition register to cmpX(current, compare).
452   // (flag == ne) => (dest_current_value != compare_value), (!swapped)
453   // (flag == eq) => (dest_current_value == compare_value), ( swapped)
454   static inline bool cmpxchgx_hint_acquire_lock()  { return true; }
455   // The stxcx will probably not be succeeded by a releasing store.
456   static inline bool cmpxchgx_hint_release_lock()  { return false; }
457   static inline bool cmpxchgx_hint_atomic_update() { return false; }
458 
459   // Cmpxchg semantics
460   enum {
461     MemBarNone = 0,
462     MemBarRel  = 1,
463     MemBarAcq  = 2,
464     MemBarFenceAfter = 4 // use powers of 2
465   };
466  private:
467   // Helper functions for word/sub-word atomics.
468   void atomic_get_and_modify_generic(Register dest_current_value, Register exchange_value,
469                                      Register addr_base, Register tmp1, Register tmp2, Register tmp3,
470                                      bool cmpxchgx_hint, bool is_add, int size);
471   void cmpxchg_loop_body(ConditionRegister flag, Register dest_current_value,
472                          Register compare_value, Register exchange_value,
473                          Register addr_base, Register tmp1, Register tmp2,
474                          Label &retry, Label &failed, bool cmpxchgx_hint, int size);
475   void cmpxchg_generic(ConditionRegister flag,
476                        Register dest_current_value, Register compare_value, Register exchange_value, Register addr_base,
477                        Register tmp1, Register tmp2,
478                        int semantics, bool cmpxchgx_hint, Register int_flag_success, bool contention_hint, bool weak, int size);
479  public:
480   // Temps and addr_base are killed if processor does not support Power 8 instructions.
481   // Result will be sign extended.
482   void getandsetb(Register dest_current_value, Register exchange_value, Register addr_base,
483                   Register tmp1, Register tmp2, Register tmp3, bool cmpxchgx_hint) {
484     atomic_get_and_modify_generic(dest_current_value, exchange_value, addr_base, tmp1, tmp2, tmp3, cmpxchgx_hint, false, 1);
485   }
486   // Temps and addr_base are killed if processor does not support Power 8 instructions.
487   // Result will be sign extended.
488   void getandseth(Register dest_current_value, Register exchange_value, Register addr_base,
489                   Register tmp1, Register tmp2, Register tmp3, bool cmpxchgx_hint) {
490     atomic_get_and_modify_generic(dest_current_value, exchange_value, addr_base, tmp1, tmp2, tmp3, cmpxchgx_hint, false, 2);
491   }
492   void getandsetw(Register dest_current_value, Register exchange_value, Register addr_base,
493                   bool cmpxchgx_hint) {
494     atomic_get_and_modify_generic(dest_current_value, exchange_value, addr_base, noreg, noreg, noreg, cmpxchgx_hint, false, 4);
495   }
496   void getandsetd(Register dest_current_value, Register exchange_value, Register addr_base,
497                   bool cmpxchgx_hint);
498   // tmp2/3 and addr_base are killed if processor does not support Power 8 instructions (tmp1 is always needed).
499   // Result will be sign extended.
500   void getandaddb(Register dest_current_value, Register inc_value, Register addr_base,
501                   Register tmp1, Register tmp2, Register tmp3, bool cmpxchgx_hint) {
502     atomic_get_and_modify_generic(dest_current_value, inc_value, addr_base, tmp1, tmp2, tmp3, cmpxchgx_hint, true, 1);
503   }
504   // tmp2/3 and addr_base are killed if processor does not support Power 8 instructions (tmp1 is always needed).
505   // Result will be sign extended.
506   void getandaddh(Register dest_current_value, Register inc_value, Register addr_base,
507                   Register tmp1, Register tmp2, Register tmp3, bool cmpxchgx_hint) {
508     atomic_get_and_modify_generic(dest_current_value, inc_value, addr_base, tmp1, tmp2, tmp3, cmpxchgx_hint, true, 2);
509   }
510   void getandaddw(Register dest_current_value, Register inc_value, Register addr_base,
511                   Register tmp1, bool cmpxchgx_hint) {
512     atomic_get_and_modify_generic(dest_current_value, inc_value, addr_base, tmp1, noreg, noreg, cmpxchgx_hint, true, 4);
513   }
514   void getandaddd(Register dest_current_value, Register exchange_value, Register addr_base,
515                   Register tmp, bool cmpxchgx_hint);
516   // Temps, addr_base and exchange_value are killed if processor does not support Power 8 instructions.
517   // compare_value must be at least 32 bit sign extended. Result will be sign extended.
518   void cmpxchgb(ConditionRegister flag,
519                 Register dest_current_value, Register compare_value, Register exchange_value, Register addr_base,
520                 Register tmp1, Register tmp2, int semantics, bool cmpxchgx_hint = false,
521                 Register int_flag_success = noreg, bool contention_hint = false, bool weak = false) {
522     cmpxchg_generic(flag, dest_current_value, compare_value, exchange_value, addr_base, tmp1, tmp2,
523                     semantics, cmpxchgx_hint, int_flag_success, contention_hint, weak, 1);
524   }
525   // Temps, addr_base and exchange_value are killed if processor does not support Power 8 instructions.
526   // compare_value must be at least 32 bit sign extended. Result will be sign extended.
527   void cmpxchgh(ConditionRegister flag,
528                 Register dest_current_value, Register compare_value, Register exchange_value, Register addr_base,
529                 Register tmp1, Register tmp2, int semantics, bool cmpxchgx_hint = false,
530                 Register int_flag_success = noreg, bool contention_hint = false, bool weak = false) {
531     cmpxchg_generic(flag, dest_current_value, compare_value, exchange_value, addr_base, tmp1, tmp2,
532                     semantics, cmpxchgx_hint, int_flag_success, contention_hint, weak, 2);
533   }
534   void cmpxchgw(ConditionRegister flag,
535                 Register dest_current_value, Register compare_value, Register exchange_value, Register addr_base,
536                 int semantics, bool cmpxchgx_hint = false,
537                 Register int_flag_success = noreg, bool contention_hint = false, bool weak = false) {
538     cmpxchg_generic(flag, dest_current_value, compare_value, exchange_value, addr_base, noreg, noreg,
539                     semantics, cmpxchgx_hint, int_flag_success, contention_hint, weak, 4);
540   }
541   void cmpxchgd(ConditionRegister flag,
542                 Register dest_current_value, RegisterOrConstant compare_value, Register exchange_value,
543                 Register addr_base, int semantics, bool cmpxchgx_hint = false,
544                 Register int_flag_success = noreg, Label* failed = nullptr, bool contention_hint = false, bool weak = false);
545 
546   // interface method calling
547   void lookup_interface_method(Register recv_klass,
548                                Register intf_klass,
549                                RegisterOrConstant itable_index,
550                                Register method_result,
551                                Register temp_reg, Register temp2_reg,
552                                Label& no_such_interface,
553                                bool return_method = true);
554 
555   // virtual method calling
556   void lookup_virtual_method(Register recv_klass,
557                              RegisterOrConstant vtable_index,
558                              Register method_result);
559 
560   // Test sub_klass against super_klass, with fast and slow paths.
561 
562   // The fast path produces a tri-state answer: yes / no / maybe-slow.
563   // One of the three labels can be null, meaning take the fall-through.
564   // If super_check_offset is -1, the value is loaded up from super_klass.
565   // No registers are killed, except temp_reg and temp2_reg.
566   // If super_check_offset is not -1, temp2_reg is not used and can be noreg.
567   void check_klass_subtype_fast_path(Register sub_klass,
568                                      Register super_klass,
569                                      Register temp1_reg,
570                                      Register temp2_reg,
571                                      Label* L_success,
572                                      Label* L_failure,
573                                      Label* L_slow_path = nullptr, // default fall through
574                                      RegisterOrConstant super_check_offset = RegisterOrConstant(-1));
575 
576   // The rest of the type check; must be wired to a corresponding fast path.
577   // It does not repeat the fast path logic, so don't use it standalone.
578   // The temp_reg can be noreg, if no temps are available.
579   // It can also be sub_klass or super_klass, meaning it's OK to kill that one.
580   // Updates the sub's secondary super cache as necessary.
581   void check_klass_subtype_slow_path(Register sub_klass,
582                                      Register super_klass,
583                                      Register temp1_reg,
584                                      Register temp2_reg,
585                                      Label* L_success = nullptr,
586                                      Register result_reg = noreg);
587 
588   // Simplified, combined version, good for typical uses.
589   // Falls through on failure.
590   void check_klass_subtype(Register sub_klass,
591                            Register super_klass,
592                            Register temp1_reg,
593                            Register temp2_reg,
594                            Label& L_success);
595 
596   void clinit_barrier(Register klass,
597                       Register thread,
598                       Label* L_fast_path = nullptr,
599                       Label* L_slow_path = nullptr);
600 
601   // Method handle support (JSR 292).
602   RegisterOrConstant argument_offset(RegisterOrConstant arg_slot, Register temp_reg, int extra_slot_offset = 0);
603 
604   void push_cont_fastpath();
605   void pop_cont_fastpath();
606   void inc_held_monitor_count(Register tmp);
607   void dec_held_monitor_count(Register tmp);
608   void atomically_flip_locked_state(bool is_unlock, Register obj, Register tmp, Label& failed, int semantics);
609   void lightweight_lock(Register obj, Register t1, Register t2, Label& slow);
610   void lightweight_unlock(Register obj, Register t1, Label& slow);
611 
612   // allocation (for C1)
613   void tlab_allocate(
614     Register obj,                      // result: pointer to object after successful allocation
615     Register var_size_in_bytes,        // object size in bytes if unknown at compile time; invalid otherwise
616     int      con_size_in_bytes,        // object size in bytes if   known at compile time
617     Register t1,                       // temp register
618     Label&   slow_case                 // continuation point if fast allocation fails
619   );
620   void incr_allocated_bytes(RegisterOrConstant size_in_bytes, Register t1, Register t2);
621 
622   enum { trampoline_stub_size = 6 * 4 };
623   address emit_trampoline_stub(int destination_toc_offset, int insts_call_instruction_offset, Register Rtoc = noreg);
624 
625   void compiler_fast_lock_object(ConditionRegister flag, Register oop, Register box,
626                                  Register tmp1, Register tmp2, Register tmp3);
627 
628   void compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
629                                    Register tmp1, Register tmp2, Register tmp3);
630 
631   void compiler_fast_lock_lightweight_object(ConditionRegister flag, Register oop, Register tmp1,
632                                              Register tmp2, Register tmp3);
633 
634   void compiler_fast_unlock_lightweight_object(ConditionRegister flag, Register oop, Register tmp1,
635                                                Register tmp2, Register tmp3);
636 
637   // Check if safepoint requested and if so branch
638   void safepoint_poll(Label& slow_path, Register temp, bool at_return, bool in_nmethod);
639 
640   void resolve_jobject(Register value, Register tmp1, Register tmp2,
641                        MacroAssembler::PreservationLevel preservation_level);
642   void resolve_global_jobject(Register value, Register tmp1, Register tmp2,
643                               MacroAssembler::PreservationLevel preservation_level);
644 
645   // Support for managing the JavaThread pointer (i.e.; the reference to
646   // thread-local information).
647 
648   // Support for last Java frame (but use call_VM instead where possible):
649   // access R16_thread->last_Java_sp.
650   void set_last_Java_frame(Register last_java_sp, Register last_Java_pc);
651   void reset_last_Java_frame(void);
652   void set_top_ijava_frame_at_SP_as_last_Java_frame(Register sp, Register tmp1);
653 
654   // Read vm result from thread: oop_result = R16_thread->result;
655   void get_vm_result  (Register oop_result);
656   void get_vm_result_2(Register metadata_result);
657 
658   static bool needs_explicit_null_check(intptr_t offset);
659   static bool uses_implicit_null_check(void* address);
660 
661   // Trap-instruction-based checks.
662   // Range checks can be distinguished from zero checks as they check 32 bit,
663   // zero checks all 64 bits (tw, td).
664   inline void trap_null_check(Register a, trap_to_bits cmp = traptoEqual);
665   static bool is_trap_null_check(int x) {
666     return is_tdi(x, traptoEqual,               -1/*any reg*/, 0) ||
667            is_tdi(x, traptoGreaterThanUnsigned, -1/*any reg*/, 0);
668   }
669 
670   inline void trap_ic_miss_check(Register a, Register b);
671   static bool is_trap_ic_miss_check(int x) {
672     return is_td(x, traptoGreaterThanUnsigned | traptoLessThanUnsigned, -1/*any reg*/, -1/*any reg*/);
673   }
674 
675   // Implicit or explicit null check, jumps to static address exception_entry.
676   inline void null_check_throw(Register a, int offset, Register temp_reg, address exception_entry);
677   inline void null_check(Register a, int offset, Label *Lis_null); // implicit only if Lis_null not provided
678 
679   // Access heap oop, handle encoding and GC barriers.
680   // Some GC barriers call C so use needs_frame = true if an extra frame is needed at the current call site.
681   inline void access_store_at(BasicType type, DecoratorSet decorators,
682                               Register base, RegisterOrConstant ind_or_offs, Register val,
683                               Register tmp1, Register tmp2, Register tmp3,
684                               MacroAssembler::PreservationLevel preservation_level);
685   inline void access_load_at(BasicType type, DecoratorSet decorators,
686                              Register base, RegisterOrConstant ind_or_offs, Register dst,
687                              Register tmp1, Register tmp2,
688                              MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null = nullptr);
689 
690  public:
691   // Specify tmp1 for better code in certain compressed oops cases. Specify Label to bail out on null oop.
692   // tmp1, tmp2 and needs_frame are used with decorators ON_PHANTOM_OOP_REF or ON_WEAK_OOP_REF.
693   inline void load_heap_oop(Register d, RegisterOrConstant offs, Register s1,
694                             Register tmp1, Register tmp2,
695                             MacroAssembler::PreservationLevel preservation_level,
696                             DecoratorSet decorators = 0, Label *L_handle_null = nullptr);
697 
698   inline void store_heap_oop(Register d, RegisterOrConstant offs, Register s1,
699                              Register tmp1, Register tmp2, Register tmp3,
700                              MacroAssembler::PreservationLevel preservation_level, DecoratorSet decorators = 0);
701 
702   // Encode/decode heap oop. Oop may not be null, else en/decoding goes wrong.
703   // src == d allowed.
704   inline Register encode_heap_oop_not_null(Register d, Register src = noreg);
705   inline Register decode_heap_oop_not_null(Register d, Register src = noreg);
706 
707   // Null allowed.
708   inline Register encode_heap_oop(Register d, Register src); // Prefer null check in GC barrier!
709   inline void decode_heap_oop(Register d);
710 
711   // Load/Store klass oop from klass field. Compress.
712   void load_klass(Register dst, Register src);
713   void load_klass_check_null(Register dst, Register src, Label* is_null = nullptr);
714   void store_klass(Register dst_oop, Register klass, Register tmp = R0);
715   void store_klass_gap(Register dst_oop, Register val = noreg); // Will store 0 if val not specified.
716 
717   void resolve_oop_handle(Register result, Register tmp1, Register tmp2,
718                           MacroAssembler::PreservationLevel preservation_level);
719   void resolve_weak_handle(Register result, Register tmp1, Register tmp2,
720                            MacroAssembler::PreservationLevel preservation_level);
721   void load_method_holder(Register holder, Register method);
722 
723   static int instr_size_for_decode_klass_not_null();
724   void decode_klass_not_null(Register dst, Register src = noreg);
725   Register encode_klass_not_null(Register dst, Register src = noreg);
726 
727   // SIGTRAP-based range checks for arrays.
728   inline void trap_range_check_l(Register a, Register b);
729   inline void trap_range_check_l(Register a, int si16);
730   static bool is_trap_range_check_l(int x) {
731     return (is_tw (x, traptoLessThanUnsigned, -1/*any reg*/, -1/*any reg*/) ||
732             is_twi(x, traptoLessThanUnsigned, -1/*any reg*/)                  );
733   }
734   inline void trap_range_check_le(Register a, int si16);
735   static bool is_trap_range_check_le(int x) {
736     return is_twi(x, traptoEqual | traptoLessThanUnsigned, -1/*any reg*/);
737   }
738   inline void trap_range_check_g(Register a, int si16);
739   static bool is_trap_range_check_g(int x) {
740     return is_twi(x, traptoGreaterThanUnsigned, -1/*any reg*/);
741   }
742   inline void trap_range_check_ge(Register a, Register b);
743   inline void trap_range_check_ge(Register a, int si16);
744   static bool is_trap_range_check_ge(int x) {
745     return (is_tw (x, traptoEqual | traptoGreaterThanUnsigned, -1/*any reg*/, -1/*any reg*/) ||
746             is_twi(x, traptoEqual | traptoGreaterThanUnsigned, -1/*any reg*/)                  );
747   }
748   static bool is_trap_range_check(int x) {
749     return is_trap_range_check_l(x) || is_trap_range_check_le(x) ||
750            is_trap_range_check_g(x) || is_trap_range_check_ge(x);
751   }
752 
753   void clear_memory_unrolled(Register base_ptr, int cnt_dwords, Register tmp = R0, int offset = 0);
754   void clear_memory_constlen(Register base_ptr, int cnt_dwords, Register tmp = R0);
755   void clear_memory_doubleword(Register base_ptr, Register cnt_dwords, Register tmp = R0, long const_cnt = -1);
756 
757   // Emitters for BigInteger.multiplyToLen intrinsic.
758   inline void multiply64(Register dest_hi, Register dest_lo,
759                          Register x, Register y);
760   void add2_with_carry(Register dest_hi, Register dest_lo,
761                        Register src1, Register src2);
762   void multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
763                              Register y, Register y_idx, Register z,
764                              Register carry, Register product_high, Register product,
765                              Register idx, Register kdx, Register tmp);
766   void multiply_add_128_x_128(Register x_xstart, Register y, Register z,
767                               Register yz_idx, Register idx, Register carry,
768                               Register product_high, Register product, Register tmp,
769                               int offset);
770   void multiply_128_x_128_loop(Register x_xstart,
771                                Register y, Register z,
772                                Register yz_idx, Register idx, Register carry,
773                                Register product_high, Register product,
774                                Register carry2, Register tmp);
775   void muladd(Register out, Register in, Register offset, Register len, Register k,
776               Register tmp1, Register tmp2, Register carry);
777   void multiply_to_len(Register x, Register xlen,
778                        Register y, Register ylen,
779                        Register z, Register zlen,
780                        Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5,
781                        Register tmp6, Register tmp7, Register tmp8, Register tmp9, Register tmp10,
782                        Register tmp11, Register tmp12, Register tmp13);
783 
784   // Emitters for CRC32 calculation.
785   // A note on invertCRC:
786   //   Unfortunately, internal representation of crc differs between CRC32 and CRC32C.
787   //   CRC32 holds it's current crc value in the externally visible representation.
788   //   CRC32C holds it's current crc value in internal format, ready for updating.
789   //   Thus, the crc value must be bit-flipped before updating it in the CRC32 case.
790   //   In the CRC32C case, it must be bit-flipped when it is given to the outside world (getValue()).
791   //   The bool invertCRC parameter indicates whether bit-flipping is required before updates.
792   void load_reverse_32(Register dst, Register src);
793   int  crc32_table_columns(Register table, Register tc0, Register tc1, Register tc2, Register tc3);
794   void fold_byte_crc32(Register crc, Register val, Register table, Register tmp);
795   void update_byte_crc32(Register crc, Register val, Register table);
796   void update_byteLoop_crc32(Register crc, Register buf, Register len, Register table,
797                              Register data, bool loopAlignment);
798   void update_1word_crc32(Register crc, Register buf, Register table, int bufDisp, int bufInc,
799                           Register t0,  Register t1,  Register t2,  Register t3,
800                           Register tc0, Register tc1, Register tc2, Register tc3);
801   void kernel_crc32_1word(Register crc, Register buf, Register len, Register table,
802                           Register t0,  Register t1,  Register t2,  Register t3,
803                           Register tc0, Register tc1, Register tc2, Register tc3,
804                           bool invertCRC);
805   void kernel_crc32_vpmsum(Register crc, Register buf, Register len, Register constants,
806                            Register t0, Register t1, Register t2, Register t3, Register t4,
807                            Register t5, Register t6, bool invertCRC);
808   void kernel_crc32_vpmsum_aligned(Register crc, Register buf, Register len, Register constants,
809                                    Register t0, Register t1, Register t2, Register t3, Register t4,
810                                    Register t5, Register t6);
811   // Version which internally decides what to use.
812   void crc32(Register crc, Register buf, Register len, Register t0, Register t1, Register t2,
813              Register t3, Register t4, Register t5, Register t6, Register t7, bool is_crc32c);
814 
815   void kernel_crc32_singleByteReg(Register crc, Register val, Register table,
816                                   bool invertCRC);
817 
818   // SHA-2 auxiliary functions and public interfaces
819  private:
820   void sha256_deque(const VectorRegister src,
821       const VectorRegister dst1, const VectorRegister dst2, const VectorRegister dst3);
822   void sha256_load_h_vec(const VectorRegister a, const VectorRegister e, const Register hptr);
823   void sha256_round(const VectorRegister* hs, const int total_hs, int& h_cnt, const VectorRegister kpw);
824   void sha256_load_w_plus_k_vec(const Register buf_in, const VectorRegister* ws,
825       const int total_ws, const Register k, const VectorRegister* kpws,
826       const int total_kpws);
827   void sha256_calc_4w(const VectorRegister w0, const VectorRegister w1,
828       const VectorRegister w2, const VectorRegister w3, const VectorRegister kpw0,
829       const VectorRegister kpw1, const VectorRegister kpw2, const VectorRegister kpw3,
830       const Register j, const Register k);
831   void sha256_update_sha_state(const VectorRegister a, const VectorRegister b,
832       const VectorRegister c, const VectorRegister d, const VectorRegister e,
833       const VectorRegister f, const VectorRegister g, const VectorRegister h,
834       const Register hptr);
835 
836   void sha512_load_w_vec(const Register buf_in, const VectorRegister* ws, const int total_ws);
837   void sha512_update_sha_state(const Register state, const VectorRegister* hs, const int total_hs);
838   void sha512_round(const VectorRegister* hs, const int total_hs, int& h_cnt, const VectorRegister kpw);
839   void sha512_load_h_vec(const Register state, const VectorRegister* hs, const int total_hs);
840   void sha512_calc_2w(const VectorRegister w0, const VectorRegister w1,
841       const VectorRegister w2, const VectorRegister w3,
842       const VectorRegister w4, const VectorRegister w5,
843       const VectorRegister w6, const VectorRegister w7,
844       const VectorRegister kpw0, const VectorRegister kpw1, const Register j,
845       const VectorRegister vRb, const Register k);
846 
847  public:
848   void sha256(bool multi_block);
849   void sha512(bool multi_block);
850 
851   void cache_wb(Address line);
852   void cache_wbsync(bool is_presync);
853 
854   //
855   // Debugging
856   //
857 
858   // assert on cr0
859   void asm_assert(bool check_equal, const char* msg);
860   void asm_assert_eq(const char* msg) { asm_assert(true, msg); }
861   void asm_assert_ne(const char* msg) { asm_assert(false, msg); }
862 
863  private:
864   void asm_assert_mems_zero(bool check_equal, int size, int mem_offset, Register mem_base,
865                             const char* msg);
866 
867  public:
868 
869   void asm_assert_mem8_is_zero(int mem_offset, Register mem_base, const char* msg) {
870     asm_assert_mems_zero(true,  8, mem_offset, mem_base, msg);
871   }
872   void asm_assert_mem8_isnot_zero(int mem_offset, Register mem_base, const char* msg) {
873     asm_assert_mems_zero(false, 8, mem_offset, mem_base, msg);
874   }
875 
876   // Calls verify_oop. If UseCompressedOops is on, decodes the oop.
877   // Preserves reg.
878   void verify_coop(Register reg, const char*);
879   // Emit code to verify that reg contains a valid oop if +VerifyOops is set.
880   void verify_oop(Register reg, const char* s = "broken oop");
881   void verify_oop_addr(RegisterOrConstant offs, Register base, const char* s = "contains broken oop");
882 
883   // TODO: verify method and klass metadata (compare against vptr?)
884   void _verify_method_ptr(Register reg, const char * msg, const char * file, int line) {}
885   void _verify_klass_ptr(Register reg, const char * msg, const char * file, int line) {}
886 
887   // Convenience method returning function entry. For the ELFv1 case
888   // creates function descriptor at the current address and returns
889   // the pointer to it. For the ELFv2 case returns the current address.
890   inline address function_entry();
891 
892 #define verify_method_ptr(reg) _verify_method_ptr(reg, "broken method " #reg, __FILE__, __LINE__)
893 #define verify_klass_ptr(reg) _verify_klass_ptr(reg, "broken klass " #reg, __FILE__, __LINE__)
894 
895  private:
896   void stop(int type, const char* msg);
897 
898  public:
899   enum {
900     stop_stop               = 0,
901     stop_untested           = 1,
902     stop_unimplemented      = 2,
903     stop_shouldnotreachhere = 3,
904     stop_msg_present        = -0x8000
905   };
906 
907   // Prints msg, dumps registers and stops execution.
908   void stop                 (const char* msg = nullptr) { stop(stop_stop,               msg); }
909   void untested             (const char* msg = nullptr) { stop(stop_untested,           msg); }
910   void unimplemented        (const char* msg = nullptr) { stop(stop_unimplemented,      msg); }
911   void should_not_reach_here(const char* msg = nullptr) { stop(stop_shouldnotreachhere, msg); }
912 
913   void zap_from_to(Register low, int before, Register high, int after, Register val, Register addr) PRODUCT_RETURN;
914 };
915 
916 // class SkipIfEqualZero:
917 //
918 // Instantiating this class will result in assembly code being output that will
919 // jump around any code emitted between the creation of the instance and it's
920 // automatic destruction at the end of a scope block, depending on the value of
921 // the flag passed to the constructor, which will be checked at run-time.
922 class SkipIfEqualZero : public StackObj {
923  private:
924   MacroAssembler* _masm;
925   Label _label;
926 
927  public:
928    // 'Temp' is a temp register that this object can use (and trash).
929    explicit SkipIfEqualZero(MacroAssembler*, Register temp, const bool* flag_addr);
930    static void skip_to_label_if_equal_zero(MacroAssembler*, Register temp,
931                                            const bool* flag_addr, Label& label);
932    ~SkipIfEqualZero();
933 };
934 
935 #endif // CPU_PPC_MACROASSEMBLER_PPC_HPP