1 /*
  2  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2014, 2020 Red Hat Inc. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.
  9  *
 10  * This code is distributed in the hope that it will be useful, but WITHOUT
 11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 13  * version 2 for more details (a copy is included in the LICENSE file that
 14  * accompanied this code).
 15  *
 16  * You should have received a copy of the GNU General Public License version
 17  * 2 along with this work; if not, write to the Free Software Foundation,
 18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 19  *
 20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 21  * or visit www.oracle.com if you need additional information or have any
 22  * questions.
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "asm/assembler.hpp"
 27 #include "asm/assembler.inline.hpp"
 28 #include "asm/macroAssembler.hpp"
 29 #include "compiler/disassembler.hpp"
 30 #include "immediate_aarch64.hpp"
 31 #include "memory/resourceArea.hpp"
 32 #include "metaprogramming/primitiveConversions.hpp"
 33 
 34 #ifndef PRODUCT
 35 const uintptr_t Assembler::asm_bp = 0x0000ffffac221240;
 36 #endif
 37 
 38 static float unpack(unsigned value);
 39 
 40 short Assembler::SIMD_Size_in_bytes[] = {
 41   // T8B, T16B, T4H, T8H, T2S, T4S, T1D, T2D, T1Q
 42        8,   16,   8,  16,   8,  16,   8,  16,  16
 43 };
 44 
 45 Assembler::SIMD_Arrangement Assembler::_esize2arrangement_table[9][2] = {
 46   // esize        isQ:false             isQ:true
 47   /*   0  */      {INVALID_ARRANGEMENT, INVALID_ARRANGEMENT},
 48   /*   1  */      {T8B,                 T16B},
 49   /*   2  */      {T4H,                 T8H},
 50   /*   3  */      {INVALID_ARRANGEMENT, INVALID_ARRANGEMENT},
 51   /*   4  */      {T2S,                 T4S},
 52   /*   5  */      {INVALID_ARRANGEMENT, INVALID_ARRANGEMENT},
 53   /*   6  */      {INVALID_ARRANGEMENT, INVALID_ARRANGEMENT},
 54   /*   7  */      {INVALID_ARRANGEMENT, INVALID_ARRANGEMENT},
 55   /*   8  */      {T1D,                 T2D}
 56   };
 57 
 58 Assembler::SIMD_RegVariant Assembler::_esize2regvariant[9] = {
 59   INVALID,
 60   B,
 61   H,
 62   INVALID,
 63   S,
 64   INVALID,
 65   INVALID,
 66   INVALID,
 67   D,
 68 };
 69 
 70 Assembler::SIMD_Arrangement Assembler::esize2arrangement(unsigned esize, bool isQ) {
 71   guarantee(esize < ARRAY_SIZE(_esize2arrangement_table) &&
 72          _esize2arrangement_table[esize][isQ] != INVALID_ARRANGEMENT, "unsupported element size");
 73   return _esize2arrangement_table[esize][isQ];
 74 }
 75 
 76 Assembler::SIMD_RegVariant Assembler::elemBytes_to_regVariant(unsigned esize) {
 77   guarantee(esize < ARRAY_SIZE(_esize2regvariant) && _esize2regvariant[esize] != INVALID,
 78          "unsupported element size");
 79   return _esize2regvariant[esize];
 80 }
 81 
 82 Assembler::SIMD_RegVariant Assembler::elemType_to_regVariant(BasicType bt) {
 83   return elemBytes_to_regVariant(type2aelembytes(bt));
 84 }
 85 
 86 unsigned Assembler::regVariant_to_elemBits(Assembler::SIMD_RegVariant T){
 87   guarantee(T != Q, "Invalid register variant");
 88   return 1 << (T + 3);
 89 }
 90 
 91 void Assembler::emit_data64(jlong data,
 92                             relocInfo::relocType rtype,
 93                             int format) {
 94   if (rtype == relocInfo::none) {
 95     emit_int64(data);
 96   } else {
 97     emit_data64(data, Relocation::spec_simple(rtype), format);
 98   }
 99 }
100 
101 void Assembler::emit_data64(jlong data,
102                             RelocationHolder const& rspec,
103                             int format) {
104 
105   assert(inst_mark() != nullptr, "must be inside InstructionMark");
106   // Do not use AbstractAssembler::relocate, which is not intended for
107   // embedded words.  Instead, relocate to the enclosing instruction.
108   code_section()->relocate(inst_mark(), rspec, format);
109   emit_int64(data);
110 }
111 
112 extern "C" {
113   void das(uint64_t start, int len) {
114     ResourceMark rm;
115     len <<= 2;
116     if (len < 0)
117       Disassembler::decode((address)start + len, (address)start);
118     else
119       Disassembler::decode((address)start, (address)start + len);
120   }
121 
122   JNIEXPORT void das1(uintptr_t insn) {
123     das(insn, 1);
124   }
125 }
126 
127 #define __ as->
128 
129 void Address::lea(MacroAssembler *as, Register r) const {
130   switch(_mode) {
131   case base_plus_offset: {
132     if (offset() == 0 && base() == r) // it's a nop
133       break;
134     if (offset() > 0)
135       __ add(r, base(), offset());
136     else
137       __ sub(r, base(), -offset());
138     break;
139   }
140   case base_plus_offset_reg: {
141     __ add(r, base(), index(), ext().op(), MAX2(ext().shift(), 0));
142     break;
143   }
144   case literal: {
145     as->code_section()->relocate(as->inst_mark(), rspec());
146     if (rspec().type() == relocInfo::none)
147       __ mov(r, target());
148     else
149       __ movptr(r, (uint64_t)target());
150     break;
151   }
152   default:
153     ShouldNotReachHere();
154   }
155 }
156 
157 #undef __
158 
159 #define starti Instruction_aarch64 current_insn(this);
160 
161 #define f current_insn.f
162 #define sf current_insn.sf
163 #define rf current_insn.rf
164 #define srf current_insn.srf
165 #define zrf current_insn.zrf
166 #define prf current_insn.prf
167 #define pgrf current_insn.pgrf
168 #define fixed current_insn.fixed
169 
170   void Assembler::adr(Register Rd, address adr) {
171     intptr_t offset = adr - pc();
172     int offset_lo = offset & 3;
173     offset >>= 2;
174     starti;
175     f(0, 31), f(offset_lo, 30, 29), f(0b10000, 28, 24), sf(offset, 23, 5);
176     rf(Rd, 0);
177   }
178 
179   void Assembler::_adrp(Register Rd, address adr) {
180     uint64_t pc_page = (uint64_t)pc() >> 12;
181     uint64_t adr_page = (uint64_t)adr >> 12;
182     intptr_t offset = adr_page - pc_page;
183     int offset_lo = offset & 3;
184     offset >>= 2;
185     starti;
186     f(1, 31), f(offset_lo, 30, 29), f(0b10000, 28, 24), sf(offset, 23, 5);
187     zrf(Rd, 0);
188   }
189 
190 // An "all-purpose" add/subtract immediate, per ARM documentation:
191 // A "programmer-friendly" assembler may accept a negative immediate
192 // between -(2^24 -1) and -1 inclusive, causing it to convert a
193 // requested ADD operation to a SUB, or vice versa, and then encode
194 // the absolute value of the immediate as for uimm24.
195 void Assembler::add_sub_immediate(Instruction_aarch64 &current_insn,
196                                   Register Rd, Register Rn, unsigned uimm, int op,
197                                   int negated_op) {
198   bool sets_flags = op & 1;   // this op sets flags
199   union {
200     unsigned u;
201     int imm;
202   };
203   u = uimm;
204   bool shift = false;
205   bool neg = imm < 0;
206   if (neg) {
207     imm = -imm;
208     op = negated_op;
209   }
210   assert(Rd != sp || imm % 16 == 0, "misaligned stack");
211   if (imm >= (1 << 11)
212       && ((imm >> 12) << 12 == imm)) {
213     imm >>= 12;
214     shift = true;
215   }
216   f(op, 31, 29), f(0b10001, 28, 24), f(shift, 23, 22), f(imm, 21, 10);
217 
218   // add/subtract immediate ops with the S bit set treat r31 as zr;
219   // with S unset they use sp.
220   if (sets_flags)
221     zrf(Rd, 0);
222   else
223     srf(Rd, 0);
224 
225   srf(Rn, 5);
226 }
227 
228 #undef f
229 #undef sf
230 #undef rf
231 #undef srf
232 #undef zrf
233 #undef prf
234 #undef pgrf
235 #undef fixed
236 
237 #undef starti
238 
239 #ifdef ASSERT
240 
241 void Address::assert_is_literal() const {
242   assert(_mode == literal, "addressing mode is non-literal: %d", _mode);
243 }
244 
245 void Address::assert_is_nonliteral() const {
246   assert(_mode != literal, "unexpected literal addressing mode");
247   assert(_mode != no_mode, "unexpected no_mode addressing mode");
248 }
249 
250 #endif // ASSERT
251 
252 static RelocationHolder address_relocation(address target, relocInfo::relocType rtype) {
253   switch (rtype) {
254   case relocInfo::oop_type:
255   case relocInfo::metadata_type:
256     // Oops are a special case. Normally they would be their own section
257     // but in cases like icBuffer they are literals in the code stream that
258     // we don't have a section for. We use none so that we get a literal address
259     // which is always patchable.
260     return RelocationHolder::none;
261   case relocInfo::external_word_type:
262     return external_word_Relocation::spec(target);
263   case relocInfo::internal_word_type:
264     return internal_word_Relocation::spec(target);
265   case relocInfo::opt_virtual_call_type:
266     return opt_virtual_call_Relocation::spec();
267   case relocInfo::static_call_type:
268     return static_call_Relocation::spec();
269   case relocInfo::runtime_call_type:
270     return runtime_call_Relocation::spec();
271   case relocInfo::poll_type:
272   case relocInfo::poll_return_type:
273     return Relocation::spec_simple(rtype);
274   case relocInfo::none:
275     return RelocationHolder::none;
276   default:
277     ShouldNotReachHere();
278     return RelocationHolder::none;
279   }
280 }
281 
282 Address::Address(address target, relocInfo::relocType rtype) :
283   _mode(literal),
284   _literal(target, address_relocation(target, rtype))
285 {}
286 
287 void Assembler::b(const Address &dest) {
288   code_section()->relocate(pc(), dest.rspec());
289   b(dest.target());
290 }
291 
292 void Assembler::bl(const Address &dest) {
293   code_section()->relocate(pc(), dest.rspec());
294   bl(dest.target());
295 }
296 
297 void Assembler::adr(Register r, const Address &dest) {
298   code_section()->relocate(pc(), dest.rspec());
299   adr(r, dest.target());
300 }
301 
302 void Assembler::br(Condition cc, Label &L) {
303   if (L.is_bound()) {
304     br(cc, target(L));
305   } else {
306     L.add_patch_at(code(), locator());
307     br(cc, pc());
308   }
309 }
310 
311 void Assembler::wrap_label(Label &L,
312                                  Assembler::uncond_branch_insn insn) {
313   if (L.is_bound()) {
314     (this->*insn)(target(L));
315   } else {
316     L.add_patch_at(code(), locator());
317     (this->*insn)(pc());
318   }
319 }
320 
321 void Assembler::wrap_label(Register r, Label &L,
322                                  compare_and_branch_insn insn) {
323   if (L.is_bound()) {
324     (this->*insn)(r, target(L));
325   } else {
326     L.add_patch_at(code(), locator());
327     (this->*insn)(r, pc());
328   }
329 }
330 
331 void Assembler::wrap_label(Register r, int bitpos, Label &L,
332                                  test_and_branch_insn insn) {
333   if (L.is_bound()) {
334     (this->*insn)(r, bitpos, target(L));
335   } else {
336     L.add_patch_at(code(), locator());
337     (this->*insn)(r, bitpos, pc());
338   }
339 }
340 
341 void Assembler::wrap_label(Label &L, prfop op, prefetch_insn insn) {
342   if (L.is_bound()) {
343     (this->*insn)(target(L), op);
344   } else {
345     L.add_patch_at(code(), locator());
346     (this->*insn)(pc(), op);
347   }
348 }
349 
350 bool Assembler::operand_valid_for_add_sub_immediate(int64_t imm) {
351   return operand_valid_for_immediate_bits(imm, 12);
352 }
353 
354 bool Assembler::operand_valid_for_sve_add_sub_immediate(int64_t imm) {
355   return operand_valid_for_immediate_bits(imm, 8);
356 }
357 
358 bool Assembler::operand_valid_for_logical_immediate(bool is32, uint64_t imm) {
359   return encode_logical_immediate(is32, imm) != 0xffffffff;
360 }
361 
362 // Check immediate encoding for movi.
363 // Return the shift amount which can be {0, 8, 16, 24} for B/H/S types. As the D type
364 // movi does not have shift variant, in this case the return value is the immediate
365 // after encoding.
366 // Return -1 if the input imm64 can not be encoded.
367 int Assembler::operand_valid_for_movi_immediate(uint64_t imm64, SIMD_Arrangement T) {
368   if (T == T1D || T == T2D) {
369      // To encode into movi, the 64-bit imm must be in the form of
370      // 'aaaaaaaabbbbbbbbccccccccddddddddeeeeeeeeffffffffgggggggghhhhhhhh'
371      // and encoded in "a:b:c:d:e:f:g:h".
372      uint64_t tmp = imm64;
373      uint64_t one_byte = 0;
374      for (int i = 0; i < 8; i++) {
375        one_byte = tmp & 0xffULL;
376        if (one_byte != 0xffULL && one_byte != 0) {
377          return -1; // can not be encoded
378        }
379        tmp = tmp >> 8;
380      }
381 
382      imm64 &= 0x0101010101010101ULL;
383      imm64 |= (imm64 >> 7);
384      imm64 |= (imm64 >> 14);
385      imm64 |= (imm64 >> 28);
386 
387      return imm64 & 0xff;
388   }
389 
390   uint32_t imm32 = imm64 & 0xffffffffULL;
391   if (T == T8B || T == T16B) {       // 8-bit variant
392     if (0 == (imm32 & ~0xff))        return 0;
393   } else if(T == T4H || T == T8H) {  // 16-bit variant
394     if (0 == (imm32 & ~0xff))        return 0;
395     if (0 == (imm32 & ~0xff00))      return 8;
396   } else if (T == T2S || T == T4S) { // 32-bit variant
397     if (0 == (imm32 & ~0xff))        return 0;
398     if (0 == (imm32 & ~0xff00))      return 8;
399     if (0 == (imm32 & ~0xff0000))    return 16;
400     if (0 == (imm32 & ~0xff000000))  return 24;
401   } else {
402     assert(false, "unsupported");
403     ShouldNotReachHere();
404   }
405 
406   return -1;
407 }
408 
409 bool Assembler::operand_valid_for_sve_logical_immediate(unsigned elembits, uint64_t imm) {
410   return encode_sve_logical_immediate(elembits, imm) != 0xffffffff;
411 }
412 
413 static uint64_t doubleTo64Bits(jdouble d) {
414   union {
415     jdouble double_value;
416     uint64_t double_bits;
417   };
418 
419   double_value = d;
420   return double_bits;
421 }
422 
423 bool Assembler::operand_valid_for_float_immediate(double imm) {
424   // If imm is all zero bits we can use ZR as the source of a
425   // floating-point value.
426   if (doubleTo64Bits(imm) == 0)
427     return true;
428 
429   // Otherwise try to encode imm then convert the encoded value back
430   // and make sure it's the exact same bit pattern.
431   unsigned result = encoding_for_fp_immediate(imm);
432   return doubleTo64Bits(imm) == fp_immediate_for_encoding(result, true);
433 }
434 
435 int AbstractAssembler::code_fill_byte() {
436   return 0;
437 }
438 
439 // n.b. this is implemented in subclass MacroAssembler
440 void Assembler::bang_stack_with_offset(int offset) { Unimplemented(); }
441 
442 bool asm_util::operand_valid_for_immediate_bits(int64_t imm, unsigned nbits) {
443   guarantee(nbits == 8 || nbits == 12, "invalid nbits value");
444   uint64_t uimm = (uint64_t)uabs((jlong)imm);
445   if (uimm < (UCONST64(1) << nbits))
446     return true;
447   if (uimm < (UCONST64(1) << (2 * nbits))
448       && ((uimm >> nbits) << nbits == uimm)) {
449     return true;
450   }
451   return false;
452 }
453 
454 // and now the routines called by the assembler which encapsulate the
455 // above encode and decode functions
456 
457 uint32_t
458 asm_util::encode_logical_immediate(bool is32, uint64_t imm)
459 {
460   if (is32) {
461     /* Allow all zeros or all ones in top 32-bits, so that
462        constant expressions like ~1 are permitted. */
463     if (imm >> 32 != 0 && imm >> 32 != 0xffffffff)
464       return 0xffffffff;
465     /* Replicate the 32 lower bits to the 32 upper bits.  */
466     imm &= 0xffffffff;
467     imm |= imm << 32;
468   }
469 
470   return encoding_for_logical_immediate(imm);
471 }
472 
473 uint32_t
474 asm_util::encode_sve_logical_immediate(unsigned elembits, uint64_t imm) {
475   guarantee(elembits == 8 || elembits == 16 ||
476             elembits == 32 || elembits == 64, "unsupported element size");
477   uint64_t upper = UCONST64(-1) << (elembits/2) << (elembits/2);
478   /* Allow all zeros or all ones in top bits, so that
479    * constant expressions like ~1 are permitted. */
480   if ((imm & ~upper) != imm && (imm | upper) != imm)
481     return 0xffffffff;
482 
483   // Replicate the immediate in different element sizes to 64 bits.
484   imm &= ~upper;
485   for (unsigned i = elembits; i < 64; i *= 2) {
486     imm |= (imm << i);
487   }
488 
489   return encoding_for_logical_immediate(imm);
490 }
491 
492 unsigned Assembler::pack(double value) {
493   float val = (float)value;
494   unsigned result = encoding_for_fp_immediate(val);
495   guarantee(unpack(result) == value,
496             "Invalid floating-point immediate operand");
497   return result;
498 }
499 
500 // Packed operands for  Floating-point Move (immediate)
501 
502 static float unpack(unsigned value) {
503   unsigned ival = fp_immediate_for_encoding(value, 0);
504   return PrimitiveConversions::cast<float>(ival);
505 }
506 
507 address Assembler::locate_next_instruction(address inst) {
508   return inst + Assembler::instruction_size;
509 }