1 /*
  2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2019, 2021, Arm Limited. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.  Oracle designates this
  9  * particular file as subject to the "Classpath" exception as provided
 10  * by Oracle in the LICENSE file that accompanied this code.
 11  *
 12  * This code is distributed in the hope that it will be useful, but WITHOUT
 13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 15  * version 2 for more details (a copy is included in the LICENSE file that
 16  * accompanied this code).
 17  *
 18  * You should have received a copy of the GNU General Public License version
 19  * 2 along with this work; if not, write to the Free Software Foundation,
 20  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 21  *
 22  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 23  * or visit www.oracle.com if you need additional information or have any
 24  * questions.
 25  */
 26 package jdk.internal.foreign.abi.aarch64;
 27 
 28 import jdk.incubator.foreign.FunctionDescriptor;
 29 import jdk.incubator.foreign.GroupLayout;
 30 import jdk.incubator.foreign.MemoryAddress;
 31 import jdk.incubator.foreign.MemoryLayout;
 32 import jdk.incubator.foreign.MemorySegment;
 33 import jdk.internal.foreign.Utils;
 34 import jdk.internal.foreign.abi.CallingSequenceBuilder;
 35 import jdk.internal.foreign.abi.UpcallHandler;
 36 import jdk.internal.foreign.abi.ABIDescriptor;
 37 import jdk.internal.foreign.abi.Binding;
 38 import jdk.internal.foreign.abi.CallingSequence;
 39 import jdk.internal.foreign.abi.ProgrammableInvoker;
 40 import jdk.internal.foreign.abi.ProgrammableUpcallHandler;
 41 import jdk.internal.foreign.abi.VMStorage;
 42 import jdk.internal.foreign.abi.SharedUtils;
 43 
 44 import java.lang.invoke.MethodHandle;
 45 import java.lang.invoke.MethodType;
 46 import java.util.List;
 47 import java.util.Optional;
 48 
 49 import static jdk.internal.foreign.PlatformLayouts.*;
 50 import static jdk.internal.foreign.abi.aarch64.AArch64Architecture.*;
 51 
 52 /**
 53  * For the AArch64 C ABI specifically, this class uses the ProgrammableInvoker API, namely CallingSequenceBuilder2
 54  * to translate a C FunctionDescriptor into a CallingSequence2, which can then be turned into a MethodHandle.
 55  *
 56  * This includes taking care of synthetic arguments like pointers to return buffers for 'in-memory' returns.
 57  */
 58 public class CallArranger {
 59     private static final int STACK_SLOT_SIZE = 8;
 60     public static final int MAX_REGISTER_ARGUMENTS = 8;
 61 
 62     private static final VMStorage INDIRECT_RESULT = r8;
 63 
 64     // This is derived from the AAPCS64 spec, restricted to what's
 65     // possible when calling to/from C code.
 66     //
 67     // The indirect result register, r8, is used to return a large
 68     // struct by value. It's treated as an input here as the caller is
 69     // responsible for allocating storage and passing this into the
 70     // function.
 71     //
 72     // Although the AAPCS64 says r0-7 and v0-7 are all valid return
 73     // registers, it's not possible to generate a C function that uses
 74     // r2-7 and v4-7 so they are omitted here.
 75     private static final ABIDescriptor C = AArch64Architecture.abiFor(
 76         new VMStorage[] { r0, r1, r2, r3, r4, r5, r6, r7, INDIRECT_RESULT},
 77         new VMStorage[] { v0, v1, v2, v3, v4, v5, v6, v7 },
 78         new VMStorage[] { r0, r1 },
 79         new VMStorage[] { v0, v1, v2, v3 },
 80         new VMStorage[] { r9, r10, r11, r12, r13, r14, r15 },
 81         new VMStorage[] { v16, v17, v18, v19, v20, v21, v22, v23, v25,
 82                           v26, v27, v28, v29, v30, v31 },
 83         16,  // Stack is always 16 byte aligned on AArch64
 84         0    // No shadow space
 85     );
 86 
 87     // record
 88     public static class Bindings {
 89         public final CallingSequence callingSequence;
 90         public final boolean isInMemoryReturn;
 91 
 92         Bindings(CallingSequence callingSequence, boolean isInMemoryReturn) {
 93             this.callingSequence = callingSequence;
 94             this.isInMemoryReturn = isInMemoryReturn;
 95         }
 96     }
 97 
 98     public static Bindings getBindings(MethodType mt, FunctionDescriptor cDesc, boolean forUpcall) {
 99         SharedUtils.checkFunctionTypes(mt, cDesc, AArch64.C_POINTER.bitSize());
100 
101         CallingSequenceBuilder csb = new CallingSequenceBuilder(forUpcall);
102 
103         BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true);
104         BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false) : new BoxBindingCalculator(false);
105 
106         boolean returnInMemory = isInMemoryReturn(cDesc.returnLayout());
107         if (returnInMemory) {
108             csb.addArgumentBindings(MemoryAddress.class, AArch64.C_POINTER,
109                     argCalc.getIndirectBindings());
110         } else if (cDesc.returnLayout().isPresent()) {
111             Class<?> carrier = mt.returnType();
112             MemoryLayout layout = cDesc.returnLayout().get();
113             csb.setReturnBindings(carrier, layout, retCalc.getBindings(carrier, layout));
114         }
115 
116         for (int i = 0; i < mt.parameterCount(); i++) {
117             Class<?> carrier = mt.parameterType(i);
118             MemoryLayout layout = cDesc.argumentLayouts().get(i);
119             csb.addArgumentBindings(carrier, layout, argCalc.getBindings(carrier, layout));
120         }
121 
122         csb.setTrivial(SharedUtils.isTrivial(cDesc));
123 
124         return new Bindings(csb.build(), returnInMemory);
125     }
126 
127     public static MethodHandle arrangeDowncall(MethodType mt, FunctionDescriptor cDesc) {
128         Bindings bindings = getBindings(mt, cDesc, false);
129 
130         MethodHandle handle = new ProgrammableInvoker(C, bindings.callingSequence).getBoundMethodHandle();
131 
132         if (bindings.isInMemoryReturn) {
133             handle = SharedUtils.adaptDowncallForIMR(handle, cDesc);
134         }
135 
136         return handle;
137     }
138 
139     public static UpcallHandler arrangeUpcall(MethodHandle target, MethodType mt, FunctionDescriptor cDesc) {
140         Bindings bindings = getBindings(mt, cDesc, true);
141 
142         if (bindings.isInMemoryReturn) {
143             target = SharedUtils.adaptUpcallForIMR(target, true /* drop return, since we don't have bindings for it */);
144         }
145 
146         return ProgrammableUpcallHandler.make(C, target, bindings.callingSequence);
147     }
148 
149     private static boolean isInMemoryReturn(Optional<MemoryLayout> returnLayout) {
150         return returnLayout
151             .filter(GroupLayout.class::isInstance)
152             .filter(g -> TypeClass.classifyLayout(g) == TypeClass.STRUCT_REFERENCE)
153             .isPresent();
154     }
155 
156     static class StorageCalculator {
157         private final boolean forArguments;
158 
159         private final int[] nRegs = new int[] { 0, 0 };
160         private long stackOffset = 0;
161 
162         public StorageCalculator(boolean forArguments) {
163             this.forArguments = forArguments;
164         }
165 
166         VMStorage stackAlloc(long size, long alignment) {
167             assert forArguments : "no stack returns";
168             alignment = Math.max(alignment, STACK_SLOT_SIZE);
169             stackOffset = Utils.alignUp(stackOffset, alignment);
170 
171             VMStorage storage =
172                 AArch64Architecture.stackStorage((int)(stackOffset / STACK_SLOT_SIZE));
173             stackOffset += size;
174             return storage;
175         }
176 
177         VMStorage stackAlloc(MemoryLayout layout) {
178             return stackAlloc(layout.byteSize(), SharedUtils.alignment(layout, true));
179         }
180 
181         VMStorage[] regAlloc(int type, int count) {
182             if (nRegs[type] + count <= MAX_REGISTER_ARGUMENTS) {
183                 VMStorage[] source =
184                     (forArguments ? C.inputStorage : C.outputStorage)[type];
185                 VMStorage[] result = new VMStorage[count];
186                 for (int i = 0; i < count; i++) {
187                     result[i] = source[nRegs[type]++];
188                 }
189                 return result;
190             } else {
191                 // Any further allocations for this register type must
192                 // be from the stack.
193                 nRegs[type] = MAX_REGISTER_ARGUMENTS;
194                 return null;
195             }
196         }
197 
198         VMStorage[] regAlloc(int type, MemoryLayout layout) {
199             return regAlloc(type, (int)Utils.alignUp(layout.byteSize(), 8) / 8);
200         }
201 
202         VMStorage nextStorage(int type, MemoryLayout layout) {
203             VMStorage[] storage = regAlloc(type, 1);
204             if (storage == null) {
205                 return stackAlloc(layout);
206             }
207 
208             return storage[0];
209         }
210 
211         void adjustForVarArgs(MemoryLayout layout) {
212             if (layout.attribute(AArch64.STACK_VARARGS_ATTRIBUTE_NAME)
213                     .map(Boolean.class::cast).orElse(false)) {
214                 // This system passes all variadic parameters on the stack. Ensure
215                 // no further arguments are allocated to registers.
216                 nRegs[StorageClasses.INTEGER] = MAX_REGISTER_ARGUMENTS;
217                 nRegs[StorageClasses.VECTOR] = MAX_REGISTER_ARGUMENTS;
218             }
219         }
220     }
221 
222     static abstract class BindingCalculator {
223         protected final StorageCalculator storageCalculator;
224 
225         protected BindingCalculator(boolean forArguments) {
226             this.storageCalculator = new StorageCalculator(forArguments);
227         }
228 
229         protected void spillStructUnbox(Binding.Builder bindings, MemoryLayout layout) {
230             // If a struct has been assigned register or HFA class but
231             // there are not enough free registers to hold the entire
232             // struct, it must be passed on the stack. I.e. not split
233             // between registers and stack.
234 
235             long offset = 0;
236             while (offset < layout.byteSize()) {
237                 long copy = Math.min(layout.byteSize() - offset, STACK_SLOT_SIZE);
238                 VMStorage storage =
239                     storageCalculator.stackAlloc(copy, STACK_SLOT_SIZE);
240                 if (offset + STACK_SLOT_SIZE < layout.byteSize()) {
241                     bindings.dup();
242                 }
243                 Class<?> type = SharedUtils.primitiveCarrierForSize(copy, false);
244                 bindings.bufferLoad(offset, type)
245                         .vmStore(storage, type);
246                 offset += STACK_SLOT_SIZE;
247             }
248         }
249 
250         protected void spillStructBox(Binding.Builder bindings, MemoryLayout layout) {
251             // If a struct has been assigned register or HFA class but
252             // there are not enough free registers to hold the entire
253             // struct, it must be passed on the stack. I.e. not split
254             // between registers and stack.
255 
256             long offset = 0;
257             while (offset < layout.byteSize()) {
258                 long copy = Math.min(layout.byteSize() - offset, STACK_SLOT_SIZE);
259                 VMStorage storage =
260                     storageCalculator.stackAlloc(copy, STACK_SLOT_SIZE);
261                 Class<?> type = SharedUtils.primitiveCarrierForSize(copy, false);
262                 bindings.dup()
263                         .vmLoad(storage, type)
264                         .bufferStore(offset, type);
265                 offset += STACK_SLOT_SIZE;
266             }
267         }
268 
269         abstract List<Binding> getBindings(Class<?> carrier, MemoryLayout layout);
270 
271         abstract List<Binding> getIndirectBindings();
272     }
273 
274     static class UnboxBindingCalculator extends BindingCalculator {
275         UnboxBindingCalculator(boolean forArguments) {
276             super(forArguments);
277         }
278 
279         @Override
280         List<Binding> getIndirectBindings() {
281             return Binding.builder()
282                 .unboxAddress()
283                 .vmStore(INDIRECT_RESULT, long.class)
284                 .build();
285         }
286 
287         @Override
288         List<Binding> getBindings(Class<?> carrier, MemoryLayout layout) {
289             TypeClass argumentClass = TypeClass.classifyLayout(layout);
290             Binding.Builder bindings = Binding.builder();
291             storageCalculator.adjustForVarArgs(layout);
292             switch (argumentClass) {
293                 case STRUCT_REGISTER: {
294                     assert carrier == MemorySegment.class;
295                     VMStorage[] regs = storageCalculator.regAlloc(
296                         StorageClasses.INTEGER, layout);
297                     if (regs != null) {
298                         int regIndex = 0;
299                         long offset = 0;
300                         while (offset < layout.byteSize()) {
301                             final long copy = Math.min(layout.byteSize() - offset, 8);
302                             VMStorage storage = regs[regIndex++];
303                             boolean useFloat = storage.type() == StorageClasses.VECTOR;
304                             Class<?> type = SharedUtils.primitiveCarrierForSize(copy, useFloat);
305                             if (offset + copy < layout.byteSize()) {
306                                 bindings.dup();
307                             }
308                             bindings.bufferLoad(offset, type)
309                                     .vmStore(storage, type);
310                             offset += copy;
311                         }
312                     } else {
313                         spillStructUnbox(bindings, layout);
314                     }
315                     break;
316                 }
317                 case STRUCT_REFERENCE: {
318                     assert carrier == MemorySegment.class;
319                     bindings.copy(layout)
320                             .baseAddress()
321                             .unboxAddress();
322                     VMStorage storage = storageCalculator.nextStorage(
323                         StorageClasses.INTEGER, AArch64.C_POINTER);
324                     bindings.vmStore(storage, long.class);
325                     break;
326                 }
327                 case STRUCT_HFA: {
328                     assert carrier == MemorySegment.class;
329                     GroupLayout group = (GroupLayout)layout;
330                     VMStorage[] regs = storageCalculator.regAlloc(
331                         StorageClasses.VECTOR, group.memberLayouts().size());
332                     if (regs != null) {
333                         long offset = 0;
334                         for (int i = 0; i < group.memberLayouts().size(); i++) {
335                             VMStorage storage = regs[i];
336                             final long size = group.memberLayouts().get(i).byteSize();
337                             boolean useFloat = storage.type() == StorageClasses.VECTOR;
338                             Class<?> type = SharedUtils.primitiveCarrierForSize(size, useFloat);
339                             if (i + 1 < group.memberLayouts().size()) {
340                                 bindings.dup();
341                             }
342                             bindings.bufferLoad(offset, type)
343                                     .vmStore(storage, type);
344                             offset += size;
345                         }
346                     } else {
347                         spillStructUnbox(bindings, layout);
348                     }
349                     break;
350                 }
351                 case POINTER: {
352                     bindings.unboxAddress();
353                     VMStorage storage =
354                         storageCalculator.nextStorage(StorageClasses.INTEGER, layout);
355                     bindings.vmStore(storage, long.class);
356                     break;
357                 }
358                 case INTEGER: {
359                     VMStorage storage =
360                         storageCalculator.nextStorage(StorageClasses.INTEGER, layout);
361                     bindings.vmStore(storage, carrier);
362                     break;
363                 }
364                 case FLOAT: {
365                     VMStorage storage =
366                         storageCalculator.nextStorage(StorageClasses.VECTOR, layout);
367                     bindings.vmStore(storage, carrier);
368                     break;
369                 }
370                 default:
371                     throw new UnsupportedOperationException("Unhandled class " + argumentClass);
372             }
373             return bindings.build();
374         }
375     }
376 
377     static class BoxBindingCalculator extends BindingCalculator{
378         BoxBindingCalculator(boolean forArguments) {
379             super(forArguments);
380         }
381 
382         @Override
383         List<Binding> getIndirectBindings() {
384             return Binding.builder()
385                 .vmLoad(INDIRECT_RESULT, long.class)
386                 .boxAddress()
387                 .build();
388         }
389 
390         @Override
391         List<Binding> getBindings(Class<?> carrier, MemoryLayout layout) {
392             TypeClass argumentClass = TypeClass.classifyLayout(layout);
393             Binding.Builder bindings = Binding.builder();
394             assert !layout.attribute(AArch64.STACK_VARARGS_ATTRIBUTE_NAME).isPresent() : "no variadic upcalls";
395             switch (argumentClass) {
396                 case STRUCT_REGISTER: {
397                     assert carrier == MemorySegment.class;
398                     bindings.allocate(layout);
399                     VMStorage[] regs = storageCalculator.regAlloc(
400                         StorageClasses.INTEGER, layout);
401                     if (regs != null) {
402                         int regIndex = 0;
403                         long offset = 0;
404                         while (offset < layout.byteSize()) {
405                             final long copy = Math.min(layout.byteSize() - offset, 8);
406                             VMStorage storage = regs[regIndex++];
407                             bindings.dup();
408                             boolean useFloat = storage.type() == StorageClasses.VECTOR;
409                             Class<?> type = SharedUtils.primitiveCarrierForSize(copy, useFloat);
410                             bindings.vmLoad(storage, type)
411                                     .bufferStore(offset, type);
412                             offset += copy;
413                         }
414                     } else {
415                         spillStructBox(bindings, layout);
416                     }
417                     break;
418                 }
419                 case STRUCT_REFERENCE: {
420                     assert carrier == MemorySegment.class;
421                     VMStorage storage = storageCalculator.nextStorage(
422                         StorageClasses.INTEGER, AArch64.C_POINTER);
423                     bindings.vmLoad(storage, long.class)
424                             .boxAddress()
425                             .toSegment(layout);
426                     break;
427                 }
428                 case STRUCT_HFA: {
429                     assert carrier == MemorySegment.class;
430                     bindings.allocate(layout);
431                     GroupLayout group = (GroupLayout)layout;
432                     VMStorage[] regs = storageCalculator.regAlloc(
433                         StorageClasses.VECTOR, group.memberLayouts().size());
434                     if (regs != null) {
435                         long offset = 0;
436                         for (int i = 0; i < group.memberLayouts().size(); i++) {
437                             VMStorage storage = regs[i];
438                             final long size = group.memberLayouts().get(i).byteSize();
439                             boolean useFloat = storage.type() == StorageClasses.VECTOR;
440                             Class<?> type = SharedUtils.primitiveCarrierForSize(size, useFloat);
441                             bindings.dup()
442                                     .vmLoad(storage, type)
443                                     .bufferStore(offset, type);
444                             offset += size;
445                         }
446                     } else {
447                         spillStructBox(bindings, layout);
448                     }
449                     break;
450                 }
451                 case POINTER: {
452                     VMStorage storage =
453                         storageCalculator.nextStorage(StorageClasses.INTEGER, layout);
454                     bindings.vmLoad(storage, long.class)
455                             .boxAddress();
456                     break;
457                 }
458                 case INTEGER: {
459                     VMStorage storage =
460                         storageCalculator.nextStorage(StorageClasses.INTEGER, layout);
461                     bindings.vmLoad(storage, carrier);
462                     break;
463                 }
464                 case FLOAT: {
465                     VMStorage storage =
466                         storageCalculator.nextStorage(StorageClasses.VECTOR, layout);
467                     bindings.vmLoad(storage, carrier);
468                     break;
469                 }
470                 default:
471                     throw new UnsupportedOperationException("Unhandled class " + argumentClass);
472             }
473             return bindings.build();
474         }
475     }
476 }