1 /*
  2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2019, 2021, Arm Limited. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.  Oracle designates this
  9  * particular file as subject to the "Classpath" exception as provided
 10  * by Oracle in the LICENSE file that accompanied this code.
 11  *
 12  * This code is distributed in the hope that it will be useful, but WITHOUT
 13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 15  * version 2 for more details (a copy is included in the LICENSE file that
 16  * accompanied this code).
 17  *
 18  * You should have received a copy of the GNU General Public License version
 19  * 2 along with this work; if not, write to the Free Software Foundation,
 20  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 21  *
 22  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 23  * or visit www.oracle.com if you need additional information or have any
 24  * questions.
 25  */
 26 package jdk.internal.foreign.abi.aarch64;
 27 
 28 import jdk.incubator.foreign.FunctionDescriptor;
 29 import jdk.incubator.foreign.GroupLayout;
 30 import jdk.incubator.foreign.MemoryAddress;
 31 import jdk.incubator.foreign.MemoryLayout;
 32 import jdk.incubator.foreign.MemorySegment;
 33 import jdk.incubator.foreign.NativeSymbol;
 34 import jdk.incubator.foreign.ResourceScope;
 35 import jdk.internal.foreign.Utils;
 36 import jdk.internal.foreign.abi.CallingSequenceBuilder;
 37 import jdk.internal.foreign.abi.ABIDescriptor;
 38 import jdk.internal.foreign.abi.Binding;
 39 import jdk.internal.foreign.abi.CallingSequence;
 40 import jdk.internal.foreign.abi.ProgrammableInvoker;
 41 import jdk.internal.foreign.abi.ProgrammableUpcallHandler;
 42 import jdk.internal.foreign.abi.VMStorage;
 43 import jdk.internal.foreign.abi.SharedUtils;
 44 import jdk.internal.foreign.abi.aarch64.linux.LinuxAArch64CallArranger;
 45 import jdk.internal.foreign.abi.aarch64.macos.MacOsAArch64CallArranger;
 46 
 47 import java.lang.invoke.MethodHandle;
 48 import java.lang.invoke.MethodType;
 49 import java.util.List;
 50 import java.util.Optional;
 51 
 52 import static jdk.internal.foreign.PlatformLayouts.*;
 53 import static jdk.internal.foreign.abi.aarch64.AArch64Architecture.*;
 54 
 55 /**
 56  * For the AArch64 C ABI specifically, this class uses the ProgrammableInvoker API, namely CallingSequenceBuilder2
 57  * to translate a C FunctionDescriptor into a CallingSequence2, which can then be turned into a MethodHandle.
 58  *
 59  * This includes taking care of synthetic arguments like pointers to return buffers for 'in-memory' returns.
 60  *
 61  * There are minor differences between the ABIs implemented on Linux, macOS, and Windows
 62  * which are handled in sub-classes. Clients should access these through the provided
 63  * public constants CallArranger.LINUX and CallArranger.MACOS.
 64  */
 65 public abstract class CallArranger {
 66     private static final int STACK_SLOT_SIZE = 8;
 67     public static final int MAX_REGISTER_ARGUMENTS = 8;
 68 
 69     private static final VMStorage INDIRECT_RESULT = r8;
 70 
 71     // This is derived from the AAPCS64 spec, restricted to what's
 72     // possible when calling to/from C code.
 73     //
 74     // The indirect result register, r8, is used to return a large
 75     // struct by value. It's treated as an input here as the caller is
 76     // responsible for allocating storage and passing this into the
 77     // function.
 78     //
 79     // Although the AAPCS64 says r0-7 and v0-7 are all valid return
 80     // registers, it's not possible to generate a C function that uses
 81     // r2-7 and v4-7 so they are omitted here.
 82     private static final ABIDescriptor C = AArch64Architecture.abiFor(
 83         new VMStorage[] { r0, r1, r2, r3, r4, r5, r6, r7, INDIRECT_RESULT},
 84         new VMStorage[] { v0, v1, v2, v3, v4, v5, v6, v7 },
 85         new VMStorage[] { r0, r1 },
 86         new VMStorage[] { v0, v1, v2, v3 },
 87         new VMStorage[] { r9, r10, r11, r12, r13, r14, r15 },
 88         new VMStorage[] { v16, v17, v18, v19, v20, v21, v22, v23, v25,
 89                           v26, v27, v28, v29, v30, v31 },
 90         16,  // Stack is always 16 byte aligned on AArch64
 91         0    // No shadow space
 92     );
 93 
 94     // record
 95     public static class Bindings {
 96         public final CallingSequence callingSequence;
 97         public final boolean isInMemoryReturn;
 98 
 99         Bindings(CallingSequence callingSequence, boolean isInMemoryReturn) {
100             this.callingSequence = callingSequence;
101             this.isInMemoryReturn = isInMemoryReturn;
102         }
103     }
104 
105     public static final CallArranger LINUX = new LinuxAArch64CallArranger();
106     public static final CallArranger MACOS = new MacOsAArch64CallArranger();
107 
108     /**
109      * Are variadic arguments assigned to registers as in the standard calling
110      * convention, or always passed on the stack?
111      *
112      * @returns true if variadic arguments should be spilled to the stack.
113      */
114     protected abstract boolean varArgsOnStack();
115 
116     protected CallArranger() {}
117 
118     public Bindings getBindings(MethodType mt, FunctionDescriptor cDesc, boolean forUpcall) {
119         CallingSequenceBuilder csb = new CallingSequenceBuilder(forUpcall);
120 
121         BindingCalculator argCalc = forUpcall ? new BoxBindingCalculator(true) : new UnboxBindingCalculator(true);
122         BindingCalculator retCalc = forUpcall ? new UnboxBindingCalculator(false) : new BoxBindingCalculator(false);
123 
124         boolean returnInMemory = isInMemoryReturn(cDesc.returnLayout());
125         if (returnInMemory) {
126             csb.addArgumentBindings(MemoryAddress.class, AArch64.C_POINTER,
127                     argCalc.getIndirectBindings());
128         } else if (cDesc.returnLayout().isPresent()) {
129             Class<?> carrier = mt.returnType();
130             MemoryLayout layout = cDesc.returnLayout().get();
131             csb.setReturnBindings(carrier, layout, retCalc.getBindings(carrier, layout));
132         }
133 
134         for (int i = 0; i < mt.parameterCount(); i++) {
135             Class<?> carrier = mt.parameterType(i);
136             MemoryLayout layout = cDesc.argumentLayouts().get(i);
137             if (varArgsOnStack() && SharedUtils.isVarargsIndex(cDesc, i)) {
138                 argCalc.storageCalculator.adjustForVarArgs();
139             }
140             csb.addArgumentBindings(carrier, layout, argCalc.getBindings(carrier, layout));
141         }
142 
143         csb.setTrivial(SharedUtils.isTrivial(cDesc));
144 
145         return new Bindings(csb.build(), returnInMemory);
146     }
147 
148     public MethodHandle arrangeDowncall(MethodType mt, FunctionDescriptor cDesc) {
149         Bindings bindings = getBindings(mt, cDesc, false);
150 
151         MethodHandle handle = new ProgrammableInvoker(C, bindings.callingSequence).getBoundMethodHandle();
152 
153         if (bindings.isInMemoryReturn) {
154             handle = SharedUtils.adaptDowncallForIMR(handle, cDesc);
155         }
156 
157         return handle;
158     }
159 
160     public NativeSymbol arrangeUpcall(MethodHandle target, MethodType mt, FunctionDescriptor cDesc, ResourceScope scope) {
161         Bindings bindings = getBindings(mt, cDesc, true);
162 
163         if (bindings.isInMemoryReturn) {
164             target = SharedUtils.adaptUpcallForIMR(target, true /* drop return, since we don't have bindings for it */);
165         }
166 
167         return ProgrammableUpcallHandler.make(C, target, bindings.callingSequence,scope);
168     }
169 
170     private static boolean isInMemoryReturn(Optional<MemoryLayout> returnLayout) {
171         return returnLayout
172             .filter(GroupLayout.class::isInstance)
173             .filter(g -> TypeClass.classifyLayout(g) == TypeClass.STRUCT_REFERENCE)
174             .isPresent();
175     }
176 
177     static class StorageCalculator {
178         private final boolean forArguments;
179 
180         private final int[] nRegs = new int[] { 0, 0 };
181         private long stackOffset = 0;
182 
183         public StorageCalculator(boolean forArguments) {
184             this.forArguments = forArguments;
185         }
186 
187         VMStorage stackAlloc(long size, long alignment) {
188             assert forArguments : "no stack returns";
189             alignment = Math.max(alignment, STACK_SLOT_SIZE);
190             stackOffset = Utils.alignUp(stackOffset, alignment);
191 
192             VMStorage storage =
193                 AArch64Architecture.stackStorage((int)(stackOffset / STACK_SLOT_SIZE));
194             stackOffset += size;
195             return storage;
196         }
197 
198         VMStorage stackAlloc(MemoryLayout layout) {
199             return stackAlloc(layout.byteSize(), SharedUtils.alignment(layout, true));
200         }
201 
202         VMStorage[] regAlloc(int type, int count) {
203             if (nRegs[type] + count <= MAX_REGISTER_ARGUMENTS) {
204                 VMStorage[] source =
205                     (forArguments ? C.inputStorage : C.outputStorage)[type];
206                 VMStorage[] result = new VMStorage[count];
207                 for (int i = 0; i < count; i++) {
208                     result[i] = source[nRegs[type]++];
209                 }
210                 return result;
211             } else {
212                 // Any further allocations for this register type must
213                 // be from the stack.
214                 nRegs[type] = MAX_REGISTER_ARGUMENTS;
215                 return null;
216             }
217         }
218 
219         VMStorage[] regAlloc(int type, MemoryLayout layout) {
220             return regAlloc(type, (int)Utils.alignUp(layout.byteSize(), 8) / 8);
221         }
222 
223         VMStorage nextStorage(int type, MemoryLayout layout) {
224             VMStorage[] storage = regAlloc(type, 1);
225             if (storage == null) {
226                 return stackAlloc(layout);
227             }
228 
229             return storage[0];
230         }
231 
232         void adjustForVarArgs() {
233             // This system passes all variadic parameters on the stack. Ensure
234             // no further arguments are allocated to registers.
235             nRegs[StorageClasses.INTEGER] = MAX_REGISTER_ARGUMENTS;
236             nRegs[StorageClasses.VECTOR] = MAX_REGISTER_ARGUMENTS;
237         }
238     }
239 
240     static abstract class BindingCalculator {
241         protected final StorageCalculator storageCalculator;
242 
243         protected BindingCalculator(boolean forArguments) {
244             this.storageCalculator = new StorageCalculator(forArguments);
245         }
246 
247         protected void spillStructUnbox(Binding.Builder bindings, MemoryLayout layout) {
248             // If a struct has been assigned register or HFA class but
249             // there are not enough free registers to hold the entire
250             // struct, it must be passed on the stack. I.e. not split
251             // between registers and stack.
252 
253             long offset = 0;
254             while (offset < layout.byteSize()) {
255                 long copy = Math.min(layout.byteSize() - offset, STACK_SLOT_SIZE);
256                 VMStorage storage =
257                     storageCalculator.stackAlloc(copy, STACK_SLOT_SIZE);
258                 if (offset + STACK_SLOT_SIZE < layout.byteSize()) {
259                     bindings.dup();
260                 }
261                 Class<?> type = SharedUtils.primitiveCarrierForSize(copy, false);
262                 bindings.bufferLoad(offset, type)
263                         .vmStore(storage, type);
264                 offset += STACK_SLOT_SIZE;
265             }
266         }
267 
268         protected void spillStructBox(Binding.Builder bindings, MemoryLayout layout) {
269             // If a struct has been assigned register or HFA class but
270             // there are not enough free registers to hold the entire
271             // struct, it must be passed on the stack. I.e. not split
272             // between registers and stack.
273 
274             long offset = 0;
275             while (offset < layout.byteSize()) {
276                 long copy = Math.min(layout.byteSize() - offset, STACK_SLOT_SIZE);
277                 VMStorage storage =
278                     storageCalculator.stackAlloc(copy, STACK_SLOT_SIZE);
279                 Class<?> type = SharedUtils.primitiveCarrierForSize(copy, false);
280                 bindings.dup()
281                         .vmLoad(storage, type)
282                         .bufferStore(offset, type);
283                 offset += STACK_SLOT_SIZE;
284             }
285         }
286 
287         abstract List<Binding> getBindings(Class<?> carrier, MemoryLayout layout);
288 
289         abstract List<Binding> getIndirectBindings();
290     }
291 
292     static class UnboxBindingCalculator extends BindingCalculator {
293         UnboxBindingCalculator(boolean forArguments) {
294             super(forArguments);
295         }
296 
297         @Override
298         List<Binding> getIndirectBindings() {
299             return Binding.builder()
300                 .unboxAddress()
301                 .vmStore(INDIRECT_RESULT, long.class)
302                 .build();
303         }
304 
305         @Override
306         List<Binding> getBindings(Class<?> carrier, MemoryLayout layout) {
307             TypeClass argumentClass = TypeClass.classifyLayout(layout);
308             Binding.Builder bindings = Binding.builder();
309             switch (argumentClass) {
310                 case STRUCT_REGISTER: {
311                     assert carrier == MemorySegment.class;
312                     VMStorage[] regs = storageCalculator.regAlloc(
313                         StorageClasses.INTEGER, layout);
314                     if (regs != null) {
315                         int regIndex = 0;
316                         long offset = 0;
317                         while (offset < layout.byteSize()) {
318                             final long copy = Math.min(layout.byteSize() - offset, 8);
319                             VMStorage storage = regs[regIndex++];
320                             boolean useFloat = storage.type() == StorageClasses.VECTOR;
321                             Class<?> type = SharedUtils.primitiveCarrierForSize(copy, useFloat);
322                             if (offset + copy < layout.byteSize()) {
323                                 bindings.dup();
324                             }
325                             bindings.bufferLoad(offset, type)
326                                     .vmStore(storage, type);
327                             offset += copy;
328                         }
329                     } else {
330                         spillStructUnbox(bindings, layout);
331                     }
332                     break;
333                 }
334                 case STRUCT_REFERENCE: {
335                     assert carrier == MemorySegment.class;
336                     bindings.copy(layout)
337                             .unboxAddress(MemorySegment.class);
338                     VMStorage storage = storageCalculator.nextStorage(
339                         StorageClasses.INTEGER, AArch64.C_POINTER);
340                     bindings.vmStore(storage, long.class);
341                     break;
342                 }
343                 case STRUCT_HFA: {
344                     assert carrier == MemorySegment.class;
345                     GroupLayout group = (GroupLayout)layout;
346                     VMStorage[] regs = storageCalculator.regAlloc(
347                         StorageClasses.VECTOR, group.memberLayouts().size());
348                     if (regs != null) {
349                         long offset = 0;
350                         for (int i = 0; i < group.memberLayouts().size(); i++) {
351                             VMStorage storage = regs[i];
352                             final long size = group.memberLayouts().get(i).byteSize();
353                             boolean useFloat = storage.type() == StorageClasses.VECTOR;
354                             Class<?> type = SharedUtils.primitiveCarrierForSize(size, useFloat);
355                             if (i + 1 < group.memberLayouts().size()) {
356                                 bindings.dup();
357                             }
358                             bindings.bufferLoad(offset, type)
359                                     .vmStore(storage, type);
360                             offset += size;
361                         }
362                     } else {
363                         spillStructUnbox(bindings, layout);
364                     }
365                     break;
366                 }
367                 case POINTER: {
368                     bindings.unboxAddress(carrier);
369                     VMStorage storage =
370                         storageCalculator.nextStorage(StorageClasses.INTEGER, layout);
371                     bindings.vmStore(storage, long.class);
372                     break;
373                 }
374                 case INTEGER: {
375                     VMStorage storage =
376                         storageCalculator.nextStorage(StorageClasses.INTEGER, layout);
377                     bindings.vmStore(storage, carrier);
378                     break;
379                 }
380                 case FLOAT: {
381                     VMStorage storage =
382                         storageCalculator.nextStorage(StorageClasses.VECTOR, layout);
383                     bindings.vmStore(storage, carrier);
384                     break;
385                 }
386                 default:
387                     throw new UnsupportedOperationException("Unhandled class " + argumentClass);
388             }
389             return bindings.build();
390         }
391     }
392 
393     static class BoxBindingCalculator extends BindingCalculator{
394         BoxBindingCalculator(boolean forArguments) {
395             super(forArguments);
396         }
397 
398         @Override
399         List<Binding> getIndirectBindings() {
400             return Binding.builder()
401                 .vmLoad(INDIRECT_RESULT, long.class)
402                 .boxAddress()
403                 .build();
404         }
405 
406         @Override
407         List<Binding> getBindings(Class<?> carrier, MemoryLayout layout) {
408             TypeClass argumentClass = TypeClass.classifyLayout(layout);
409             Binding.Builder bindings = Binding.builder();
410             switch (argumentClass) {
411                 case STRUCT_REGISTER: {
412                     assert carrier == MemorySegment.class;
413                     bindings.allocate(layout);
414                     VMStorage[] regs = storageCalculator.regAlloc(
415                         StorageClasses.INTEGER, layout);
416                     if (regs != null) {
417                         int regIndex = 0;
418                         long offset = 0;
419                         while (offset < layout.byteSize()) {
420                             final long copy = Math.min(layout.byteSize() - offset, 8);
421                             VMStorage storage = regs[regIndex++];
422                             bindings.dup();
423                             boolean useFloat = storage.type() == StorageClasses.VECTOR;
424                             Class<?> type = SharedUtils.primitiveCarrierForSize(copy, useFloat);
425                             bindings.vmLoad(storage, type)
426                                     .bufferStore(offset, type);
427                             offset += copy;
428                         }
429                     } else {
430                         spillStructBox(bindings, layout);
431                     }
432                     break;
433                 }
434                 case STRUCT_REFERENCE: {
435                     assert carrier == MemorySegment.class;
436                     VMStorage storage = storageCalculator.nextStorage(
437                         StorageClasses.INTEGER, AArch64.C_POINTER);
438                     bindings.vmLoad(storage, long.class)
439                             .boxAddress()
440                             .toSegment(layout);
441                     break;
442                 }
443                 case STRUCT_HFA: {
444                     assert carrier == MemorySegment.class;
445                     bindings.allocate(layout);
446                     GroupLayout group = (GroupLayout)layout;
447                     VMStorage[] regs = storageCalculator.regAlloc(
448                         StorageClasses.VECTOR, group.memberLayouts().size());
449                     if (regs != null) {
450                         long offset = 0;
451                         for (int i = 0; i < group.memberLayouts().size(); i++) {
452                             VMStorage storage = regs[i];
453                             final long size = group.memberLayouts().get(i).byteSize();
454                             boolean useFloat = storage.type() == StorageClasses.VECTOR;
455                             Class<?> type = SharedUtils.primitiveCarrierForSize(size, useFloat);
456                             bindings.dup()
457                                     .vmLoad(storage, type)
458                                     .bufferStore(offset, type);
459                             offset += size;
460                         }
461                     } else {
462                         spillStructBox(bindings, layout);
463                     }
464                     break;
465                 }
466                 case POINTER: {
467                     VMStorage storage =
468                         storageCalculator.nextStorage(StorageClasses.INTEGER, layout);
469                     bindings.vmLoad(storage, long.class)
470                             .boxAddress();
471                     break;
472                 }
473                 case INTEGER: {
474                     VMStorage storage =
475                         storageCalculator.nextStorage(StorageClasses.INTEGER, layout);
476                     bindings.vmLoad(storage, carrier);
477                     break;
478                 }
479                 case FLOAT: {
480                     VMStorage storage =
481                         storageCalculator.nextStorage(StorageClasses.VECTOR, layout);
482                     bindings.vmLoad(storage, carrier);
483                     break;
484                 }
485                 default:
486                     throw new UnsupportedOperationException("Unhandled class " + argumentClass);
487             }
488             return bindings.build();
489         }
490     }
491 }