1 /*
  2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2020, 2021, Arm Limited. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.  Oracle designates this
  9  * particular file as subject to the "Classpath" exception as provided
 10  * by Oracle in the LICENSE file that accompanied this code.
 11  *
 12  * This code is distributed in the hope that it will be useful, but WITHOUT
 13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 15  * version 2 for more details (a copy is included in the LICENSE file that
 16  * accompanied this code).
 17  *
 18  * You should have received a copy of the GNU General Public License version
 19  * 2 along with this work; if not, write to the Free Software Foundation,
 20  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 21  *
 22  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 23  * or visit www.oracle.com if you need additional information or have any
 24  * questions.
 25  */
 26 package jdk.internal.foreign.abi.aarch64.linux;
 27 
 28 import jdk.incubator.foreign.*;
 29 import jdk.internal.foreign.Utils;
 30 import jdk.internal.foreign.abi.SharedUtils;
 31 import jdk.internal.foreign.abi.aarch64.*;
 32 import jdk.internal.misc.Unsafe;
 33 
 34 import java.lang.invoke.VarHandle;
 35 import java.lang.ref.Cleaner;
 36 import java.nio.ByteOrder;
 37 import java.util.ArrayList;
 38 import java.util.List;
 39 import java.util.Objects;
 40 
 41 import static jdk.internal.foreign.PlatformLayouts.AArch64;
 42 import static jdk.incubator.foreign.CLinker.VaList;
 43 import static jdk.incubator.foreign.MemoryLayout.PathElement.groupElement;
 44 import static jdk.internal.foreign.abi.SharedUtils.SimpleVaArg;
 45 import static jdk.internal.foreign.abi.SharedUtils.THROWING_ALLOCATOR;
 46 import static jdk.internal.foreign.abi.SharedUtils.checkCompatibleType;
 47 import static jdk.internal.foreign.abi.SharedUtils.vhPrimitiveOrAddress;
 48 import static jdk.internal.foreign.abi.aarch64.CallArranger.MAX_REGISTER_ARGUMENTS;
 49 
 50 /**
 51  * Standard va_list implementation as defined by AAPCS document and used on
 52  * Linux. Variadic parameters may be passed in registers or on the stack.
 53  */
 54 public non-sealed class LinuxAArch64VaList implements VaList {
 55     private static final Unsafe U = Unsafe.getUnsafe();
 56 
 57     static final Class<?> CARRIER = MemoryAddress.class;
 58 
 59     // See AAPCS Appendix B "Variable Argument Lists" for definition of
 60     // va_list on AArch64.
 61     //
 62     // typedef struct __va_list {
 63     //     void *__stack;   // next stack param
 64     //     void *__gr_top;  // end of GP arg reg save area
 65     //     void *__vr_top;  // end of FP/SIMD arg reg save area
 66     //     int __gr_offs;   // offset from __gr_top to next GP register arg
 67     //     int __vr_offs;   // offset from __vr_top to next FP/SIMD register arg
 68     // } va_list;
 69 
 70     static final GroupLayout LAYOUT = MemoryLayout.structLayout(
 71         AArch64.C_POINTER.withName("__stack"),
 72         AArch64.C_POINTER.withName("__gr_top"),
 73         AArch64.C_POINTER.withName("__vr_top"),
 74         AArch64.C_INT.withName("__gr_offs"),
 75         AArch64.C_INT.withName("__vr_offs")
 76     ).withName("__va_list");
 77 
 78     private static final MemoryLayout GP_REG
 79         = MemoryLayout.valueLayout(64, ByteOrder.nativeOrder());
 80     private static final MemoryLayout FP_REG
 81         = MemoryLayout.valueLayout(128, ByteOrder.nativeOrder());
 82 
 83     private static final MemoryLayout LAYOUT_GP_REGS
 84         = MemoryLayout.sequenceLayout(MAX_REGISTER_ARGUMENTS, GP_REG);
 85     private static final MemoryLayout LAYOUT_FP_REGS
 86         = MemoryLayout.sequenceLayout(MAX_REGISTER_ARGUMENTS, FP_REG);
 87 
 88     private static final int GP_SLOT_SIZE = (int) GP_REG.byteSize();
 89     private static final int FP_SLOT_SIZE = (int) FP_REG.byteSize();
 90 
 91     private static final int MAX_GP_OFFSET = (int) LAYOUT_GP_REGS.byteSize();
 92     private static final int MAX_FP_OFFSET = (int) LAYOUT_FP_REGS.byteSize();
 93 
 94     private static final VarHandle VH_stack
 95         = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("__stack")));
 96     private static final VarHandle VH_gr_top
 97         = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("__gr_top")));
 98     private static final VarHandle VH_vr_top
 99         = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("__vr_top")));
100     private static final VarHandle VH_gr_offs
101         = LAYOUT.varHandle(int.class, groupElement("__gr_offs"));
102     private static final VarHandle VH_vr_offs
103         = LAYOUT.varHandle(int.class, groupElement("__vr_offs"));
104 
105     private static final Cleaner cleaner = Cleaner.create();
106     private static final VaList EMPTY
107         = new SharedUtils.EmptyVaList(emptyListAddress());
108 
109     private final MemorySegment segment;
110     private final MemorySegment gpRegsArea;
111     private final MemorySegment fpRegsArea;
112 
113     private LinuxAArch64VaList(MemorySegment segment, MemorySegment gpRegsArea, MemorySegment fpRegsArea) {
114         this.segment = segment;
115         this.gpRegsArea = gpRegsArea;
116         this.fpRegsArea = fpRegsArea;
117     }
118 
119     private static LinuxAArch64VaList readFromSegment(MemorySegment segment) {
120         MemorySegment gpRegsArea = grTop(segment).addOffset(-MAX_GP_OFFSET).asSegment(
121                 MAX_GP_OFFSET, segment.scope());
122 
123         MemorySegment fpRegsArea = vrTop(segment).addOffset(-MAX_FP_OFFSET).asSegment(
124                 MAX_FP_OFFSET, segment.scope());
125         return new LinuxAArch64VaList(segment, gpRegsArea, fpRegsArea);
126     }
127 
128     private static MemoryAddress emptyListAddress() {
129         long ptr = U.allocateMemory(LAYOUT.byteSize());
130         MemorySegment ms = MemoryAddress.ofLong(ptr).asSegment(
131                 LAYOUT.byteSize(), () -> U.freeMemory(ptr), ResourceScope.newSharedScope());
132         cleaner.register(LinuxAArch64VaList.class, () -> ms.scope().close());
133         VH_stack.set(ms, MemoryAddress.NULL);
134         VH_gr_top.set(ms, MemoryAddress.NULL);
135         VH_vr_top.set(ms, MemoryAddress.NULL);
136         VH_gr_offs.set(ms, 0);
137         VH_vr_offs.set(ms, 0);
138         return ms.address();
139     }
140 
141     public static VaList empty() {
142         return EMPTY;
143     }
144 
145     private MemoryAddress grTop() {
146         return grTop(segment);
147     }
148 
149     private static MemoryAddress grTop(MemorySegment segment) {
150         return (MemoryAddress) VH_gr_top.get(segment);
151     }
152 
153     private MemoryAddress vrTop() {
154         return vrTop(segment);
155     }
156 
157     private static MemoryAddress vrTop(MemorySegment segment) {
158         return (MemoryAddress) VH_vr_top.get(segment);
159     }
160 
161     private int grOffs() {
162         final int offs = (int) VH_gr_offs.get(segment);
163         assert offs <= 0;
164         return offs;
165     }
166 
167     private int vrOffs() {
168         final int offs = (int) VH_vr_offs.get(segment);
169         assert offs <= 0;
170         return offs;
171     }
172 
173     private MemoryAddress stackPtr() {
174         return (MemoryAddress) VH_stack.get(segment);
175     }
176 
177     private void stackPtr(MemoryAddress ptr) {
178         VH_stack.set(segment, ptr);
179     }
180 
181     private void consumeGPSlots(int num) {
182         final int old = (int) VH_gr_offs.get(segment);
183         VH_gr_offs.set(segment, old + num * GP_SLOT_SIZE);
184     }
185 
186     private void consumeFPSlots(int num) {
187         final int old = (int) VH_vr_offs.get(segment);
188         VH_vr_offs.set(segment, old + num * FP_SLOT_SIZE);
189     }
190 
191     private long currentGPOffset() {
192         // Offset from start of GP register segment. __gr_top points to the top
193         // (highest address) of the GP registers area. __gr_offs is the negative
194         // offset of next saved register from the top.
195 
196         return gpRegsArea.byteSize() + grOffs();
197     }
198 
199     private long currentFPOffset() {
200         // Offset from start of FP register segment. __vr_top points to the top
201         // (highest address) of the FP registers area. __vr_offs is the negative
202         // offset of next saved register from the top.
203 
204         return fpRegsArea.byteSize() + vrOffs();
205     }
206 
207     private void preAlignStack(MemoryLayout layout) {
208         if (layout.byteAlignment() > 8) {
209             stackPtr(Utils.alignUp(stackPtr(), 16));
210         }
211     }
212 
213     private void postAlignStack(MemoryLayout layout) {
214         stackPtr(Utils.alignUp(stackPtr().addOffset(layout.byteSize()), 8));
215     }
216 
217     @Override
218     public int vargAsInt(MemoryLayout layout) {
219         return (int) read(int.class, layout);
220     }
221 
222     @Override
223     public long vargAsLong(MemoryLayout layout) {
224         return (long) read(long.class, layout);
225     }
226 
227     @Override
228     public double vargAsDouble(MemoryLayout layout) {
229         return (double) read(double.class, layout);
230     }
231 
232     @Override
233     public MemoryAddress vargAsAddress(MemoryLayout layout) {
234         return (MemoryAddress) read(MemoryAddress.class, layout);
235     }
236 
237     @Override
238     public MemorySegment vargAsSegment(MemoryLayout layout, SegmentAllocator allocator) {
239         Objects.requireNonNull(allocator);
240         return (MemorySegment) read(MemorySegment.class, layout, allocator);
241     }
242 
243     @Override
244     public MemorySegment vargAsSegment(MemoryLayout layout, ResourceScope scope) {
245         return vargAsSegment(layout, SegmentAllocator.ofScope(scope));
246     }
247 
248     private Object read(Class<?> carrier, MemoryLayout layout) {
249         return read(carrier, layout, THROWING_ALLOCATOR);
250     }
251 
252     private Object read(Class<?> carrier, MemoryLayout layout, SegmentAllocator allocator) {
253         Objects.requireNonNull(layout);
254         checkCompatibleType(carrier, layout, LinuxAArch64Linker.ADDRESS_SIZE);
255 
256         TypeClass typeClass = TypeClass.classifyLayout(layout);
257         if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass, layout)) {
258             preAlignStack(layout);
259             return switch (typeClass) {
260                 case STRUCT_REGISTER, STRUCT_HFA, STRUCT_REFERENCE -> {
261                     MemorySegment slice = stackPtr().asSegment(layout.byteSize(), scope());
262                     MemorySegment seg = allocator.allocate(layout);
263                     seg.copyFrom(slice);
264                     postAlignStack(layout);
265                     yield seg;
266                 }
267                 case POINTER, INTEGER, FLOAT -> {
268                     VarHandle reader = vhPrimitiveOrAddress(carrier, layout);
269                     MemorySegment slice = stackPtr().asSegment(layout.byteSize(), scope());
270                     Object res = reader.get(slice);
271                     postAlignStack(layout);
272                     yield res;
273                 }
274             };
275         } else {
276             return switch (typeClass) {
277                 case STRUCT_REGISTER -> {
278                     // Struct is passed packed in integer registers.
279                     MemorySegment value = allocator.allocate(layout);
280                     long offset = 0;
281                     while (offset < layout.byteSize()) {
282                         final long copy = Math.min(layout.byteSize() - offset, 8);
283                         MemorySegment slice = value.asSlice(offset, copy);
284                         slice.copyFrom(gpRegsArea.asSlice(currentGPOffset(), copy));
285                         consumeGPSlots(1);
286                         offset += copy;
287                     }
288                     yield value;
289                 }
290                 case STRUCT_HFA -> {
291                     // Struct is passed with each element in a separate floating
292                     // point register.
293                     MemorySegment value = allocator.allocate(layout);
294                     GroupLayout group = (GroupLayout)layout;
295                     long offset = 0;
296                     for (MemoryLayout elem : group.memberLayouts()) {
297                         assert elem.byteSize() <= 8;
298                         final long copy = elem.byteSize();
299                         MemorySegment slice = value.asSlice(offset, copy);
300                         slice.copyFrom(fpRegsArea.asSlice(currentFPOffset(), copy));
301                         consumeFPSlots(1);
302                         offset += copy;
303                     }
304                     yield value;
305                 }
306                 case STRUCT_REFERENCE -> {
307                     // Struct is passed indirectly via a pointer in an integer register.
308                     VarHandle ptrReader
309                         = SharedUtils.vhPrimitiveOrAddress(MemoryAddress.class, AArch64.C_POINTER);
310                     MemoryAddress ptr = (MemoryAddress) ptrReader.get(
311                         gpRegsArea.asSlice(currentGPOffset()));
312                     consumeGPSlots(1);
313 
314                     MemorySegment slice = ptr.asSegment(layout.byteSize(), scope());
315                     MemorySegment seg = allocator.allocate(layout);
316                     seg.copyFrom(slice);
317                     yield seg;
318                 }
319                 case POINTER, INTEGER -> {
320                     VarHandle reader = SharedUtils.vhPrimitiveOrAddress(carrier, layout);
321                     Object res = reader.get(gpRegsArea.asSlice(currentGPOffset()));
322                     consumeGPSlots(1);
323                     yield res;
324                 }
325                 case FLOAT -> {
326                     VarHandle reader = layout.varHandle(carrier);
327                     Object res = reader.get(fpRegsArea.asSlice(currentFPOffset()));
328                     consumeFPSlots(1);
329                     yield res;
330                 }
331             };
332         }
333     }
334 
335     @Override
336     public void skip(MemoryLayout... layouts) {
337         Objects.requireNonNull(layouts);
338         for (MemoryLayout layout : layouts) {
339             Objects.requireNonNull(layout);
340             TypeClass typeClass = TypeClass.classifyLayout(layout);
341             if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass, layout)) {
342                 preAlignStack(layout);
343                 postAlignStack(layout);
344             } else if (typeClass == TypeClass.FLOAT || typeClass == TypeClass.STRUCT_HFA) {
345                 consumeFPSlots(numSlots(layout));
346             } else if (typeClass == TypeClass.STRUCT_REFERENCE) {
347                 consumeGPSlots(1);
348             } else {
349                 consumeGPSlots(numSlots(layout));
350             }
351         }
352     }
353 
354     static LinuxAArch64VaList.Builder builder(ResourceScope scope) {
355         return new LinuxAArch64VaList.Builder(scope);
356     }
357 
358     public static VaList ofAddress(MemoryAddress ma, ResourceScope scope) {
359         return readFromSegment(ma.asSegment(LAYOUT.byteSize(), scope));
360     }
361 
362     @Override
363     public ResourceScope scope() {
364         return segment.scope();
365     }
366 
367     @Override
368     public VaList copy() {
369         MemorySegment copy = MemorySegment.allocateNative(LAYOUT, segment.scope());
370         copy.copyFrom(segment);
371         return new LinuxAArch64VaList(copy, gpRegsArea, fpRegsArea);
372     }
373 
374     @Override
375     public MemoryAddress address() {
376         return segment.address();
377     }
378 
379     private static int numSlots(MemoryLayout layout) {
380         return (int) Utils.alignUp(layout.byteSize(), 8) / 8;
381     }
382 
383     private static boolean isRegOverflow(long currentGPOffset, long currentFPOffset,
384                                          TypeClass typeClass, MemoryLayout layout) {
385         if (typeClass == TypeClass.FLOAT || typeClass == TypeClass.STRUCT_HFA) {
386             return currentFPOffset > MAX_FP_OFFSET - numSlots(layout) * FP_SLOT_SIZE;
387         } else if (typeClass == TypeClass.STRUCT_REFERENCE) {
388             return currentGPOffset > MAX_GP_OFFSET - GP_SLOT_SIZE;
389         } else {
390             return currentGPOffset > MAX_GP_OFFSET - numSlots(layout) * GP_SLOT_SIZE;
391         }
392     }
393 
394     @Override
395     public String toString() {
396         return "LinuxAArch64VaList{"
397             + "__stack=" + stackPtr()
398             + ", __gr_top=" + grTop()
399             + ", __vr_top=" + vrTop()
400             + ", __gr_offs=" + grOffs()
401             + ", __vr_offs=" + vrOffs()
402             + '}';
403     }
404 
405     public static non-sealed class Builder implements VaList.Builder {
406         private final ResourceScope scope;
407         private final MemorySegment gpRegs;
408         private final MemorySegment fpRegs;
409 
410         private long currentGPOffset = 0;
411         private long currentFPOffset = 0;
412         private final List<SimpleVaArg> stackArgs = new ArrayList<>();
413 
414         Builder(ResourceScope scope) {
415             this.scope = scope;
416             this.gpRegs = MemorySegment.allocateNative(LAYOUT_GP_REGS, scope);
417             this.fpRegs = MemorySegment.allocateNative(LAYOUT_FP_REGS, scope);
418         }
419 
420         @Override
421         public Builder vargFromInt(ValueLayout layout, int value) {
422             return arg(int.class, layout, value);
423         }
424 
425         @Override
426         public Builder vargFromLong(ValueLayout layout, long value) {
427             return arg(long.class, layout, value);
428         }
429 
430         @Override
431         public Builder vargFromDouble(ValueLayout layout, double value) {
432             return arg(double.class, layout, value);
433         }
434 
435         @Override
436         public Builder vargFromAddress(ValueLayout layout, Addressable value) {
437             return arg(MemoryAddress.class, layout, value.address());
438         }
439 
440         @Override
441         public Builder vargFromSegment(GroupLayout layout, MemorySegment value) {
442             return arg(MemorySegment.class, layout, value);
443         }
444 
445         private Builder arg(Class<?> carrier, MemoryLayout layout, Object value) {
446             Objects.requireNonNull(layout);
447             Objects.requireNonNull(value);
448             checkCompatibleType(carrier, layout, LinuxAArch64Linker.ADDRESS_SIZE);
449 
450             TypeClass typeClass = TypeClass.classifyLayout(layout);
451             if (isRegOverflow(currentGPOffset, currentFPOffset, typeClass, layout)) {
452                 stackArgs.add(new SimpleVaArg(carrier, layout, value));
453             } else {
454                 switch (typeClass) {
455                     case STRUCT_REGISTER -> {
456                         // Struct is passed packed in integer registers.
457                         MemorySegment valueSegment = (MemorySegment) value;
458                         long offset = 0;
459                         while (offset < layout.byteSize()) {
460                             final long copy = Math.min(layout.byteSize() - offset, 8);
461                             MemorySegment slice = valueSegment.asSlice(offset, copy);
462                             gpRegs.asSlice(currentGPOffset, copy).copyFrom(slice);
463                             currentGPOffset += GP_SLOT_SIZE;
464                             offset += copy;
465                         }
466                     }
467                     case STRUCT_HFA -> {
468                         // Struct is passed with each element in a separate floating
469                         // point register.
470                         MemorySegment valueSegment = (MemorySegment) value;
471                         GroupLayout group = (GroupLayout)layout;
472                         long offset = 0;
473                         for (MemoryLayout elem : group.memberLayouts()) {
474                             assert elem.byteSize() <= 8;
475                             final long copy = elem.byteSize();
476                             MemorySegment slice = valueSegment.asSlice(offset, copy);
477                             fpRegs.asSlice(currentFPOffset, copy).copyFrom(slice);
478                             currentFPOffset += FP_SLOT_SIZE;
479                             offset += copy;
480                         }
481                     }
482                     case STRUCT_REFERENCE -> {
483                         // Struct is passed indirectly via a pointer in an integer register.
484                         MemorySegment valueSegment = (MemorySegment) value;
485                         VarHandle writer
486                             = SharedUtils.vhPrimitiveOrAddress(MemoryAddress.class,
487                                                                AArch64.C_POINTER);
488                         writer.set(gpRegs.asSlice(currentGPOffset),
489                                    valueSegment.address());
490                         currentGPOffset += GP_SLOT_SIZE;
491                     }
492                     case POINTER, INTEGER -> {
493                         VarHandle writer = SharedUtils.vhPrimitiveOrAddress(carrier, layout);
494                         writer.set(gpRegs.asSlice(currentGPOffset), value);
495                         currentGPOffset += GP_SLOT_SIZE;
496                     }
497                     case FLOAT -> {
498                         VarHandle writer = layout.varHandle(carrier);
499                         writer.set(fpRegs.asSlice(currentFPOffset), value);
500                         currentFPOffset += FP_SLOT_SIZE;
501                     }
502                 }
503             }
504             return this;
505         }
506 
507         private boolean isEmpty() {
508             return currentGPOffset == 0 && currentFPOffset == 0 && stackArgs.isEmpty();
509         }
510 
511         public VaList build() {
512             if (isEmpty()) {
513                 return EMPTY;
514             }
515 
516             SegmentAllocator allocator = SegmentAllocator.arenaAllocator(scope);
517             MemorySegment vaListSegment = allocator.allocate(LAYOUT);
518             MemoryAddress stackArgsPtr = MemoryAddress.NULL;
519             if (!stackArgs.isEmpty()) {
520                 long stackArgsSize = stackArgs.stream()
521                     .reduce(0L, (acc, e) -> acc + Utils.alignUp(e.layout.byteSize(), 8), Long::sum);
522                 MemorySegment stackArgsSegment = allocator.allocate(stackArgsSize, 16);
523                 stackArgsPtr = stackArgsSegment.address();
524                 for (SimpleVaArg arg : stackArgs) {
525                     final long alignedSize = Utils.alignUp(arg.layout.byteSize(), 8);
526                     stackArgsSegment = Utils.alignUp(stackArgsSegment, alignedSize);
527                     VarHandle writer = arg.varHandle();
528                     writer.set(stackArgsSegment, arg.value);
529                     stackArgsSegment = stackArgsSegment.asSlice(alignedSize);
530                 }
531             }
532 
533             VH_gr_top.set(vaListSegment, gpRegs.asSlice(gpRegs.byteSize()).address());
534             VH_vr_top.set(vaListSegment, fpRegs.asSlice(fpRegs.byteSize()).address());
535             VH_stack.set(vaListSegment, stackArgsPtr);
536             VH_gr_offs.set(vaListSegment, -MAX_GP_OFFSET);
537             VH_vr_offs.set(vaListSegment, -MAX_FP_OFFSET);
538 
539             assert gpRegs.scope().ownerThread() == vaListSegment.scope().ownerThread();
540             assert fpRegs.scope().ownerThread() == vaListSegment.scope().ownerThread();
541             return new LinuxAArch64VaList(vaListSegment, gpRegs, fpRegs);
542         }
543     }
544 }