1 /*
  2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2020, 2021, Arm Limited. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.  Oracle designates this
  9  * particular file as subject to the "Classpath" exception as provided
 10  * by Oracle in the LICENSE file that accompanied this code.
 11  *
 12  * This code is distributed in the hope that it will be useful, but WITHOUT
 13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 15  * version 2 for more details (a copy is included in the LICENSE file that
 16  * accompanied this code).
 17  *
 18  * You should have received a copy of the GNU General Public License version
 19  * 2 along with this work; if not, write to the Free Software Foundation,
 20  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 21  *
 22  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 23  * or visit www.oracle.com if you need additional information or have any
 24  * questions.
 25  */
 26 package jdk.internal.foreign.abi.aarch64.linux;
 27 
 28 import jdk.incubator.foreign.*;
 29 import jdk.internal.foreign.ResourceScopeImpl;
 30 import jdk.internal.foreign.Scoped;
 31 import jdk.internal.foreign.Utils;
 32 import jdk.internal.foreign.abi.SharedUtils;
 33 import jdk.internal.foreign.abi.aarch64.*;
 34 import jdk.internal.misc.Unsafe;
 35 
 36 import java.lang.invoke.VarHandle;
 37 import java.lang.ref.Cleaner;
 38 import java.util.ArrayList;
 39 import java.util.List;
 40 import java.util.Objects;
 41 
 42 import static jdk.internal.foreign.PlatformLayouts.AArch64;
 43 
 44 import static jdk.incubator.foreign.MemoryLayout.PathElement.groupElement;
 45 import static jdk.internal.foreign.abi.SharedUtils.SimpleVaArg;
 46 import static jdk.internal.foreign.abi.SharedUtils.THROWING_ALLOCATOR;
 47 import static jdk.internal.foreign.abi.aarch64.CallArranger.MAX_REGISTER_ARGUMENTS;
 48 
 49 /**
 50  * Standard va_list implementation as defined by AAPCS document and used on
 51  * Linux. Variadic parameters may be passed in registers or on the stack.
 52  */
 53 public non-sealed class LinuxAArch64VaList implements VaList, Scoped {
 54     private static final Unsafe U = Unsafe.getUnsafe();
 55 
 56     static final Class<?> CARRIER = MemoryAddress.class;
 57 
 58     // See AAPCS Appendix B "Variable Argument Lists" for definition of
 59     // va_list on AArch64.
 60     //
 61     // typedef struct __va_list {
 62     //     void *__stack;   // next stack param
 63     //     void *__gr_top;  // end of GP arg reg save area
 64     //     void *__vr_top;  // end of FP/SIMD arg reg save area
 65     //     int __gr_offs;   // offset from __gr_top to next GP register arg
 66     //     int __vr_offs;   // offset from __vr_top to next FP/SIMD register arg
 67     // } va_list;
 68 
 69     static final GroupLayout LAYOUT = MemoryLayout.structLayout(
 70         AArch64.C_POINTER.withName("__stack"),
 71         AArch64.C_POINTER.withName("__gr_top"),
 72         AArch64.C_POINTER.withName("__vr_top"),
 73         AArch64.C_INT.withName("__gr_offs"),
 74         AArch64.C_INT.withName("__vr_offs")
 75     ).withName("__va_list");
 76 
 77     private static final MemoryLayout GP_REG
 78         = MemoryLayout.paddingLayout(64).withBitAlignment(64);
 79     private static final MemoryLayout FP_REG
 80         = MemoryLayout.paddingLayout(128).withBitAlignment(128);
 81 
 82     private static final MemoryLayout LAYOUT_GP_REGS
 83         = MemoryLayout.sequenceLayout(MAX_REGISTER_ARGUMENTS, GP_REG);
 84     private static final MemoryLayout LAYOUT_FP_REGS
 85         = MemoryLayout.sequenceLayout(MAX_REGISTER_ARGUMENTS, FP_REG);
 86 
 87     private static final int GP_SLOT_SIZE = (int) GP_REG.byteSize();
 88     private static final int FP_SLOT_SIZE = (int) FP_REG.byteSize();
 89 
 90     private static final int MAX_GP_OFFSET = (int) LAYOUT_GP_REGS.byteSize();
 91     private static final int MAX_FP_OFFSET = (int) LAYOUT_FP_REGS.byteSize();
 92 
 93     private static final VarHandle VH_stack = LAYOUT.varHandle(groupElement("__stack"));
 94     private static final VarHandle VH_gr_top = LAYOUT.varHandle(groupElement("__gr_top"));
 95     private static final VarHandle VH_vr_top = LAYOUT.varHandle(groupElement("__vr_top"));
 96     private static final VarHandle VH_gr_offs
 97         = LAYOUT.varHandle(groupElement("__gr_offs"));
 98     private static final VarHandle VH_vr_offs
 99         = LAYOUT.varHandle(groupElement("__vr_offs"));
100 
101     private static final VaList EMPTY
102         = new SharedUtils.EmptyVaList(emptyListAddress());
103 
104     private final MemorySegment segment;
105     private final MemorySegment gpRegsArea;
106     private final MemorySegment fpRegsArea;
107 
108     private LinuxAArch64VaList(MemorySegment segment, MemorySegment gpRegsArea, MemorySegment fpRegsArea) {
109         this.segment = segment;
110         this.gpRegsArea = gpRegsArea;
111         this.fpRegsArea = fpRegsArea;
112     }
113 
114     private static LinuxAArch64VaList readFromSegment(MemorySegment segment) {
115         MemorySegment gpRegsArea = MemorySegment.ofAddress(grTop(segment).addOffset(-MAX_GP_OFFSET),
116                 MAX_GP_OFFSET, segment.scope());
117 
118         MemorySegment fpRegsArea = MemorySegment.ofAddress(vrTop(segment).addOffset(-MAX_FP_OFFSET),
119                 MAX_FP_OFFSET, segment.scope());
120         return new LinuxAArch64VaList(segment, gpRegsArea, fpRegsArea);
121     }
122 
123     private static MemoryAddress emptyListAddress() {
124         long ptr = U.allocateMemory(LAYOUT.byteSize());
125         ResourceScope scope = ResourceScope.newImplicitScope();
126         scope.addCloseAction(() -> U.freeMemory(ptr));
127         MemorySegment ms = MemorySegment.ofAddress(MemoryAddress.ofLong(ptr),
128                 LAYOUT.byteSize(), scope);
129         VH_stack.set(ms, MemoryAddress.NULL);
130         VH_gr_top.set(ms, MemoryAddress.NULL);
131         VH_vr_top.set(ms, MemoryAddress.NULL);
132         VH_gr_offs.set(ms, 0);
133         VH_vr_offs.set(ms, 0);
134         return ms.address();
135     }
136 
137     public static VaList empty() {
138         return EMPTY;
139     }
140 
141     private MemoryAddress grTop() {
142         return grTop(segment);
143     }
144 
145     private static MemoryAddress grTop(MemorySegment segment) {
146         return (MemoryAddress) VH_gr_top.get(segment);
147     }
148 
149     private MemoryAddress vrTop() {
150         return vrTop(segment);
151     }
152 
153     private static MemoryAddress vrTop(MemorySegment segment) {
154         return (MemoryAddress) VH_vr_top.get(segment);
155     }
156 
157     private int grOffs() {
158         final int offs = (int) VH_gr_offs.get(segment);
159         assert offs <= 0;
160         return offs;
161     }
162 
163     private int vrOffs() {
164         final int offs = (int) VH_vr_offs.get(segment);
165         assert offs <= 0;
166         return offs;
167     }
168 
169     private MemoryAddress stackPtr() {
170         return (MemoryAddress) VH_stack.get(segment);
171     }
172 
173     private void stackPtr(MemoryAddress ptr) {
174         VH_stack.set(segment, ptr);
175     }
176 
177     private void consumeGPSlots(int num) {
178         final int old = (int) VH_gr_offs.get(segment);
179         VH_gr_offs.set(segment, old + num * GP_SLOT_SIZE);
180     }
181 
182     private void consumeFPSlots(int num) {
183         final int old = (int) VH_vr_offs.get(segment);
184         VH_vr_offs.set(segment, old + num * FP_SLOT_SIZE);
185     }
186 
187     private long currentGPOffset() {
188         // Offset from start of GP register segment. __gr_top points to the top
189         // (highest address) of the GP registers area. __gr_offs is the negative
190         // offset of next saved register from the top.
191 
192         return gpRegsArea.byteSize() + grOffs();
193     }
194 
195     private long currentFPOffset() {
196         // Offset from start of FP register segment. __vr_top points to the top
197         // (highest address) of the FP registers area. __vr_offs is the negative
198         // offset of next saved register from the top.
199 
200         return fpRegsArea.byteSize() + vrOffs();
201     }
202 
203     private void preAlignStack(MemoryLayout layout) {
204         if (layout.byteAlignment() > 8) {
205             stackPtr(Utils.alignUp(stackPtr(), 16));
206         }
207     }
208 
209     private void postAlignStack(MemoryLayout layout) {
210         stackPtr(Utils.alignUp(stackPtr().addOffset(layout.byteSize()), 8));
211     }
212 
213     @Override
214     public int nextVarg(ValueLayout.OfInt layout) {
215         return (int) read(int.class, layout);
216     }
217 
218     @Override
219     public long nextVarg(ValueLayout.OfLong layout) {
220         return (long) read(long.class, layout);
221     }
222 
223     @Override
224     public double nextVarg(ValueLayout.OfDouble layout) {
225         return (double) read(double.class, layout);
226     }
227 
228     @Override
229     public MemoryAddress nextVarg(ValueLayout.OfAddress layout) {
230         return (MemoryAddress) read(MemoryAddress.class, layout);
231     }
232 
233     @Override
234     public MemorySegment nextVarg(GroupLayout layout, SegmentAllocator allocator) {
235         Objects.requireNonNull(allocator);
236         return (MemorySegment) read(MemorySegment.class, layout, allocator);
237     }
238 
239     private Object read(Class<?> carrier, MemoryLayout layout) {
240         return read(carrier, layout, THROWING_ALLOCATOR);
241     }
242 
243     private Object read(Class<?> carrier, MemoryLayout layout, SegmentAllocator allocator) {
244         Objects.requireNonNull(layout);
245         TypeClass typeClass = TypeClass.classifyLayout(layout);
246         if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass, layout)) {
247             preAlignStack(layout);
248             return switch (typeClass) {
249                 case STRUCT_REGISTER, STRUCT_HFA, STRUCT_REFERENCE -> {
250                     MemorySegment slice = MemorySegment.ofAddress(stackPtr(), layout.byteSize(), scope());
251                     MemorySegment seg = allocator.allocate(layout);
252                     seg.copyFrom(slice);
253                     postAlignStack(layout);
254                     yield seg;
255                 }
256                 case POINTER, INTEGER, FLOAT -> {
257                     VarHandle reader = layout.varHandle();
258                     MemorySegment slice = MemorySegment.ofAddress(stackPtr(), layout.byteSize(), scope());
259                     Object res = reader.get(slice);
260                     postAlignStack(layout);
261                     yield res;
262                 }
263             };
264         } else {
265             return switch (typeClass) {
266                 case STRUCT_REGISTER -> {
267                     // Struct is passed packed in integer registers.
268                     MemorySegment value = allocator.allocate(layout);
269                     long offset = 0;
270                     while (offset < layout.byteSize()) {
271                         final long copy = Math.min(layout.byteSize() - offset, 8);
272                         MemorySegment.copy(gpRegsArea, currentGPOffset(), value, offset, copy);
273                         consumeGPSlots(1);
274                         offset += copy;
275                     }
276                     yield value;
277                 }
278                 case STRUCT_HFA -> {
279                     // Struct is passed with each element in a separate floating
280                     // point register.
281                     MemorySegment value = allocator.allocate(layout);
282                     GroupLayout group = (GroupLayout)layout;
283                     long offset = 0;
284                     for (MemoryLayout elem : group.memberLayouts()) {
285                         assert elem.byteSize() <= 8;
286                         final long copy = elem.byteSize();
287                         MemorySegment.copy(fpRegsArea, currentFPOffset(), value, offset, copy);
288                         consumeFPSlots(1);
289                         offset += copy;
290                     }
291                     yield value;
292                 }
293                 case STRUCT_REFERENCE -> {
294                     // Struct is passed indirectly via a pointer in an integer register.
295                     VarHandle ptrReader = AArch64.C_POINTER.varHandle();
296                     MemoryAddress ptr = (MemoryAddress) ptrReader.get(
297                         gpRegsArea.asSlice(currentGPOffset()));
298                     consumeGPSlots(1);
299 
300                     MemorySegment slice = MemorySegment.ofAddress(ptr, layout.byteSize(), scope());
301                     MemorySegment seg = allocator.allocate(layout);
302                     seg.copyFrom(slice);
303                     yield seg;
304                 }
305                 case POINTER, INTEGER -> {
306                     VarHandle reader = layout.varHandle();
307                     Object res = reader.get(gpRegsArea.asSlice(currentGPOffset()));
308                     consumeGPSlots(1);
309                     yield res;
310                 }
311                 case FLOAT -> {
312                     VarHandle reader = layout.varHandle();
313                     Object res = reader.get(fpRegsArea.asSlice(currentFPOffset()));
314                     consumeFPSlots(1);
315                     yield res;
316                 }
317             };
318         }
319     }
320 
321     @Override
322     public void skip(MemoryLayout... layouts) {
323         Objects.requireNonNull(layouts);
324         ((ResourceScopeImpl)segment.scope()).checkValidStateSlow();
325         for (MemoryLayout layout : layouts) {
326             Objects.requireNonNull(layout);
327             TypeClass typeClass = TypeClass.classifyLayout(layout);
328             if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass, layout)) {
329                 preAlignStack(layout);
330                 postAlignStack(layout);
331             } else if (typeClass == TypeClass.FLOAT || typeClass == TypeClass.STRUCT_HFA) {
332                 consumeFPSlots(numSlots(layout));
333             } else if (typeClass == TypeClass.STRUCT_REFERENCE) {
334                 consumeGPSlots(1);
335             } else {
336                 consumeGPSlots(numSlots(layout));
337             }
338         }
339     }
340 
341     static LinuxAArch64VaList.Builder builder(ResourceScope scope) {
342         return new LinuxAArch64VaList.Builder(scope);
343     }
344 
345     public static VaList ofAddress(MemoryAddress ma, ResourceScope scope) {
346         return readFromSegment(MemorySegment.ofAddress(ma, LAYOUT.byteSize(), scope));
347     }
348 
349     @Override
350     public ResourceScope scope() {
351         return segment.scope();
352     }
353 
354     @Override
355     public VaList copy() {
356         MemorySegment copy = MemorySegment.allocateNative(LAYOUT, segment.scope());
357         copy.copyFrom(segment);
358         return new LinuxAArch64VaList(copy, gpRegsArea, fpRegsArea);
359     }
360 
361     @Override
362     public MemoryAddress address() {
363         return segment.address();
364     }
365 
366     private static int numSlots(MemoryLayout layout) {
367         return (int) Utils.alignUp(layout.byteSize(), 8) / 8;
368     }
369 
370     private static boolean isRegOverflow(long currentGPOffset, long currentFPOffset,
371                                          TypeClass typeClass, MemoryLayout layout) {
372         if (typeClass == TypeClass.FLOAT || typeClass == TypeClass.STRUCT_HFA) {
373             return currentFPOffset > MAX_FP_OFFSET - numSlots(layout) * FP_SLOT_SIZE;
374         } else if (typeClass == TypeClass.STRUCT_REFERENCE) {
375             return currentGPOffset > MAX_GP_OFFSET - GP_SLOT_SIZE;
376         } else {
377             return currentGPOffset > MAX_GP_OFFSET - numSlots(layout) * GP_SLOT_SIZE;
378         }
379     }
380 
381     @Override
382     public String toString() {
383         return "LinuxAArch64VaList{"
384             + "__stack=" + stackPtr()
385             + ", __gr_top=" + grTop()
386             + ", __vr_top=" + vrTop()
387             + ", __gr_offs=" + grOffs()
388             + ", __vr_offs=" + vrOffs()
389             + '}';
390     }
391 
392     public static non-sealed class Builder implements VaList.Builder {
393         private final ResourceScope scope;
394         private final MemorySegment gpRegs;
395         private final MemorySegment fpRegs;
396 
397         private long currentGPOffset = 0;
398         private long currentFPOffset = 0;
399         private final List<SimpleVaArg> stackArgs = new ArrayList<>();
400 
401         Builder(ResourceScope scope) {
402             this.scope = scope;
403             this.gpRegs = MemorySegment.allocateNative(LAYOUT_GP_REGS, scope);
404             this.fpRegs = MemorySegment.allocateNative(LAYOUT_FP_REGS, scope);
405         }
406 
407         @Override
408         public Builder addVarg(ValueLayout.OfInt layout, int value) {
409             return arg(int.class, layout, value);
410         }
411 
412         @Override
413         public Builder addVarg(ValueLayout.OfLong layout, long value) {
414             return arg(long.class, layout, value);
415         }
416 
417         @Override
418         public Builder addVarg(ValueLayout.OfDouble layout, double value) {
419             return arg(double.class, layout, value);
420         }
421 
422         @Override
423         public Builder addVarg(ValueLayout.OfAddress layout, Addressable value) {
424             return arg(MemoryAddress.class, layout, value.address());
425         }
426 
427         @Override
428         public Builder addVarg(GroupLayout layout, MemorySegment value) {
429             return arg(MemorySegment.class, layout, value);
430         }
431 
432         private Builder arg(Class<?> carrier, MemoryLayout layout, Object value) {
433             Objects.requireNonNull(layout);
434             Objects.requireNonNull(value);
435             TypeClass typeClass = TypeClass.classifyLayout(layout);
436             if (isRegOverflow(currentGPOffset, currentFPOffset, typeClass, layout)) {
437                 stackArgs.add(new SimpleVaArg(carrier, layout, value));
438             } else {
439                 switch (typeClass) {
440                     case STRUCT_REGISTER -> {
441                         // Struct is passed packed in integer registers.
442                         MemorySegment valueSegment = (MemorySegment) value;
443                         long offset = 0;
444                         while (offset < layout.byteSize()) {
445                             final long copy = Math.min(layout.byteSize() - offset, 8);
446                             MemorySegment.copy(valueSegment, offset, gpRegs, currentGPOffset, copy);
447                             currentGPOffset += GP_SLOT_SIZE;
448                             offset += copy;
449                         }
450                     }
451                     case STRUCT_HFA -> {
452                         // Struct is passed with each element in a separate floating
453                         // point register.
454                         MemorySegment valueSegment = (MemorySegment) value;
455                         GroupLayout group = (GroupLayout)layout;
456                         long offset = 0;
457                         for (MemoryLayout elem : group.memberLayouts()) {
458                             assert elem.byteSize() <= 8;
459                             final long copy = elem.byteSize();
460                             MemorySegment.copy(valueSegment, offset, fpRegs, currentFPOffset, copy);
461                             currentFPOffset += FP_SLOT_SIZE;
462                             offset += copy;
463                         }
464                     }
465                     case STRUCT_REFERENCE -> {
466                         // Struct is passed indirectly via a pointer in an integer register.
467                         MemorySegment valueSegment = (MemorySegment) value;
468                         VarHandle writer = AArch64.C_POINTER.varHandle();
469                         writer.set(gpRegs.asSlice(currentGPOffset),
470                                    valueSegment.address());
471                         currentGPOffset += GP_SLOT_SIZE;
472                     }
473                     case POINTER, INTEGER -> {
474                         VarHandle writer = layout.varHandle();
475                         writer.set(gpRegs.asSlice(currentGPOffset), value);
476                         currentGPOffset += GP_SLOT_SIZE;
477                     }
478                     case FLOAT -> {
479                         VarHandle writer = layout.varHandle();
480                         writer.set(fpRegs.asSlice(currentFPOffset), value);
481                         currentFPOffset += FP_SLOT_SIZE;
482                     }
483                 }
484             }
485             return this;
486         }
487 
488         private boolean isEmpty() {
489             return currentGPOffset == 0 && currentFPOffset == 0 && stackArgs.isEmpty();
490         }
491 
492         public VaList build() {
493             if (isEmpty()) {
494                 return EMPTY;
495             }
496 
497             SegmentAllocator allocator = SegmentAllocator.newNativeArena(scope);
498             MemorySegment vaListSegment = allocator.allocate(LAYOUT);
499             MemoryAddress stackArgsPtr = MemoryAddress.NULL;
500             if (!stackArgs.isEmpty()) {
501                 long stackArgsSize = stackArgs.stream()
502                     .reduce(0L, (acc, e) -> acc + Utils.alignUp(e.layout.byteSize(), 8), Long::sum);
503                 MemorySegment stackArgsSegment = allocator.allocate(stackArgsSize, 16);
504                 stackArgsPtr = stackArgsSegment.address();
505                 for (SimpleVaArg arg : stackArgs) {
506                     final long alignedSize = Utils.alignUp(arg.layout.byteSize(), 8);
507                     stackArgsSegment = Utils.alignUp(stackArgsSegment, alignedSize);
508                     VarHandle writer = arg.varHandle();
509                     writer.set(stackArgsSegment, arg.value);
510                     stackArgsSegment = stackArgsSegment.asSlice(alignedSize);
511                 }
512             }
513 
514             VH_gr_top.set(vaListSegment, gpRegs.asSlice(gpRegs.byteSize()).address());
515             VH_vr_top.set(vaListSegment, fpRegs.asSlice(fpRegs.byteSize()).address());
516             VH_stack.set(vaListSegment, stackArgsPtr);
517             VH_gr_offs.set(vaListSegment, -MAX_GP_OFFSET);
518             VH_vr_offs.set(vaListSegment, -MAX_FP_OFFSET);
519 
520             assert gpRegs.scope().ownerThread() == vaListSegment.scope().ownerThread();
521             assert fpRegs.scope().ownerThread() == vaListSegment.scope().ownerThread();
522             return new LinuxAArch64VaList(vaListSegment, gpRegs, fpRegs);
523         }
524     }
525 }
--- EOF ---