1 /*
  2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * Copyright (c) 2020, 2021, Arm Limited. All rights reserved.
  4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  5  *
  6  * This code is free software; you can redistribute it and/or modify it
  7  * under the terms of the GNU General Public License version 2 only, as
  8  * published by the Free Software Foundation.  Oracle designates this
  9  * particular file as subject to the "Classpath" exception as provided
 10  * by Oracle in the LICENSE file that accompanied this code.
 11  *
 12  * This code is distributed in the hope that it will be useful, but WITHOUT
 13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 15  * version 2 for more details (a copy is included in the LICENSE file that
 16  * accompanied this code).
 17  *
 18  * You should have received a copy of the GNU General Public License version
 19  * 2 along with this work; if not, write to the Free Software Foundation,
 20  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 21  *
 22  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 23  * or visit www.oracle.com if you need additional information or have any
 24  * questions.
 25  */
 26 package jdk.internal.foreign.abi.aarch64.linux;
 27 
 28 import jdk.incubator.foreign.*;
 29 import jdk.internal.foreign.ResourceScopeImpl;
 30 import jdk.internal.foreign.Scoped;
 31 import jdk.internal.foreign.Utils;
 32 import jdk.internal.foreign.abi.SharedUtils;
 33 import jdk.internal.foreign.abi.aarch64.*;
 34 import jdk.internal.misc.Unsafe;
 35 
 36 import java.lang.invoke.VarHandle;

 37 import java.util.ArrayList;
 38 import java.util.List;
 39 import java.util.Objects;
 40 
 41 import static jdk.internal.foreign.PlatformLayouts.AArch64;
 42 
 43 import static jdk.incubator.foreign.MemoryLayout.PathElement.groupElement;
 44 import static jdk.internal.foreign.abi.SharedUtils.SimpleVaArg;
 45 import static jdk.internal.foreign.abi.SharedUtils.THROWING_ALLOCATOR;
 46 import static jdk.internal.foreign.abi.aarch64.CallArranger.MAX_REGISTER_ARGUMENTS;
 47 
 48 /**
 49  * Standard va_list implementation as defined by AAPCS document and used on
 50  * Linux. Variadic parameters may be passed in registers or on the stack.
 51  */
 52 public non-sealed class LinuxAArch64VaList implements VaList, Scoped {
 53     private static final Unsafe U = Unsafe.getUnsafe();
 54 
 55     static final Class<?> CARRIER = MemoryAddress.class;
 56 
 57     // See AAPCS Appendix B "Variable Argument Lists" for definition of
 58     // va_list on AArch64.
 59     //
 60     // typedef struct __va_list {
 61     //     void *__stack;   // next stack param
 62     //     void *__gr_top;  // end of GP arg reg save area
 63     //     void *__vr_top;  // end of FP/SIMD arg reg save area
 64     //     int __gr_offs;   // offset from __gr_top to next GP register arg
 65     //     int __vr_offs;   // offset from __vr_top to next FP/SIMD register arg
 66     // } va_list;
 67 
 68     static final GroupLayout LAYOUT = MemoryLayout.structLayout(
 69         AArch64.C_POINTER.withName("__stack"),
 70         AArch64.C_POINTER.withName("__gr_top"),
 71         AArch64.C_POINTER.withName("__vr_top"),
 72         AArch64.C_INT.withName("__gr_offs"),
 73         AArch64.C_INT.withName("__vr_offs")
 74     ).withName("__va_list");
 75 
 76     private static final MemoryLayout GP_REG
 77         = MemoryLayout.paddingLayout(64).withBitAlignment(64);
 78     private static final MemoryLayout FP_REG
 79         = MemoryLayout.paddingLayout(128).withBitAlignment(128);
 80 
 81     private static final MemoryLayout LAYOUT_GP_REGS
 82         = MemoryLayout.sequenceLayout(MAX_REGISTER_ARGUMENTS, GP_REG);
 83     private static final MemoryLayout LAYOUT_FP_REGS
 84         = MemoryLayout.sequenceLayout(MAX_REGISTER_ARGUMENTS, FP_REG);
 85 
 86     private static final int GP_SLOT_SIZE = (int) GP_REG.byteSize();
 87     private static final int FP_SLOT_SIZE = (int) FP_REG.byteSize();
 88 
 89     private static final int MAX_GP_OFFSET = (int) LAYOUT_GP_REGS.byteSize();
 90     private static final int MAX_FP_OFFSET = (int) LAYOUT_FP_REGS.byteSize();
 91 
 92     private static final VarHandle VH_stack = LAYOUT.varHandle(groupElement("__stack"));
 93     private static final VarHandle VH_gr_top = LAYOUT.varHandle(groupElement("__gr_top"));
 94     private static final VarHandle VH_vr_top = LAYOUT.varHandle(groupElement("__vr_top"));
 95     private static final VarHandle VH_gr_offs
 96         = LAYOUT.varHandle(groupElement("__gr_offs"));
 97     private static final VarHandle VH_vr_offs
 98         = LAYOUT.varHandle(groupElement("__vr_offs"));
 99 
100     private static final VaList EMPTY
101         = new SharedUtils.EmptyVaList(emptyListAddress());
102 
103     private final MemorySegment segment;
104     private final MemorySegment gpRegsArea;
105     private final MemorySegment fpRegsArea;
106 
107     private LinuxAArch64VaList(MemorySegment segment, MemorySegment gpRegsArea, MemorySegment fpRegsArea) {
108         this.segment = segment;
109         this.gpRegsArea = gpRegsArea;
110         this.fpRegsArea = fpRegsArea;
111     }
112 
113     private static LinuxAArch64VaList readFromSegment(MemorySegment segment) {
114         MemorySegment gpRegsArea = MemorySegment.ofAddress(grTop(segment).addOffset(-MAX_GP_OFFSET),
115                 MAX_GP_OFFSET, segment.scope());
116 
117         MemorySegment fpRegsArea = MemorySegment.ofAddress(vrTop(segment).addOffset(-MAX_FP_OFFSET),
118                 MAX_FP_OFFSET, segment.scope());
119         return new LinuxAArch64VaList(segment, gpRegsArea, fpRegsArea);
120     }
121 
122     private static MemoryAddress emptyListAddress() {
123         long ptr = U.allocateMemory(LAYOUT.byteSize());
124         ResourceScope scope = ResourceScope.newImplicitScope();
125         scope.addCloseAction(() -> U.freeMemory(ptr));
126         MemorySegment ms = MemorySegment.ofAddress(MemoryAddress.ofLong(ptr),
127                 LAYOUT.byteSize(), scope);
128         VH_stack.set(ms, MemoryAddress.NULL);
129         VH_gr_top.set(ms, MemoryAddress.NULL);
130         VH_vr_top.set(ms, MemoryAddress.NULL);
131         VH_gr_offs.set(ms, 0);
132         VH_vr_offs.set(ms, 0);
133         return ms.address();
134     }
135 
136     public static VaList empty() {
137         return EMPTY;
138     }
139 
140     private MemoryAddress grTop() {
141         return grTop(segment);
142     }
143 
144     private static MemoryAddress grTop(MemorySegment segment) {
145         return (MemoryAddress) VH_gr_top.get(segment);
146     }
147 
148     private MemoryAddress vrTop() {
149         return vrTop(segment);
150     }
151 
152     private static MemoryAddress vrTop(MemorySegment segment) {
153         return (MemoryAddress) VH_vr_top.get(segment);
154     }
155 
156     private int grOffs() {
157         final int offs = (int) VH_gr_offs.get(segment);
158         assert offs <= 0;
159         return offs;
160     }
161 
162     private int vrOffs() {
163         final int offs = (int) VH_vr_offs.get(segment);
164         assert offs <= 0;
165         return offs;
166     }
167 
168     private MemoryAddress stackPtr() {
169         return (MemoryAddress) VH_stack.get(segment);
170     }
171 
172     private void stackPtr(MemoryAddress ptr) {
173         VH_stack.set(segment, ptr);
174     }
175 
176     private void consumeGPSlots(int num) {
177         final int old = (int) VH_gr_offs.get(segment);
178         VH_gr_offs.set(segment, old + num * GP_SLOT_SIZE);
179     }
180 
181     private void consumeFPSlots(int num) {
182         final int old = (int) VH_vr_offs.get(segment);
183         VH_vr_offs.set(segment, old + num * FP_SLOT_SIZE);
184     }
185 
186     private long currentGPOffset() {
187         // Offset from start of GP register segment. __gr_top points to the top
188         // (highest address) of the GP registers area. __gr_offs is the negative
189         // offset of next saved register from the top.
190 
191         return gpRegsArea.byteSize() + grOffs();
192     }
193 
194     private long currentFPOffset() {
195         // Offset from start of FP register segment. __vr_top points to the top
196         // (highest address) of the FP registers area. __vr_offs is the negative
197         // offset of next saved register from the top.
198 
199         return fpRegsArea.byteSize() + vrOffs();
200     }
201 
202     private void preAlignStack(MemoryLayout layout) {
203         if (layout.byteAlignment() > 8) {
204             stackPtr(Utils.alignUp(stackPtr(), 16));
205         }
206     }
207 
208     private void postAlignStack(MemoryLayout layout) {
209         stackPtr(Utils.alignUp(stackPtr().addOffset(layout.byteSize()), 8));
210     }
211 
212     @Override
213     public int nextVarg(ValueLayout.OfInt layout) {
214         return (int) read(int.class, layout);
215     }
216 
217     @Override
218     public long nextVarg(ValueLayout.OfLong layout) {
219         return (long) read(long.class, layout);
220     }
221 
222     @Override
223     public double nextVarg(ValueLayout.OfDouble layout) {
224         return (double) read(double.class, layout);
225     }
226 
227     @Override
228     public MemoryAddress nextVarg(ValueLayout.OfAddress layout) {
229         return (MemoryAddress) read(MemoryAddress.class, layout);
230     }
231 
232     @Override
233     public MemorySegment nextVarg(GroupLayout layout, SegmentAllocator allocator) {
234         Objects.requireNonNull(allocator);
235         return (MemorySegment) read(MemorySegment.class, layout, allocator);
236     }
237 
238     private Object read(Class<?> carrier, MemoryLayout layout) {
239         return read(carrier, layout, THROWING_ALLOCATOR);
240     }
241 
242     private Object read(Class<?> carrier, MemoryLayout layout, SegmentAllocator allocator) {
243         Objects.requireNonNull(layout);
244         TypeClass typeClass = TypeClass.classifyLayout(layout);
245         if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass, layout)) {
246             preAlignStack(layout);
247             return switch (typeClass) {
248                 case STRUCT_REGISTER, STRUCT_HFA, STRUCT_REFERENCE -> {
249                     MemorySegment slice = MemorySegment.ofAddress(stackPtr(), layout.byteSize(), scope());
250                     MemorySegment seg = allocator.allocate(layout);
251                     seg.copyFrom(slice);
252                     postAlignStack(layout);
253                     yield seg;
254                 }
255                 case POINTER, INTEGER, FLOAT -> {
256                     VarHandle reader = layout.varHandle();
257                     MemorySegment slice = MemorySegment.ofAddress(stackPtr(), layout.byteSize(), scope());
258                     Object res = reader.get(slice);
259                     postAlignStack(layout);
260                     yield res;
261                 }
262             };
263         } else {
264             return switch (typeClass) {
265                 case STRUCT_REGISTER -> {
266                     // Struct is passed packed in integer registers.
267                     MemorySegment value = allocator.allocate(layout);
268                     long offset = 0;
269                     while (offset < layout.byteSize()) {
270                         final long copy = Math.min(layout.byteSize() - offset, 8);
271                         MemorySegment.copy(gpRegsArea, currentGPOffset(), value, offset, copy);
272                         consumeGPSlots(1);
273                         offset += copy;
274                     }
275                     yield value;
276                 }
277                 case STRUCT_HFA -> {
278                     // Struct is passed with each element in a separate floating
279                     // point register.
280                     MemorySegment value = allocator.allocate(layout);
281                     GroupLayout group = (GroupLayout)layout;
282                     long offset = 0;
283                     for (MemoryLayout elem : group.memberLayouts()) {
284                         assert elem.byteSize() <= 8;
285                         final long copy = elem.byteSize();
286                         MemorySegment.copy(fpRegsArea, currentFPOffset(), value, offset, copy);
287                         consumeFPSlots(1);
288                         offset += copy;
289                     }
290                     yield value;
291                 }
292                 case STRUCT_REFERENCE -> {
293                     // Struct is passed indirectly via a pointer in an integer register.
294                     VarHandle ptrReader = AArch64.C_POINTER.varHandle();
295                     MemoryAddress ptr = (MemoryAddress) ptrReader.get(
296                         gpRegsArea.asSlice(currentGPOffset()));
297                     consumeGPSlots(1);
298 
299                     MemorySegment slice = MemorySegment.ofAddress(ptr, layout.byteSize(), scope());
300                     MemorySegment seg = allocator.allocate(layout);
301                     seg.copyFrom(slice);
302                     yield seg;
303                 }
304                 case POINTER, INTEGER -> {
305                     VarHandle reader = layout.varHandle();
306                     Object res = reader.get(gpRegsArea.asSlice(currentGPOffset()));
307                     consumeGPSlots(1);
308                     yield res;
309                 }
310                 case FLOAT -> {
311                     VarHandle reader = layout.varHandle();
312                     Object res = reader.get(fpRegsArea.asSlice(currentFPOffset()));
313                     consumeFPSlots(1);
314                     yield res;
315                 }
316             };
317         }
318     }
319 
320     @Override
321     public void skip(MemoryLayout... layouts) {
322         Objects.requireNonNull(layouts);
323         ((ResourceScopeImpl)segment.scope()).checkValidStateSlow();
324         for (MemoryLayout layout : layouts) {
325             Objects.requireNonNull(layout);
326             TypeClass typeClass = TypeClass.classifyLayout(layout);
327             if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass, layout)) {
328                 preAlignStack(layout);
329                 postAlignStack(layout);
330             } else if (typeClass == TypeClass.FLOAT || typeClass == TypeClass.STRUCT_HFA) {
331                 consumeFPSlots(numSlots(layout));
332             } else if (typeClass == TypeClass.STRUCT_REFERENCE) {
333                 consumeGPSlots(1);
334             } else {
335                 consumeGPSlots(numSlots(layout));
336             }
337         }
338     }
339 
340     static LinuxAArch64VaList.Builder builder(ResourceScope scope) {
341         return new LinuxAArch64VaList.Builder(scope);
342     }
343 
344     public static VaList ofAddress(MemoryAddress ma, ResourceScope scope) {
345         return readFromSegment(MemorySegment.ofAddress(ma, LAYOUT.byteSize(), scope));
346     }
347 
348     @Override
349     public ResourceScope scope() {
350         return segment.scope();
351     }
352 
353     @Override
354     public VaList copy() {
355         MemorySegment copy = MemorySegment.allocateNative(LAYOUT, segment.scope());
356         copy.copyFrom(segment);
357         return new LinuxAArch64VaList(copy, gpRegsArea, fpRegsArea);
358     }
359 
360     @Override
361     public MemoryAddress address() {
362         return segment.address();
363     }
364 
365     private static int numSlots(MemoryLayout layout) {
366         return (int) Utils.alignUp(layout.byteSize(), 8) / 8;
367     }
368 
369     private static boolean isRegOverflow(long currentGPOffset, long currentFPOffset,
370                                          TypeClass typeClass, MemoryLayout layout) {
371         if (typeClass == TypeClass.FLOAT || typeClass == TypeClass.STRUCT_HFA) {
372             return currentFPOffset > MAX_FP_OFFSET - numSlots(layout) * FP_SLOT_SIZE;
373         } else if (typeClass == TypeClass.STRUCT_REFERENCE) {
374             return currentGPOffset > MAX_GP_OFFSET - GP_SLOT_SIZE;
375         } else {
376             return currentGPOffset > MAX_GP_OFFSET - numSlots(layout) * GP_SLOT_SIZE;
377         }
378     }
379 
380     @Override
381     public String toString() {
382         return "LinuxAArch64VaList{"
383             + "__stack=" + stackPtr()
384             + ", __gr_top=" + grTop()
385             + ", __vr_top=" + vrTop()
386             + ", __gr_offs=" + grOffs()
387             + ", __vr_offs=" + vrOffs()
388             + '}';
389     }
390 
391     public static non-sealed class Builder implements VaList.Builder {
392         private final ResourceScope scope;
393         private final MemorySegment gpRegs;
394         private final MemorySegment fpRegs;
395 
396         private long currentGPOffset = 0;
397         private long currentFPOffset = 0;
398         private final List<SimpleVaArg> stackArgs = new ArrayList<>();
399 
400         Builder(ResourceScope scope) {
401             this.scope = scope;
402             this.gpRegs = MemorySegment.allocateNative(LAYOUT_GP_REGS, scope);
403             this.fpRegs = MemorySegment.allocateNative(LAYOUT_FP_REGS, scope);
404         }
405 
406         @Override
407         public Builder addVarg(ValueLayout.OfInt layout, int value) {
408             return arg(int.class, layout, value);
409         }
410 
411         @Override
412         public Builder addVarg(ValueLayout.OfLong layout, long value) {
413             return arg(long.class, layout, value);
414         }
415 
416         @Override
417         public Builder addVarg(ValueLayout.OfDouble layout, double value) {
418             return arg(double.class, layout, value);
419         }
420 
421         @Override
422         public Builder addVarg(ValueLayout.OfAddress layout, Addressable value) {
423             return arg(MemoryAddress.class, layout, value.address());
424         }
425 
426         @Override
427         public Builder addVarg(GroupLayout layout, MemorySegment value) {
428             return arg(MemorySegment.class, layout, value);
429         }
430 
431         private Builder arg(Class<?> carrier, MemoryLayout layout, Object value) {
432             Objects.requireNonNull(layout);
433             Objects.requireNonNull(value);
434             TypeClass typeClass = TypeClass.classifyLayout(layout);
435             if (isRegOverflow(currentGPOffset, currentFPOffset, typeClass, layout)) {
436                 stackArgs.add(new SimpleVaArg(carrier, layout, value));
437             } else {
438                 switch (typeClass) {
439                     case STRUCT_REGISTER -> {
440                         // Struct is passed packed in integer registers.
441                         MemorySegment valueSegment = (MemorySegment) value;
442                         long offset = 0;
443                         while (offset < layout.byteSize()) {
444                             final long copy = Math.min(layout.byteSize() - offset, 8);
445                             MemorySegment.copy(valueSegment, offset, gpRegs, currentGPOffset, copy);
446                             currentGPOffset += GP_SLOT_SIZE;
447                             offset += copy;
448                         }
449                     }
450                     case STRUCT_HFA -> {
451                         // Struct is passed with each element in a separate floating
452                         // point register.
453                         MemorySegment valueSegment = (MemorySegment) value;
454                         GroupLayout group = (GroupLayout)layout;
455                         long offset = 0;
456                         for (MemoryLayout elem : group.memberLayouts()) {
457                             assert elem.byteSize() <= 8;
458                             final long copy = elem.byteSize();
459                             MemorySegment.copy(valueSegment, offset, fpRegs, currentFPOffset, copy);
460                             currentFPOffset += FP_SLOT_SIZE;
461                             offset += copy;
462                         }
463                     }
464                     case STRUCT_REFERENCE -> {
465                         // Struct is passed indirectly via a pointer in an integer register.
466                         MemorySegment valueSegment = (MemorySegment) value;
467                         VarHandle writer = AArch64.C_POINTER.varHandle();
468                         writer.set(gpRegs.asSlice(currentGPOffset),
469                                    valueSegment.address());
470                         currentGPOffset += GP_SLOT_SIZE;
471                     }
472                     case POINTER, INTEGER -> {
473                         VarHandle writer = layout.varHandle();
474                         writer.set(gpRegs.asSlice(currentGPOffset), value);
475                         currentGPOffset += GP_SLOT_SIZE;
476                     }
477                     case FLOAT -> {
478                         VarHandle writer = layout.varHandle();
479                         writer.set(fpRegs.asSlice(currentFPOffset), value);
480                         currentFPOffset += FP_SLOT_SIZE;
481                     }
482                 }
483             }
484             return this;
485         }
486 
487         private boolean isEmpty() {
488             return currentGPOffset == 0 && currentFPOffset == 0 && stackArgs.isEmpty();
489         }
490 
491         public VaList build() {
492             if (isEmpty()) {
493                 return EMPTY;
494             }
495 
496             SegmentAllocator allocator = SegmentAllocator.newNativeArena(scope);
497             MemorySegment vaListSegment = allocator.allocate(LAYOUT);
498             MemoryAddress stackArgsPtr = MemoryAddress.NULL;
499             if (!stackArgs.isEmpty()) {
500                 long stackArgsSize = stackArgs.stream()
501                     .reduce(0L, (acc, e) -> acc + Utils.alignUp(e.layout.byteSize(), 8), Long::sum);
502                 MemorySegment stackArgsSegment = allocator.allocate(stackArgsSize, 16);
503                 stackArgsPtr = stackArgsSegment.address();
504                 for (SimpleVaArg arg : stackArgs) {
505                     final long alignedSize = Utils.alignUp(arg.layout.byteSize(), 8);
506                     stackArgsSegment = Utils.alignUp(stackArgsSegment, alignedSize);
507                     VarHandle writer = arg.varHandle();
508                     writer.set(stackArgsSegment, arg.value);
509                     stackArgsSegment = stackArgsSegment.asSlice(alignedSize);
510                 }
511             }
512 
513             VH_gr_top.set(vaListSegment, gpRegs.asSlice(gpRegs.byteSize()).address());
514             VH_vr_top.set(vaListSegment, fpRegs.asSlice(fpRegs.byteSize()).address());
515             VH_stack.set(vaListSegment, stackArgsPtr);
516             VH_gr_offs.set(vaListSegment, -MAX_GP_OFFSET);
517             VH_vr_offs.set(vaListSegment, -MAX_FP_OFFSET);
518 
519             assert gpRegs.scope().ownerThread() == vaListSegment.scope().ownerThread();
520             assert fpRegs.scope().ownerThread() == vaListSegment.scope().ownerThread();
521             return new LinuxAArch64VaList(vaListSegment, gpRegs, fpRegs);
522         }
523     }
524 }
--- EOF ---