< prev index next >

src/jdk.incubator.foreign/share/classes/jdk/internal/foreign/abi/aarch64/linux/LinuxAArch64VaList.java

Print this page

  9  * particular file as subject to the "Classpath" exception as provided
 10  * by Oracle in the LICENSE file that accompanied this code.
 11  *
 12  * This code is distributed in the hope that it will be useful, but WITHOUT
 13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 15  * version 2 for more details (a copy is included in the LICENSE file that
 16  * accompanied this code).
 17  *
 18  * You should have received a copy of the GNU General Public License version
 19  * 2 along with this work; if not, write to the Free Software Foundation,
 20  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 21  *
 22  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 23  * or visit www.oracle.com if you need additional information or have any
 24  * questions.
 25  */
 26 package jdk.internal.foreign.abi.aarch64.linux;
 27 
 28 import jdk.incubator.foreign.*;


 29 import jdk.internal.foreign.Utils;
 30 import jdk.internal.foreign.abi.SharedUtils;
 31 import jdk.internal.foreign.abi.aarch64.*;
 32 import jdk.internal.misc.Unsafe;
 33 
 34 import java.lang.invoke.VarHandle;
 35 import java.lang.ref.Cleaner;
 36 import java.nio.ByteOrder;
 37 import java.util.ArrayList;
 38 import java.util.List;
 39 import java.util.Objects;
 40 
 41 import static jdk.internal.foreign.PlatformLayouts.AArch64;
 42 import static jdk.incubator.foreign.CLinker.VaList;
 43 import static jdk.incubator.foreign.MemoryLayout.PathElement.groupElement;
 44 import static jdk.internal.foreign.abi.SharedUtils.SimpleVaArg;
 45 import static jdk.internal.foreign.abi.SharedUtils.THROWING_ALLOCATOR;
 46 import static jdk.internal.foreign.abi.SharedUtils.checkCompatibleType;
 47 import static jdk.internal.foreign.abi.SharedUtils.vhPrimitiveOrAddress;
 48 import static jdk.internal.foreign.abi.aarch64.CallArranger.MAX_REGISTER_ARGUMENTS;
 49 
 50 /**
 51  * Standard va_list implementation as defined by AAPCS document and used on
 52  * Linux. Variadic parameters may be passed in registers or on the stack.
 53  */
 54 public non-sealed class LinuxAArch64VaList implements VaList {
 55     private static final Unsafe U = Unsafe.getUnsafe();
 56 
 57     static final Class<?> CARRIER = MemoryAddress.class;
 58 
 59     // See AAPCS Appendix B "Variable Argument Lists" for definition of
 60     // va_list on AArch64.
 61     //
 62     // typedef struct __va_list {
 63     //     void *__stack;   // next stack param
 64     //     void *__gr_top;  // end of GP arg reg save area
 65     //     void *__vr_top;  // end of FP/SIMD arg reg save area
 66     //     int __gr_offs;   // offset from __gr_top to next GP register arg
 67     //     int __vr_offs;   // offset from __vr_top to next FP/SIMD register arg
 68     // } va_list;
 69 
 70     static final GroupLayout LAYOUT = MemoryLayout.structLayout(
 71         AArch64.C_POINTER.withName("__stack"),
 72         AArch64.C_POINTER.withName("__gr_top"),
 73         AArch64.C_POINTER.withName("__vr_top"),
 74         AArch64.C_INT.withName("__gr_offs"),
 75         AArch64.C_INT.withName("__vr_offs")
 76     ).withName("__va_list");
 77 
 78     private static final MemoryLayout GP_REG
 79         = MemoryLayout.valueLayout(64, ByteOrder.nativeOrder());
 80     private static final MemoryLayout FP_REG
 81         = MemoryLayout.valueLayout(128, ByteOrder.nativeOrder());
 82 
 83     private static final MemoryLayout LAYOUT_GP_REGS
 84         = MemoryLayout.sequenceLayout(MAX_REGISTER_ARGUMENTS, GP_REG);
 85     private static final MemoryLayout LAYOUT_FP_REGS
 86         = MemoryLayout.sequenceLayout(MAX_REGISTER_ARGUMENTS, FP_REG);
 87 
 88     private static final int GP_SLOT_SIZE = (int) GP_REG.byteSize();
 89     private static final int FP_SLOT_SIZE = (int) FP_REG.byteSize();
 90 
 91     private static final int MAX_GP_OFFSET = (int) LAYOUT_GP_REGS.byteSize();
 92     private static final int MAX_FP_OFFSET = (int) LAYOUT_FP_REGS.byteSize();
 93 
 94     private static final VarHandle VH_stack
 95         = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("__stack")));
 96     private static final VarHandle VH_gr_top
 97         = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("__gr_top")));
 98     private static final VarHandle VH_vr_top
 99         = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("__vr_top")));
100     private static final VarHandle VH_gr_offs
101         = LAYOUT.varHandle(int.class, groupElement("__gr_offs"));
102     private static final VarHandle VH_vr_offs
103         = LAYOUT.varHandle(int.class, groupElement("__vr_offs"));
104 
105     private static final Cleaner cleaner = Cleaner.create();
106     private static final VaList EMPTY
107         = new SharedUtils.EmptyVaList(emptyListAddress());
108 
109     private final MemorySegment segment;
110     private final MemorySegment gpRegsArea;
111     private final MemorySegment fpRegsArea;
112 
113     private LinuxAArch64VaList(MemorySegment segment, MemorySegment gpRegsArea, MemorySegment fpRegsArea) {
114         this.segment = segment;
115         this.gpRegsArea = gpRegsArea;
116         this.fpRegsArea = fpRegsArea;
117     }
118 
119     private static LinuxAArch64VaList readFromSegment(MemorySegment segment) {
120         MemorySegment gpRegsArea = grTop(segment).addOffset(-MAX_GP_OFFSET).asSegment(
121                 MAX_GP_OFFSET, segment.scope());
122 
123         MemorySegment fpRegsArea = vrTop(segment).addOffset(-MAX_FP_OFFSET).asSegment(
124                 MAX_FP_OFFSET, segment.scope());
125         return new LinuxAArch64VaList(segment, gpRegsArea, fpRegsArea);
126     }
127 
128     private static MemoryAddress emptyListAddress() {
129         long ptr = U.allocateMemory(LAYOUT.byteSize());
130         MemorySegment ms = MemoryAddress.ofLong(ptr).asSegment(
131                 LAYOUT.byteSize(), () -> U.freeMemory(ptr), ResourceScope.newSharedScope());
132         cleaner.register(LinuxAArch64VaList.class, () -> ms.scope().close());

133         VH_stack.set(ms, MemoryAddress.NULL);
134         VH_gr_top.set(ms, MemoryAddress.NULL);
135         VH_vr_top.set(ms, MemoryAddress.NULL);
136         VH_gr_offs.set(ms, 0);
137         VH_vr_offs.set(ms, 0);
138         return ms.address();
139     }
140 
141     public static VaList empty() {
142         return EMPTY;
143     }
144 
145     private MemoryAddress grTop() {
146         return grTop(segment);
147     }
148 
149     private static MemoryAddress grTop(MemorySegment segment) {
150         return (MemoryAddress) VH_gr_top.get(segment);
151     }
152 

198 
199     private long currentFPOffset() {
200         // Offset from start of FP register segment. __vr_top points to the top
201         // (highest address) of the FP registers area. __vr_offs is the negative
202         // offset of next saved register from the top.
203 
204         return fpRegsArea.byteSize() + vrOffs();
205     }
206 
207     private void preAlignStack(MemoryLayout layout) {
208         if (layout.byteAlignment() > 8) {
209             stackPtr(Utils.alignUp(stackPtr(), 16));
210         }
211     }
212 
213     private void postAlignStack(MemoryLayout layout) {
214         stackPtr(Utils.alignUp(stackPtr().addOffset(layout.byteSize()), 8));
215     }
216 
217     @Override
218     public int vargAsInt(MemoryLayout layout) {
219         return (int) read(int.class, layout);
220     }
221 
222     @Override
223     public long vargAsLong(MemoryLayout layout) {
224         return (long) read(long.class, layout);
225     }
226 
227     @Override
228     public double vargAsDouble(MemoryLayout layout) {
229         return (double) read(double.class, layout);
230     }
231 
232     @Override
233     public MemoryAddress vargAsAddress(MemoryLayout layout) {
234         return (MemoryAddress) read(MemoryAddress.class, layout);
235     }
236 
237     @Override
238     public MemorySegment vargAsSegment(MemoryLayout layout, SegmentAllocator allocator) {
239         Objects.requireNonNull(allocator);
240         return (MemorySegment) read(MemorySegment.class, layout, allocator);
241     }
242 
243     @Override
244     public MemorySegment vargAsSegment(MemoryLayout layout, ResourceScope scope) {
245         return vargAsSegment(layout, SegmentAllocator.ofScope(scope));
246     }
247 
248     private Object read(Class<?> carrier, MemoryLayout layout) {
249         return read(carrier, layout, THROWING_ALLOCATOR);
250     }
251 
252     private Object read(Class<?> carrier, MemoryLayout layout, SegmentAllocator allocator) {
253         Objects.requireNonNull(layout);
254         checkCompatibleType(carrier, layout, LinuxAArch64Linker.ADDRESS_SIZE);
255 
256         TypeClass typeClass = TypeClass.classifyLayout(layout);
257         if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass, layout)) {
258             preAlignStack(layout);
259             return switch (typeClass) {
260                 case STRUCT_REGISTER, STRUCT_HFA, STRUCT_REFERENCE -> {
261                     MemorySegment slice = stackPtr().asSegment(layout.byteSize(), scope());
262                     MemorySegment seg = allocator.allocate(layout);
263                     seg.copyFrom(slice);
264                     postAlignStack(layout);
265                     yield seg;
266                 }
267                 case POINTER, INTEGER, FLOAT -> {
268                     VarHandle reader = vhPrimitiveOrAddress(carrier, layout);
269                     MemorySegment slice = stackPtr().asSegment(layout.byteSize(), scope());
270                     Object res = reader.get(slice);
271                     postAlignStack(layout);
272                     yield res;
273                 }
274             };
275         } else {
276             return switch (typeClass) {
277                 case STRUCT_REGISTER -> {
278                     // Struct is passed packed in integer registers.
279                     MemorySegment value = allocator.allocate(layout);
280                     long offset = 0;
281                     while (offset < layout.byteSize()) {
282                         final long copy = Math.min(layout.byteSize() - offset, 8);
283                         MemorySegment slice = value.asSlice(offset, copy);
284                         slice.copyFrom(gpRegsArea.asSlice(currentGPOffset(), copy));
285                         consumeGPSlots(1);
286                         offset += copy;
287                     }
288                     yield value;
289                 }
290                 case STRUCT_HFA -> {
291                     // Struct is passed with each element in a separate floating
292                     // point register.
293                     MemorySegment value = allocator.allocate(layout);
294                     GroupLayout group = (GroupLayout)layout;
295                     long offset = 0;
296                     for (MemoryLayout elem : group.memberLayouts()) {
297                         assert elem.byteSize() <= 8;
298                         final long copy = elem.byteSize();
299                         MemorySegment slice = value.asSlice(offset, copy);
300                         slice.copyFrom(fpRegsArea.asSlice(currentFPOffset(), copy));
301                         consumeFPSlots(1);
302                         offset += copy;
303                     }
304                     yield value;
305                 }
306                 case STRUCT_REFERENCE -> {
307                     // Struct is passed indirectly via a pointer in an integer register.
308                     VarHandle ptrReader
309                         = SharedUtils.vhPrimitiveOrAddress(MemoryAddress.class, AArch64.C_POINTER);
310                     MemoryAddress ptr = (MemoryAddress) ptrReader.get(
311                         gpRegsArea.asSlice(currentGPOffset()));
312                     consumeGPSlots(1);
313 
314                     MemorySegment slice = ptr.asSegment(layout.byteSize(), scope());
315                     MemorySegment seg = allocator.allocate(layout);
316                     seg.copyFrom(slice);
317                     yield seg;
318                 }
319                 case POINTER, INTEGER -> {
320                     VarHandle reader = SharedUtils.vhPrimitiveOrAddress(carrier, layout);
321                     Object res = reader.get(gpRegsArea.asSlice(currentGPOffset()));
322                     consumeGPSlots(1);
323                     yield res;
324                 }
325                 case FLOAT -> {
326                     VarHandle reader = layout.varHandle(carrier);
327                     Object res = reader.get(fpRegsArea.asSlice(currentFPOffset()));
328                     consumeFPSlots(1);
329                     yield res;
330                 }
331             };
332         }
333     }
334 
335     @Override
336     public void skip(MemoryLayout... layouts) {
337         Objects.requireNonNull(layouts);

338         for (MemoryLayout layout : layouts) {
339             Objects.requireNonNull(layout);
340             TypeClass typeClass = TypeClass.classifyLayout(layout);
341             if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass, layout)) {
342                 preAlignStack(layout);
343                 postAlignStack(layout);
344             } else if (typeClass == TypeClass.FLOAT || typeClass == TypeClass.STRUCT_HFA) {
345                 consumeFPSlots(numSlots(layout));
346             } else if (typeClass == TypeClass.STRUCT_REFERENCE) {
347                 consumeGPSlots(1);
348             } else {
349                 consumeGPSlots(numSlots(layout));
350             }
351         }
352     }
353 
354     static LinuxAArch64VaList.Builder builder(ResourceScope scope) {
355         return new LinuxAArch64VaList.Builder(scope);
356     }
357 
358     public static VaList ofAddress(MemoryAddress ma, ResourceScope scope) {
359         return readFromSegment(ma.asSegment(LAYOUT.byteSize(), scope));
360     }
361 
362     @Override
363     public ResourceScope scope() {
364         return segment.scope();
365     }
366 
367     @Override
368     public VaList copy() {
369         MemorySegment copy = MemorySegment.allocateNative(LAYOUT, segment.scope());
370         copy.copyFrom(segment);
371         return new LinuxAArch64VaList(copy, gpRegsArea, fpRegsArea);
372     }
373 
374     @Override
375     public MemoryAddress address() {
376         return segment.address();
377     }
378 
379     private static int numSlots(MemoryLayout layout) {

401             + ", __vr_offs=" + vrOffs()
402             + '}';
403     }
404 
405     public static non-sealed class Builder implements VaList.Builder {
406         private final ResourceScope scope;
407         private final MemorySegment gpRegs;
408         private final MemorySegment fpRegs;
409 
410         private long currentGPOffset = 0;
411         private long currentFPOffset = 0;
412         private final List<SimpleVaArg> stackArgs = new ArrayList<>();
413 
414         Builder(ResourceScope scope) {
415             this.scope = scope;
416             this.gpRegs = MemorySegment.allocateNative(LAYOUT_GP_REGS, scope);
417             this.fpRegs = MemorySegment.allocateNative(LAYOUT_FP_REGS, scope);
418         }
419 
420         @Override
421         public Builder vargFromInt(ValueLayout layout, int value) {
422             return arg(int.class, layout, value);
423         }
424 
425         @Override
426         public Builder vargFromLong(ValueLayout layout, long value) {
427             return arg(long.class, layout, value);
428         }
429 
430         @Override
431         public Builder vargFromDouble(ValueLayout layout, double value) {
432             return arg(double.class, layout, value);
433         }
434 
435         @Override
436         public Builder vargFromAddress(ValueLayout layout, Addressable value) {
437             return arg(MemoryAddress.class, layout, value.address());
438         }
439 
440         @Override
441         public Builder vargFromSegment(GroupLayout layout, MemorySegment value) {
442             return arg(MemorySegment.class, layout, value);
443         }
444 
445         private Builder arg(Class<?> carrier, MemoryLayout layout, Object value) {
446             Objects.requireNonNull(layout);
447             Objects.requireNonNull(value);
448             checkCompatibleType(carrier, layout, LinuxAArch64Linker.ADDRESS_SIZE);
449 
450             TypeClass typeClass = TypeClass.classifyLayout(layout);
451             if (isRegOverflow(currentGPOffset, currentFPOffset, typeClass, layout)) {
452                 stackArgs.add(new SimpleVaArg(carrier, layout, value));
453             } else {
454                 switch (typeClass) {
455                     case STRUCT_REGISTER -> {
456                         // Struct is passed packed in integer registers.
457                         MemorySegment valueSegment = (MemorySegment) value;
458                         long offset = 0;
459                         while (offset < layout.byteSize()) {
460                             final long copy = Math.min(layout.byteSize() - offset, 8);
461                             MemorySegment slice = valueSegment.asSlice(offset, copy);
462                             gpRegs.asSlice(currentGPOffset, copy).copyFrom(slice);
463                             currentGPOffset += GP_SLOT_SIZE;
464                             offset += copy;
465                         }
466                     }
467                     case STRUCT_HFA -> {
468                         // Struct is passed with each element in a separate floating
469                         // point register.
470                         MemorySegment valueSegment = (MemorySegment) value;
471                         GroupLayout group = (GroupLayout)layout;
472                         long offset = 0;
473                         for (MemoryLayout elem : group.memberLayouts()) {
474                             assert elem.byteSize() <= 8;
475                             final long copy = elem.byteSize();
476                             MemorySegment slice = valueSegment.asSlice(offset, copy);
477                             fpRegs.asSlice(currentFPOffset, copy).copyFrom(slice);
478                             currentFPOffset += FP_SLOT_SIZE;
479                             offset += copy;
480                         }
481                     }
482                     case STRUCT_REFERENCE -> {
483                         // Struct is passed indirectly via a pointer in an integer register.
484                         MemorySegment valueSegment = (MemorySegment) value;
485                         VarHandle writer
486                             = SharedUtils.vhPrimitiveOrAddress(MemoryAddress.class,
487                                                                AArch64.C_POINTER);
488                         writer.set(gpRegs.asSlice(currentGPOffset),
489                                    valueSegment.address());
490                         currentGPOffset += GP_SLOT_SIZE;
491                     }
492                     case POINTER, INTEGER -> {
493                         VarHandle writer = SharedUtils.vhPrimitiveOrAddress(carrier, layout);
494                         writer.set(gpRegs.asSlice(currentGPOffset), value);
495                         currentGPOffset += GP_SLOT_SIZE;
496                     }
497                     case FLOAT -> {
498                         VarHandle writer = layout.varHandle(carrier);
499                         writer.set(fpRegs.asSlice(currentFPOffset), value);
500                         currentFPOffset += FP_SLOT_SIZE;
501                     }
502                 }
503             }
504             return this;
505         }
506 
507         private boolean isEmpty() {
508             return currentGPOffset == 0 && currentFPOffset == 0 && stackArgs.isEmpty();
509         }
510 
511         public VaList build() {
512             if (isEmpty()) {
513                 return EMPTY;
514             }
515 
516             SegmentAllocator allocator = SegmentAllocator.arenaAllocator(scope);
517             MemorySegment vaListSegment = allocator.allocate(LAYOUT);
518             MemoryAddress stackArgsPtr = MemoryAddress.NULL;
519             if (!stackArgs.isEmpty()) {
520                 long stackArgsSize = stackArgs.stream()
521                     .reduce(0L, (acc, e) -> acc + Utils.alignUp(e.layout.byteSize(), 8), Long::sum);
522                 MemorySegment stackArgsSegment = allocator.allocate(stackArgsSize, 16);
523                 stackArgsPtr = stackArgsSegment.address();
524                 for (SimpleVaArg arg : stackArgs) {
525                     final long alignedSize = Utils.alignUp(arg.layout.byteSize(), 8);
526                     stackArgsSegment = Utils.alignUp(stackArgsSegment, alignedSize);
527                     VarHandle writer = arg.varHandle();
528                     writer.set(stackArgsSegment, arg.value);
529                     stackArgsSegment = stackArgsSegment.asSlice(alignedSize);
530                 }
531             }
532 
533             VH_gr_top.set(vaListSegment, gpRegs.asSlice(gpRegs.byteSize()).address());
534             VH_vr_top.set(vaListSegment, fpRegs.asSlice(fpRegs.byteSize()).address());
535             VH_stack.set(vaListSegment, stackArgsPtr);
536             VH_gr_offs.set(vaListSegment, -MAX_GP_OFFSET);

  9  * particular file as subject to the "Classpath" exception as provided
 10  * by Oracle in the LICENSE file that accompanied this code.
 11  *
 12  * This code is distributed in the hope that it will be useful, but WITHOUT
 13  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 14  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 15  * version 2 for more details (a copy is included in the LICENSE file that
 16  * accompanied this code).
 17  *
 18  * You should have received a copy of the GNU General Public License version
 19  * 2 along with this work; if not, write to the Free Software Foundation,
 20  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 21  *
 22  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 23  * or visit www.oracle.com if you need additional information or have any
 24  * questions.
 25  */
 26 package jdk.internal.foreign.abi.aarch64.linux;
 27 
 28 import jdk.incubator.foreign.*;
 29 import jdk.internal.foreign.ResourceScopeImpl;
 30 import jdk.internal.foreign.Scoped;
 31 import jdk.internal.foreign.Utils;
 32 import jdk.internal.foreign.abi.SharedUtils;
 33 import jdk.internal.foreign.abi.aarch64.*;
 34 import jdk.internal.misc.Unsafe;
 35 
 36 import java.lang.invoke.VarHandle;
 37 import java.lang.ref.Cleaner;

 38 import java.util.ArrayList;
 39 import java.util.List;
 40 import java.util.Objects;
 41 
 42 import static jdk.internal.foreign.PlatformLayouts.AArch64;
 43 
 44 import static jdk.incubator.foreign.MemoryLayout.PathElement.groupElement;
 45 import static jdk.internal.foreign.abi.SharedUtils.SimpleVaArg;
 46 import static jdk.internal.foreign.abi.SharedUtils.THROWING_ALLOCATOR;


 47 import static jdk.internal.foreign.abi.aarch64.CallArranger.MAX_REGISTER_ARGUMENTS;
 48 
 49 /**
 50  * Standard va_list implementation as defined by AAPCS document and used on
 51  * Linux. Variadic parameters may be passed in registers or on the stack.
 52  */
 53 public non-sealed class LinuxAArch64VaList implements VaList, Scoped {
 54     private static final Unsafe U = Unsafe.getUnsafe();
 55 
 56     static final Class<?> CARRIER = MemoryAddress.class;
 57 
 58     // See AAPCS Appendix B "Variable Argument Lists" for definition of
 59     // va_list on AArch64.
 60     //
 61     // typedef struct __va_list {
 62     //     void *__stack;   // next stack param
 63     //     void *__gr_top;  // end of GP arg reg save area
 64     //     void *__vr_top;  // end of FP/SIMD arg reg save area
 65     //     int __gr_offs;   // offset from __gr_top to next GP register arg
 66     //     int __vr_offs;   // offset from __vr_top to next FP/SIMD register arg
 67     // } va_list;
 68 
 69     static final GroupLayout LAYOUT = MemoryLayout.structLayout(
 70         AArch64.C_POINTER.withName("__stack"),
 71         AArch64.C_POINTER.withName("__gr_top"),
 72         AArch64.C_POINTER.withName("__vr_top"),
 73         AArch64.C_INT.withName("__gr_offs"),
 74         AArch64.C_INT.withName("__vr_offs")
 75     ).withName("__va_list");
 76 
 77     private static final MemoryLayout GP_REG
 78         = MemoryLayout.paddingLayout(64).withBitAlignment(64);
 79     private static final MemoryLayout FP_REG
 80         = MemoryLayout.paddingLayout(128).withBitAlignment(128);
 81 
 82     private static final MemoryLayout LAYOUT_GP_REGS
 83         = MemoryLayout.sequenceLayout(MAX_REGISTER_ARGUMENTS, GP_REG);
 84     private static final MemoryLayout LAYOUT_FP_REGS
 85         = MemoryLayout.sequenceLayout(MAX_REGISTER_ARGUMENTS, FP_REG);
 86 
 87     private static final int GP_SLOT_SIZE = (int) GP_REG.byteSize();
 88     private static final int FP_SLOT_SIZE = (int) FP_REG.byteSize();
 89 
 90     private static final int MAX_GP_OFFSET = (int) LAYOUT_GP_REGS.byteSize();
 91     private static final int MAX_FP_OFFSET = (int) LAYOUT_FP_REGS.byteSize();
 92 
 93     private static final VarHandle VH_stack = LAYOUT.varHandle(groupElement("__stack"));
 94     private static final VarHandle VH_gr_top = LAYOUT.varHandle(groupElement("__gr_top"));
 95     private static final VarHandle VH_vr_top = LAYOUT.varHandle(groupElement("__vr_top"));



 96     private static final VarHandle VH_gr_offs
 97         = LAYOUT.varHandle(groupElement("__gr_offs"));
 98     private static final VarHandle VH_vr_offs
 99         = LAYOUT.varHandle(groupElement("__vr_offs"));
100 

101     private static final VaList EMPTY
102         = new SharedUtils.EmptyVaList(emptyListAddress());
103 
104     private final MemorySegment segment;
105     private final MemorySegment gpRegsArea;
106     private final MemorySegment fpRegsArea;
107 
108     private LinuxAArch64VaList(MemorySegment segment, MemorySegment gpRegsArea, MemorySegment fpRegsArea) {
109         this.segment = segment;
110         this.gpRegsArea = gpRegsArea;
111         this.fpRegsArea = fpRegsArea;
112     }
113 
114     private static LinuxAArch64VaList readFromSegment(MemorySegment segment) {
115         MemorySegment gpRegsArea = MemorySegment.ofAddressNative(grTop(segment).addOffset(-MAX_GP_OFFSET),
116                 MAX_GP_OFFSET, segment.scope());
117 
118         MemorySegment fpRegsArea = MemorySegment.ofAddressNative(vrTop(segment).addOffset(-MAX_FP_OFFSET),
119                 MAX_FP_OFFSET, segment.scope());
120         return new LinuxAArch64VaList(segment, gpRegsArea, fpRegsArea);
121     }
122 
123     private static MemoryAddress emptyListAddress() {
124         long ptr = U.allocateMemory(LAYOUT.byteSize());
125         ResourceScope scope = ResourceScope.newImplicitScope();
126         scope.addCloseAction(() -> U.freeMemory(ptr));
127         MemorySegment ms = MemorySegment.ofAddressNative(MemoryAddress.ofLong(ptr),
128                 LAYOUT.byteSize(), scope);
129         VH_stack.set(ms, MemoryAddress.NULL);
130         VH_gr_top.set(ms, MemoryAddress.NULL);
131         VH_vr_top.set(ms, MemoryAddress.NULL);
132         VH_gr_offs.set(ms, 0);
133         VH_vr_offs.set(ms, 0);
134         return ms.address();
135     }
136 
137     public static VaList empty() {
138         return EMPTY;
139     }
140 
141     private MemoryAddress grTop() {
142         return grTop(segment);
143     }
144 
145     private static MemoryAddress grTop(MemorySegment segment) {
146         return (MemoryAddress) VH_gr_top.get(segment);
147     }
148 

194 
195     private long currentFPOffset() {
196         // Offset from start of FP register segment. __vr_top points to the top
197         // (highest address) of the FP registers area. __vr_offs is the negative
198         // offset of next saved register from the top.
199 
200         return fpRegsArea.byteSize() + vrOffs();
201     }
202 
203     private void preAlignStack(MemoryLayout layout) {
204         if (layout.byteAlignment() > 8) {
205             stackPtr(Utils.alignUp(stackPtr(), 16));
206         }
207     }
208 
209     private void postAlignStack(MemoryLayout layout) {
210         stackPtr(Utils.alignUp(stackPtr().addOffset(layout.byteSize()), 8));
211     }
212 
213     @Override
214     public int nextVarg(ValueLayout.OfInt layout) {
215         return (int) read(int.class, layout);
216     }
217 
218     @Override
219     public long nextVarg(ValueLayout.OfLong layout) {
220         return (long) read(long.class, layout);
221     }
222 
223     @Override
224     public double nextVarg(ValueLayout.OfDouble layout) {
225         return (double) read(double.class, layout);
226     }
227 
228     @Override
229     public MemoryAddress nextVarg(ValueLayout.OfAddress layout) {
230         return (MemoryAddress) read(MemoryAddress.class, layout);
231     }
232 
233     @Override
234     public MemorySegment nextVarg(GroupLayout layout, SegmentAllocator allocator) {
235         Objects.requireNonNull(allocator);
236         return (MemorySegment) read(MemorySegment.class, layout, allocator);
237     }
238 





239     private Object read(Class<?> carrier, MemoryLayout layout) {
240         return read(carrier, layout, THROWING_ALLOCATOR);
241     }
242 
243     private Object read(Class<?> carrier, MemoryLayout layout, SegmentAllocator allocator) {
244         Objects.requireNonNull(layout);


245         TypeClass typeClass = TypeClass.classifyLayout(layout);
246         if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass, layout)) {
247             preAlignStack(layout);
248             return switch (typeClass) {
249                 case STRUCT_REGISTER, STRUCT_HFA, STRUCT_REFERENCE -> {
250                     MemorySegment slice = MemorySegment.ofAddressNative(stackPtr(), layout.byteSize(), scope());
251                     MemorySegment seg = allocator.allocate(layout);
252                     seg.copyFrom(slice);
253                     postAlignStack(layout);
254                     yield seg;
255                 }
256                 case POINTER, INTEGER, FLOAT -> {
257                     VarHandle reader = layout.varHandle();
258                     MemorySegment slice = MemorySegment.ofAddressNative(stackPtr(), layout.byteSize(), scope());
259                     Object res = reader.get(slice);
260                     postAlignStack(layout);
261                     yield res;
262                 }
263             };
264         } else {
265             return switch (typeClass) {
266                 case STRUCT_REGISTER -> {
267                     // Struct is passed packed in integer registers.
268                     MemorySegment value = allocator.allocate(layout);
269                     long offset = 0;
270                     while (offset < layout.byteSize()) {
271                         final long copy = Math.min(layout.byteSize() - offset, 8);
272                         MemorySegment.copy(gpRegsArea, currentGPOffset(), value, offset, copy);

273                         consumeGPSlots(1);
274                         offset += copy;
275                     }
276                     yield value;
277                 }
278                 case STRUCT_HFA -> {
279                     // Struct is passed with each element in a separate floating
280                     // point register.
281                     MemorySegment value = allocator.allocate(layout);
282                     GroupLayout group = (GroupLayout)layout;
283                     long offset = 0;
284                     for (MemoryLayout elem : group.memberLayouts()) {
285                         assert elem.byteSize() <= 8;
286                         final long copy = elem.byteSize();
287                         MemorySegment.copy(fpRegsArea, currentFPOffset(), value, offset, copy);

288                         consumeFPSlots(1);
289                         offset += copy;
290                     }
291                     yield value;
292                 }
293                 case STRUCT_REFERENCE -> {
294                     // Struct is passed indirectly via a pointer in an integer register.
295                     VarHandle ptrReader = AArch64.C_POINTER.varHandle();

296                     MemoryAddress ptr = (MemoryAddress) ptrReader.get(
297                         gpRegsArea.asSlice(currentGPOffset()));
298                     consumeGPSlots(1);
299 
300                     MemorySegment slice = MemorySegment.ofAddressNative(ptr, layout.byteSize(), scope());
301                     MemorySegment seg = allocator.allocate(layout);
302                     seg.copyFrom(slice);
303                     yield seg;
304                 }
305                 case POINTER, INTEGER -> {
306                     VarHandle reader = layout.varHandle();
307                     Object res = reader.get(gpRegsArea.asSlice(currentGPOffset()));
308                     consumeGPSlots(1);
309                     yield res;
310                 }
311                 case FLOAT -> {
312                     VarHandle reader = layout.varHandle();
313                     Object res = reader.get(fpRegsArea.asSlice(currentFPOffset()));
314                     consumeFPSlots(1);
315                     yield res;
316                 }
317             };
318         }
319     }
320 
321     @Override
322     public void skip(MemoryLayout... layouts) {
323         Objects.requireNonNull(layouts);
324         ((ResourceScopeImpl)segment.scope()).checkValidStateSlow();
325         for (MemoryLayout layout : layouts) {
326             Objects.requireNonNull(layout);
327             TypeClass typeClass = TypeClass.classifyLayout(layout);
328             if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass, layout)) {
329                 preAlignStack(layout);
330                 postAlignStack(layout);
331             } else if (typeClass == TypeClass.FLOAT || typeClass == TypeClass.STRUCT_HFA) {
332                 consumeFPSlots(numSlots(layout));
333             } else if (typeClass == TypeClass.STRUCT_REFERENCE) {
334                 consumeGPSlots(1);
335             } else {
336                 consumeGPSlots(numSlots(layout));
337             }
338         }
339     }
340 
341     static LinuxAArch64VaList.Builder builder(ResourceScope scope) {
342         return new LinuxAArch64VaList.Builder(scope);
343     }
344 
345     public static VaList ofAddress(MemoryAddress ma, ResourceScope scope) {
346         return readFromSegment(MemorySegment.ofAddressNative(ma, LAYOUT.byteSize(), scope));
347     }
348 
349     @Override
350     public ResourceScope scope() {
351         return segment.scope();
352     }
353 
354     @Override
355     public VaList copy() {
356         MemorySegment copy = MemorySegment.allocateNative(LAYOUT, segment.scope());
357         copy.copyFrom(segment);
358         return new LinuxAArch64VaList(copy, gpRegsArea, fpRegsArea);
359     }
360 
361     @Override
362     public MemoryAddress address() {
363         return segment.address();
364     }
365 
366     private static int numSlots(MemoryLayout layout) {

388             + ", __vr_offs=" + vrOffs()
389             + '}';
390     }
391 
392     public static non-sealed class Builder implements VaList.Builder {
393         private final ResourceScope scope;
394         private final MemorySegment gpRegs;
395         private final MemorySegment fpRegs;
396 
397         private long currentGPOffset = 0;
398         private long currentFPOffset = 0;
399         private final List<SimpleVaArg> stackArgs = new ArrayList<>();
400 
401         Builder(ResourceScope scope) {
402             this.scope = scope;
403             this.gpRegs = MemorySegment.allocateNative(LAYOUT_GP_REGS, scope);
404             this.fpRegs = MemorySegment.allocateNative(LAYOUT_FP_REGS, scope);
405         }
406 
407         @Override
408         public Builder addVarg(ValueLayout.OfInt layout, int value) {
409             return arg(int.class, layout, value);
410         }
411 
412         @Override
413         public Builder addVarg(ValueLayout.OfLong layout, long value) {
414             return arg(long.class, layout, value);
415         }
416 
417         @Override
418         public Builder addVarg(ValueLayout.OfDouble layout, double value) {
419             return arg(double.class, layout, value);
420         }
421 
422         @Override
423         public Builder addVarg(ValueLayout.OfAddress layout, Addressable value) {
424             return arg(MemoryAddress.class, layout, value.address());
425         }
426 
427         @Override
428         public Builder addVarg(GroupLayout layout, MemorySegment value) {
429             return arg(MemorySegment.class, layout, value);
430         }
431 
432         private Builder arg(Class<?> carrier, MemoryLayout layout, Object value) {
433             Objects.requireNonNull(layout);
434             Objects.requireNonNull(value);


435             TypeClass typeClass = TypeClass.classifyLayout(layout);
436             if (isRegOverflow(currentGPOffset, currentFPOffset, typeClass, layout)) {
437                 stackArgs.add(new SimpleVaArg(carrier, layout, value));
438             } else {
439                 switch (typeClass) {
440                     case STRUCT_REGISTER -> {
441                         // Struct is passed packed in integer registers.
442                         MemorySegment valueSegment = (MemorySegment) value;
443                         long offset = 0;
444                         while (offset < layout.byteSize()) {
445                             final long copy = Math.min(layout.byteSize() - offset, 8);
446                             MemorySegment.copy(valueSegment, offset, gpRegs, currentGPOffset, copy);

447                             currentGPOffset += GP_SLOT_SIZE;
448                             offset += copy;
449                         }
450                     }
451                     case STRUCT_HFA -> {
452                         // Struct is passed with each element in a separate floating
453                         // point register.
454                         MemorySegment valueSegment = (MemorySegment) value;
455                         GroupLayout group = (GroupLayout)layout;
456                         long offset = 0;
457                         for (MemoryLayout elem : group.memberLayouts()) {
458                             assert elem.byteSize() <= 8;
459                             final long copy = elem.byteSize();
460                             MemorySegment.copy(valueSegment, offset, fpRegs, currentFPOffset, copy);

461                             currentFPOffset += FP_SLOT_SIZE;
462                             offset += copy;
463                         }
464                     }
465                     case STRUCT_REFERENCE -> {
466                         // Struct is passed indirectly via a pointer in an integer register.
467                         MemorySegment valueSegment = (MemorySegment) value;
468                         VarHandle writer = AArch64.C_POINTER.varHandle();


469                         writer.set(gpRegs.asSlice(currentGPOffset),
470                                    valueSegment.address());
471                         currentGPOffset += GP_SLOT_SIZE;
472                     }
473                     case POINTER, INTEGER -> {
474                         VarHandle writer = layout.varHandle();
475                         writer.set(gpRegs.asSlice(currentGPOffset), value);
476                         currentGPOffset += GP_SLOT_SIZE;
477                     }
478                     case FLOAT -> {
479                         VarHandle writer = layout.varHandle();
480                         writer.set(fpRegs.asSlice(currentFPOffset), value);
481                         currentFPOffset += FP_SLOT_SIZE;
482                     }
483                 }
484             }
485             return this;
486         }
487 
488         private boolean isEmpty() {
489             return currentGPOffset == 0 && currentFPOffset == 0 && stackArgs.isEmpty();
490         }
491 
492         public VaList build() {
493             if (isEmpty()) {
494                 return EMPTY;
495             }
496 
497             SegmentAllocator allocator = SegmentAllocator.newNativeArena(scope);
498             MemorySegment vaListSegment = allocator.allocate(LAYOUT);
499             MemoryAddress stackArgsPtr = MemoryAddress.NULL;
500             if (!stackArgs.isEmpty()) {
501                 long stackArgsSize = stackArgs.stream()
502                     .reduce(0L, (acc, e) -> acc + Utils.alignUp(e.layout.byteSize(), 8), Long::sum);
503                 MemorySegment stackArgsSegment = allocator.allocate(stackArgsSize, 16);
504                 stackArgsPtr = stackArgsSegment.address();
505                 for (SimpleVaArg arg : stackArgs) {
506                     final long alignedSize = Utils.alignUp(arg.layout.byteSize(), 8);
507                     stackArgsSegment = Utils.alignUp(stackArgsSegment, alignedSize);
508                     VarHandle writer = arg.varHandle();
509                     writer.set(stackArgsSegment, arg.value);
510                     stackArgsSegment = stackArgsSegment.asSlice(alignedSize);
511                 }
512             }
513 
514             VH_gr_top.set(vaListSegment, gpRegs.asSlice(gpRegs.byteSize()).address());
515             VH_vr_top.set(vaListSegment, fpRegs.asSlice(fpRegs.byteSize()).address());
516             VH_stack.set(vaListSegment, stackArgsPtr);
517             VH_gr_offs.set(vaListSegment, -MAX_GP_OFFSET);
< prev index next >