1 /*
  2  *  Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
  3  *  DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  *  This code is free software; you can redistribute it and/or modify it
  6  *  under the terms of the GNU General Public License version 2 only, as
  7  *  published by the Free Software Foundation.  Oracle designates this
  8  *  particular file as subject to the "Classpath" exception as provided
  9  *  by Oracle in the LICENSE file that accompanied this code.
 10  *
 11  *  This code is distributed in the hope that it will be useful, but WITHOUT
 12  *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  *  version 2 for more details (a copy is included in the LICENSE file that
 15  *  accompanied this code).
 16  *
 17  *  You should have received a copy of the GNU General Public License version
 18  *  2 along with this work; if not, write to the Free Software Foundation,
 19  *  Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  *   Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  *  or visit www.oracle.com if you need additional information or have any
 23  *  questions.
 24  *
 25  */
 26 package jdk.internal.foreign.abi.x64.sysv;
 27 
 28 import jdk.incubator.foreign.*;
 29 import jdk.internal.foreign.Utils;
 30 import jdk.internal.foreign.abi.SharedUtils;
 31 import jdk.internal.misc.Unsafe;
 32 
 33 import java.lang.invoke.VarHandle;
 34 import java.lang.ref.Cleaner;
 35 import java.nio.ByteOrder;
 36 import java.util.ArrayList;
 37 import java.util.List;
 38 import java.util.Objects;
 39 
 40 import static jdk.internal.foreign.PlatformLayouts.SysV;
 41 import static jdk.incubator.foreign.CLinker.VaList;
 42 import static jdk.incubator.foreign.MemoryLayout.PathElement.groupElement;
 43 import static jdk.internal.foreign.abi.SharedUtils.SimpleVaArg;
 44 import static jdk.internal.foreign.abi.SharedUtils.THROWING_ALLOCATOR;
 45 import static jdk.internal.foreign.abi.SharedUtils.checkCompatibleType;
 46 import static jdk.internal.foreign.abi.SharedUtils.vhPrimitiveOrAddress;
 47 
 48 // See https://software.intel.com/sites/default/files/article/402129/mpx-linux64-abi.pdf "3.5.7 Variable Argument Lists"
 49 public non-sealed class SysVVaList implements VaList {
 50     private static final Unsafe U = Unsafe.getUnsafe();
 51 
 52     static final Class<?> CARRIER = MemoryAddress.class;
 53 
 54 //    struct typedef __va_list_tag __va_list_tag {
 55 //        unsigned int               gp_offset;            /*     0     4 */
 56 //        unsigned int               fp_offset;            /*     4     4 */
 57 //        void *                     overflow_arg_area;    /*     8     8 */
 58 //        void *                     reg_save_area;        /*    16     8 */
 59 //
 60 //        /* size: 24, cachelines: 1, members: 4 */
 61 //        /* last cacheline: 24 bytes */
 62 //    };
 63     static final GroupLayout LAYOUT = MemoryLayout.structLayout(
 64         SysV.C_INT.withName("gp_offset"),
 65         SysV.C_INT.withName("fp_offset"),
 66         SysV.C_POINTER.withName("overflow_arg_area"),
 67         SysV.C_POINTER.withName("reg_save_area")
 68     ).withName("__va_list_tag");
 69 
 70     private static final MemoryLayout GP_REG = MemoryLayout.valueLayout(64, ByteOrder.nativeOrder());
 71     private static final MemoryLayout FP_REG = MemoryLayout.valueLayout(128, ByteOrder.nativeOrder());
 72 
 73     private static final GroupLayout LAYOUT_REG_SAVE_AREA = MemoryLayout.structLayout(
 74         GP_REG.withName("%rdi"),
 75         GP_REG.withName("%rsi"),
 76         GP_REG.withName("%rdx"),
 77         GP_REG.withName("%rcx"),
 78         GP_REG.withName("%r8"),
 79         GP_REG.withName("%r9"),
 80         FP_REG.withName("%xmm0"),
 81         FP_REG.withName("%xmm1"),
 82         FP_REG.withName("%xmm2"),
 83         FP_REG.withName("%xmm3"),
 84         FP_REG.withName("%xmm4"),
 85         FP_REG.withName("%xmm5"),
 86         FP_REG.withName("%xmm6"),
 87         FP_REG.withName("%xmm7")
 88 // specification and implementation differ as to whether the following are part of a reg save area
 89 // Let's go with the implementation, since then it actually works :)
 90 //        FP_REG.withName("%xmm8"),
 91 //        FP_REG.withName("%xmm9"),
 92 //        FP_REG.withName("%xmm10"),
 93 //        FP_REG.withName("%xmm11"),
 94 //        FP_REG.withName("%xmm12"),
 95 //        FP_REG.withName("%xmm13"),
 96 //        FP_REG.withName("%xmm14"),
 97 //        FP_REG.withName("%xmm15")
 98     );
 99 
100     private static final long FP_OFFSET = LAYOUT_REG_SAVE_AREA.byteOffset(groupElement("%xmm0"));
101 
102     private static final int GP_SLOT_SIZE = (int) GP_REG.byteSize();
103     private static final int FP_SLOT_SIZE = (int) FP_REG.byteSize();
104 
105     private static final int MAX_GP_OFFSET = (int) FP_OFFSET; // 6 regs used
106     private static final int MAX_FP_OFFSET = (int) LAYOUT_REG_SAVE_AREA.byteSize(); // 8 16 byte regs
107 
108     private static final VarHandle VH_fp_offset = LAYOUT.varHandle(int.class, groupElement("fp_offset"));
109     private static final VarHandle VH_gp_offset = LAYOUT.varHandle(int.class, groupElement("gp_offset"));
110     private static final VarHandle VH_overflow_arg_area
111         = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("overflow_arg_area")));
112     private static final VarHandle VH_reg_save_area
113         = MemoryHandles.asAddressVarHandle(LAYOUT.varHandle(long.class, groupElement("reg_save_area")));
114 
115     private static final Cleaner cleaner = Cleaner.create();
116     private static final VaList EMPTY = new SharedUtils.EmptyVaList(emptyListAddress());
117 
118     private final MemorySegment segment;
119     private final MemorySegment regSaveArea;
120 
121     private SysVVaList(MemorySegment segment, MemorySegment regSaveArea) {
122         this.segment = segment;
123         this.regSaveArea = regSaveArea;
124     }
125 
126     private static SysVVaList readFromSegment(MemorySegment segment) {
127         MemorySegment regSaveArea = getRegSaveArea(segment);
128         return new SysVVaList(segment, regSaveArea);
129     }
130 
131     private static MemoryAddress emptyListAddress() {
132         long ptr = U.allocateMemory(LAYOUT.byteSize());
133         MemorySegment base = MemoryAddress.ofLong(ptr).asSegment(
134                 LAYOUT.byteSize(), () -> U.freeMemory(ptr), ResourceScope.newSharedScope());
135         cleaner.register(SysVVaList.class, () -> base.scope().close());
136         VH_gp_offset.set(base, MAX_GP_OFFSET);
137         VH_fp_offset.set(base, MAX_FP_OFFSET);
138         VH_overflow_arg_area.set(base, MemoryAddress.NULL);
139         VH_reg_save_area.set(base, MemoryAddress.NULL);
140         return base.address();
141     }
142 
143     public static VaList empty() {
144         return EMPTY;
145     }
146 
147     private int currentGPOffset() {
148         return (int) VH_gp_offset.get(segment);
149     }
150 
151     private void currentGPOffset(int i) {
152         VH_gp_offset.set(segment, i);
153     }
154 
155     private int currentFPOffset() {
156         return (int) VH_fp_offset.get(segment);
157     }
158 
159     private void currentFPOffset(int i) {
160         VH_fp_offset.set(segment, i);
161     }
162 
163     private MemoryAddress stackPtr() {
164         return (MemoryAddress) VH_overflow_arg_area.get(segment);
165     }
166 
167     private void stackPtr(MemoryAddress ptr) {
168         VH_overflow_arg_area.set(segment, ptr);
169     }
170 
171     private MemorySegment regSaveArea() {
172         return getRegSaveArea(segment);
173     }
174 
175     private static MemorySegment getRegSaveArea(MemorySegment segment) {
176         return ((MemoryAddress)VH_reg_save_area.get(segment)).asSegment(
177                 LAYOUT_REG_SAVE_AREA.byteSize(), segment.scope());
178     }
179 
180     private void preAlignStack(MemoryLayout layout) {
181         if (layout.byteAlignment() > 8) {
182             stackPtr(Utils.alignUp(stackPtr(), 16));
183         }
184     }
185 
186     private void postAlignStack(MemoryLayout layout) {
187         stackPtr(Utils.alignUp(stackPtr().addOffset(layout.byteSize()), 8));
188     }
189 
190     @Override
191     public int vargAsInt(MemoryLayout layout) {
192         return (int) read(int.class, layout);
193     }
194 
195     @Override
196     public long vargAsLong(MemoryLayout layout) {
197         return (long) read(long.class, layout);
198     }
199 
200     @Override
201     public double vargAsDouble(MemoryLayout layout) {
202         return (double) read(double.class, layout);
203     }
204 
205     @Override
206     public MemoryAddress vargAsAddress(MemoryLayout layout) {
207         return (MemoryAddress) read(MemoryAddress.class, layout);
208     }
209 
210     @Override
211     public MemorySegment vargAsSegment(MemoryLayout layout, SegmentAllocator allocator) {
212         Objects.requireNonNull(allocator);
213         return (MemorySegment) read(MemorySegment.class, layout, allocator);
214     }
215 
216     @Override
217     public MemorySegment vargAsSegment(MemoryLayout layout, ResourceScope scope) {
218         return vargAsSegment(layout, SegmentAllocator.ofScope(scope));
219     }
220 
221     private Object read(Class<?> carrier, MemoryLayout layout) {
222         return read(carrier, layout, THROWING_ALLOCATOR);
223     }
224 
225     private Object read(Class<?> carrier, MemoryLayout layout, SegmentAllocator allocator) {
226         Objects.requireNonNull(layout);
227         checkCompatibleType(carrier, layout, SysVx64Linker.ADDRESS_SIZE);
228         TypeClass typeClass = TypeClass.classifyLayout(layout);
229         if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)
230                 || typeClass.inMemory()) {
231             preAlignStack(layout);
232             return switch (typeClass.kind()) {
233                 case STRUCT -> {
234                     MemorySegment slice = stackPtr().asSegment(layout.byteSize(), scope());
235                     MemorySegment seg = allocator.allocate(layout);
236                     seg.copyFrom(slice);
237                     postAlignStack(layout);
238                     yield seg;
239                 }
240                 case POINTER, INTEGER, FLOAT -> {
241                     VarHandle reader = vhPrimitiveOrAddress(carrier, layout);
242                     try (ResourceScope localScope = ResourceScope.newConfinedScope()) {
243                         MemorySegment slice = stackPtr().asSegment(layout.byteSize(), localScope);
244                         Object res = reader.get(slice);
245                         postAlignStack(layout);
246                         yield res;
247                     }
248                 }
249             };
250         } else {
251             return switch (typeClass.kind()) {
252                 case STRUCT -> {
253                     MemorySegment value = allocator.allocate(layout);
254                     int classIdx = 0;
255                     long offset = 0;
256                     while (offset < layout.byteSize()) {
257                         final long copy = Math.min(layout.byteSize() - offset, 8);
258                         boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE;
259                         MemorySegment slice = value.asSlice(offset, copy);
260                         if (isSSE) {
261                             slice.copyFrom(regSaveArea.asSlice(currentFPOffset(), copy));
262                             currentFPOffset(currentFPOffset() + FP_SLOT_SIZE);
263                         } else {
264                             slice.copyFrom(regSaveArea.asSlice(currentGPOffset(), copy));
265                             currentGPOffset(currentGPOffset() + GP_SLOT_SIZE);
266                         }
267                         offset += copy;
268                     }
269                     yield value;
270                 }
271                 case POINTER, INTEGER -> {
272                     VarHandle reader = SharedUtils.vhPrimitiveOrAddress(carrier, layout);
273                     Object res = reader.get(regSaveArea.asSlice(currentGPOffset()));
274                     currentGPOffset(currentGPOffset() + GP_SLOT_SIZE);
275                     yield res;
276                 }
277                 case FLOAT -> {
278                     VarHandle reader = layout.varHandle(carrier);
279                     Object res = reader.get(regSaveArea.asSlice(currentFPOffset()));
280                     currentFPOffset(currentFPOffset() + FP_SLOT_SIZE);
281                     yield res;
282                 }
283             };
284         }
285     }
286 
287     @Override
288     public void skip(MemoryLayout... layouts) {
289         Objects.requireNonNull(layouts);
290         for (MemoryLayout layout : layouts) {
291             Objects.requireNonNull(layout);
292             TypeClass typeClass = TypeClass.classifyLayout(layout);
293             if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)) {
294                 preAlignStack(layout);
295                 postAlignStack(layout);
296             } else {
297                 currentGPOffset(currentGPOffset() + (((int) typeClass.nIntegerRegs()) * GP_SLOT_SIZE));
298                 currentFPOffset(currentFPOffset() + (((int) typeClass.nVectorRegs()) * FP_SLOT_SIZE));
299             }
300         }
301     }
302 
303     static SysVVaList.Builder builder(ResourceScope scope) {
304         return new SysVVaList.Builder(scope);
305     }
306 
307     public static VaList ofAddress(MemoryAddress ma, ResourceScope scope) {
308         return readFromSegment(ma.asSegment(LAYOUT.byteSize(), scope));
309     }
310 
311     @Override
312     public ResourceScope scope() {
313         return segment.scope();
314     }
315 
316     @Override
317     public VaList copy() {
318         MemorySegment copy = MemorySegment.allocateNative(LAYOUT, segment.scope());
319         copy.copyFrom(segment);
320         return new SysVVaList(copy, regSaveArea);
321     }
322 
323     @Override
324     public MemoryAddress address() {
325         return segment.address();
326     }
327 
328     private static boolean isRegOverflow(long currentGPOffset, long currentFPOffset, TypeClass typeClass) {
329         return currentGPOffset > MAX_GP_OFFSET - typeClass.nIntegerRegs() * GP_SLOT_SIZE
330                 || currentFPOffset > MAX_FP_OFFSET - typeClass.nVectorRegs() * FP_SLOT_SIZE;
331     }
332 
333     @Override
334     public String toString() {
335         return "SysVVaList{"
336                + "gp_offset=" + currentGPOffset()
337                + ", fp_offset=" + currentFPOffset()
338                + ", overflow_arg_area=" + stackPtr()
339                + ", reg_save_area=" + regSaveArea()
340                + '}';
341     }
342 
343     public static non-sealed class Builder implements VaList.Builder {
344         private final ResourceScope scope;
345         private final MemorySegment reg_save_area;
346         private long currentGPOffset = 0;
347         private long currentFPOffset = FP_OFFSET;
348         private final List<SimpleVaArg> stackArgs = new ArrayList<>();
349 
350         public Builder(ResourceScope scope) {
351             this.scope = scope;
352             this.reg_save_area = MemorySegment.allocateNative(LAYOUT_REG_SAVE_AREA, scope);
353         }
354 
355         @Override
356         public Builder vargFromInt(ValueLayout layout, int value) {
357             return arg(int.class, layout, value);
358         }
359 
360         @Override
361         public Builder vargFromLong(ValueLayout layout, long value) {
362             return arg(long.class, layout, value);
363         }
364 
365         @Override
366         public Builder vargFromDouble(ValueLayout layout, double value) {
367             return arg(double.class, layout, value);
368         }
369 
370         @Override
371         public Builder vargFromAddress(ValueLayout layout, Addressable value) {
372             return arg(MemoryAddress.class, layout, value.address());
373         }
374 
375         @Override
376         public Builder vargFromSegment(GroupLayout layout, MemorySegment value) {
377             return arg(MemorySegment.class, layout, value);
378         }
379 
380         private Builder arg(Class<?> carrier, MemoryLayout layout, Object value) {
381             Objects.requireNonNull(layout);
382             Objects.requireNonNull(value);
383             checkCompatibleType(carrier, layout, SysVx64Linker.ADDRESS_SIZE);
384             TypeClass typeClass = TypeClass.classifyLayout(layout);
385             if (isRegOverflow(currentGPOffset, currentFPOffset, typeClass)
386                     || typeClass.inMemory()) {
387                 // stack it!
388                 stackArgs.add(new SimpleVaArg(carrier, layout, value));
389             } else {
390                 switch (typeClass.kind()) {
391                     case STRUCT -> {
392                         MemorySegment valueSegment = (MemorySegment) value;
393                         int classIdx = 0;
394                         long offset = 0;
395                         while (offset < layout.byteSize()) {
396                             final long copy = Math.min(layout.byteSize() - offset, 8);
397                             boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE;
398                             MemorySegment slice = valueSegment.asSlice(offset, copy);
399                             if (isSSE) {
400                                 reg_save_area.asSlice(currentFPOffset, copy).copyFrom(slice);
401                                 currentFPOffset += FP_SLOT_SIZE;
402                             } else {
403                                 reg_save_area.asSlice(currentGPOffset, copy).copyFrom(slice);
404                                 currentGPOffset += GP_SLOT_SIZE;
405                             }
406                             offset += copy;
407                         }
408                     }
409                     case POINTER, INTEGER -> {
410                         VarHandle writer = SharedUtils.vhPrimitiveOrAddress(carrier, layout);
411                         writer.set(reg_save_area.asSlice(currentGPOffset), value);
412                         currentGPOffset += GP_SLOT_SIZE;
413                     }
414                     case FLOAT -> {
415                         VarHandle writer = layout.varHandle(carrier);
416                         writer.set(reg_save_area.asSlice(currentFPOffset), value);
417                         currentFPOffset += FP_SLOT_SIZE;
418                     }
419                 }
420             }
421             return this;
422         }
423 
424         private boolean isEmpty() {
425             return currentGPOffset == 0 && currentFPOffset == FP_OFFSET && stackArgs.isEmpty();
426         }
427 
428         public VaList build() {
429             if (isEmpty()) {
430                 return EMPTY;
431             }
432 
433             SegmentAllocator allocator = SegmentAllocator.arenaAllocator(scope);
434             MemorySegment vaListSegment = allocator.allocate(LAYOUT);
435             MemoryAddress stackArgsPtr = MemoryAddress.NULL;
436             if (!stackArgs.isEmpty()) {
437                 long stackArgsSize = stackArgs.stream().reduce(0L, (acc, e) -> acc + e.layout.byteSize(), Long::sum);
438                 MemorySegment stackArgsSegment = allocator.allocate(stackArgsSize, 16);
439                 MemorySegment maOverflowArgArea = stackArgsSegment;
440                 for (SimpleVaArg arg : stackArgs) {
441                     if (arg.layout.byteSize() > 8) {
442                         maOverflowArgArea = Utils.alignUp(maOverflowArgArea, Math.min(16, arg.layout.byteSize()));
443                     }
444                     if (arg.value instanceof MemorySegment) {
445                         maOverflowArgArea.copyFrom((MemorySegment) arg.value);
446                     } else {
447                         VarHandle writer = arg.varHandle();
448                         writer.set(maOverflowArgArea, arg.value);
449                     }
450                     maOverflowArgArea = maOverflowArgArea.asSlice(arg.layout.byteSize());
451                 }
452                 stackArgsPtr = stackArgsSegment.address();
453             }
454 
455             VH_fp_offset.set(vaListSegment, (int) FP_OFFSET);
456             VH_overflow_arg_area.set(vaListSegment, stackArgsPtr);
457             VH_reg_save_area.set(vaListSegment, reg_save_area.address());
458             assert reg_save_area.scope().ownerThread() == vaListSegment.scope().ownerThread();
459             return new SysVVaList(vaListSegment, reg_save_area);
460         }
461     }
462 }