1 /*
  2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.  Oracle designates this
  8  * particular file as subject to the "Classpath" exception as provided
  9  * by Oracle in the LICENSE file that accompanied this code.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  */
 25 package jdk.internal.foreign.abi;
 26 
 27 import jdk.incubator.foreign.Addressable;
 28 import jdk.incubator.foreign.CLinker;
 29 import jdk.incubator.foreign.FunctionDescriptor;
 30 import jdk.incubator.foreign.GroupLayout;
 31 import jdk.incubator.foreign.MemoryAddress;
 32 import jdk.incubator.foreign.MemoryLayout;
 33 import jdk.incubator.foreign.MemorySegment;
 34 import jdk.incubator.foreign.NativeSymbol;
 35 import jdk.incubator.foreign.ResourceScope;
 36 import jdk.incubator.foreign.SegmentAllocator;
 37 import jdk.incubator.foreign.SequenceLayout;
 38 import jdk.incubator.foreign.VaList;
 39 import jdk.incubator.foreign.ValueLayout;
 40 import jdk.internal.access.JavaLangAccess;
 41 import jdk.internal.access.JavaLangInvokeAccess;
 42 import jdk.internal.access.SharedSecrets;
 43 import jdk.internal.foreign.Scoped;
 44 import jdk.internal.foreign.CABI;
 45 import jdk.internal.foreign.MemoryAddressImpl;
 46 import jdk.internal.foreign.ResourceScopeImpl;
 47 import jdk.internal.foreign.Utils;
 48 import jdk.internal.foreign.abi.aarch64.linux.LinuxAArch64Linker;
 49 import jdk.internal.foreign.abi.aarch64.macos.MacOsAArch64Linker;
 50 import jdk.internal.foreign.abi.x64.sysv.SysVx64Linker;
 51 import jdk.internal.foreign.abi.x64.windows.Windowsx64Linker;
 52 import jdk.internal.vm.annotation.ForceInline;
 53 
 54 import java.lang.invoke.MethodHandle;
 55 import java.lang.invoke.MethodHandles;
 56 import java.lang.invoke.MethodType;
 57 import java.lang.invoke.VarHandle;
 58 import java.lang.ref.Reference;
 59 import java.nio.charset.StandardCharsets;
 60 import java.util.ArrayList;
 61 import java.util.Arrays;
 62 import java.util.List;
 63 import java.util.Map;
 64 import java.util.Objects;
 65 import java.util.function.Consumer;
 66 import java.util.function.UnaryOperator;
 67 import java.util.stream.Collectors;
 68 import java.util.stream.IntStream;
 69 
 70 import static java.lang.invoke.MethodHandles.collectArguments;
 71 import static java.lang.invoke.MethodHandles.constant;
 72 import static java.lang.invoke.MethodHandles.dropArguments;
 73 import static java.lang.invoke.MethodHandles.dropReturn;
 74 import static java.lang.invoke.MethodHandles.empty;
 75 import static java.lang.invoke.MethodHandles.foldArguments;
 76 import static java.lang.invoke.MethodHandles.identity;
 77 import static java.lang.invoke.MethodHandles.insertArguments;
 78 import static java.lang.invoke.MethodHandles.permuteArguments;
 79 import static java.lang.invoke.MethodHandles.tryFinally;
 80 import static java.lang.invoke.MethodType.methodType;
 81 import static jdk.incubator.foreign.ValueLayout.ADDRESS;
 82 import static jdk.incubator.foreign.ValueLayout.JAVA_BOOLEAN;
 83 import static jdk.incubator.foreign.ValueLayout.JAVA_BYTE;
 84 import static jdk.incubator.foreign.ValueLayout.JAVA_CHAR;
 85 import static jdk.incubator.foreign.ValueLayout.JAVA_DOUBLE;
 86 import static jdk.incubator.foreign.ValueLayout.JAVA_FLOAT;
 87 import static jdk.incubator.foreign.ValueLayout.JAVA_INT;
 88 import static jdk.incubator.foreign.ValueLayout.JAVA_LONG;
 89 import static jdk.incubator.foreign.ValueLayout.JAVA_SHORT;
 90 
 91 public class SharedUtils {
 92 
 93     private static final JavaLangAccess JLA = SharedSecrets.getJavaLangAccess();
 94     private static final JavaLangInvokeAccess JLIA = SharedSecrets.getJavaLangInvokeAccess();
 95 
 96     private static final MethodHandle MH_ALLOC_BUFFER;
 97     private static final MethodHandle MH_BASEADDRESS;
 98     private static final MethodHandle MH_BUFFER_COPY;
 99     private static final MethodHandle MH_MAKE_CONTEXT_NO_ALLOCATOR;
100     private static final MethodHandle MH_MAKE_CONTEXT_BOUNDED_ALLOCATOR;
101     private static final MethodHandle MH_CLOSE_CONTEXT;
102     private static final MethodHandle MH_REACHBILITY_FENCE;
103     private static final MethodHandle MH_HANDLE_UNCAUGHT_EXCEPTION;
104     private static final MethodHandle ACQUIRE_MH;
105     private static final MethodHandle RELEASE_MH;
106 
107     static {
108         try {
109             MethodHandles.Lookup lookup = MethodHandles.lookup();
110             MH_ALLOC_BUFFER = lookup.findVirtual(SegmentAllocator.class, "allocate",
111                     methodType(MemorySegment.class, MemoryLayout.class));
112             MH_BASEADDRESS = lookup.findVirtual(MemorySegment.class, "address",
113                     methodType(MemoryAddress.class));
114             MH_BUFFER_COPY = lookup.findStatic(SharedUtils.class, "bufferCopy",
115                     methodType(MemoryAddress.class, MemoryAddress.class, MemorySegment.class));
116             MH_MAKE_CONTEXT_NO_ALLOCATOR = lookup.findStatic(Binding.Context.class, "ofScope",
117                     methodType(Binding.Context.class));
118             MH_MAKE_CONTEXT_BOUNDED_ALLOCATOR = lookup.findStatic(Binding.Context.class, "ofBoundedAllocator",
119                     methodType(Binding.Context.class, long.class));
120             MH_CLOSE_CONTEXT = lookup.findVirtual(Binding.Context.class, "close",
121                     methodType(void.class));
122             MH_REACHBILITY_FENCE = lookup.findStatic(Reference.class, "reachabilityFence",
123                     methodType(void.class, Object.class));
124             MH_HANDLE_UNCAUGHT_EXCEPTION = lookup.findStatic(SharedUtils.class, "handleUncaughtException",
125                     methodType(void.class, Throwable.class));
126             ACQUIRE_MH = MethodHandles.lookup().findStatic(SharedUtils.class, "acquire",
127                     MethodType.methodType(void.class, Scoped[].class));
128             RELEASE_MH = MethodHandles.lookup().findStatic(SharedUtils.class, "release",
129                     MethodType.methodType(void.class, Scoped[].class));
130         } catch (ReflectiveOperationException e) {
131             throw new BootstrapMethodError(e);
132         }
133     }
134 
135     // this allocator should be used when no allocation is expected
136     public static final SegmentAllocator THROWING_ALLOCATOR = (size, align) -> {
137         throw new IllegalStateException("Cannot get here");
138     };
139 
140     /**
141      * Align the specified type from a given address
142      * @return The address the data should be at based on alignment requirement
143      */
144     public static long align(MemoryLayout t, boolean isVar, long addr) {
145         return alignUp(addr, alignment(t, isVar));
146     }
147 
148     public static long alignUp(long addr, long alignment) {
149         return ((addr - 1) | (alignment - 1)) + 1;
150     }
151 
152     /**
153      * The alignment requirement for a given type
154      * @param isVar indicate if the type is a standalone variable. This change how
155      * array is aligned. for example.
156      */
157     public static long alignment(MemoryLayout t, boolean isVar) {
158         if (t instanceof ValueLayout) {
159             return alignmentOfScalar((ValueLayout) t);
160         } else if (t instanceof SequenceLayout) {
161             // when array is used alone
162             return alignmentOfArray((SequenceLayout) t, isVar);
163         } else if (t instanceof GroupLayout) {
164             return alignmentOfContainer((GroupLayout) t);
165         } else if (t.isPadding()) {
166             return 1;
167         } else {
168             throw new IllegalArgumentException("Invalid type: " + t);
169         }
170     }
171 
172     private static long alignmentOfScalar(ValueLayout st) {
173         return st.byteSize();
174     }
175 
176     private static long alignmentOfArray(SequenceLayout ar, boolean isVar) {
177         if (ar.elementCount().orElseThrow() == 0) {
178             // VLA or incomplete
179             return 16;
180         } else if ((ar.byteSize()) >= 16 && isVar) {
181             return 16;
182         } else {
183             // align as element type
184             MemoryLayout elementType = ar.elementLayout();
185             return alignment(elementType, false);
186         }
187     }
188 
189     private static long alignmentOfContainer(GroupLayout ct) {
190         // Most strict member
191         return ct.memberLayouts().stream().mapToLong(t -> alignment(t, false)).max().orElse(1);
192     }
193 
194     /**
195      * Takes a MethodHandle that takes an input buffer as a first argument (a MemoryAddress), and returns nothing,
196      * and adapts it to return a MemorySegment, by allocating a MemorySegment for the input
197      * buffer, calling the target MethodHandle, and then returning the allocated MemorySegment.
198      *
199      * This allows viewing a MethodHandle that makes use of in memory return (IMR) as a MethodHandle that just returns
200      * a MemorySegment without requiring a pre-allocated buffer as an explicit input.
201      *
202      * @param handle the target handle to adapt
203      * @param cDesc the function descriptor of the native function (with actual return layout)
204      * @return the adapted handle
205      */
206     public static MethodHandle adaptDowncallForIMR(MethodHandle handle, FunctionDescriptor cDesc) {
207         if (handle.type().returnType() != void.class)
208             throw new IllegalArgumentException("return expected to be void for in memory returns: " + handle.type());
209         if (handle.type().parameterType(2) != MemoryAddress.class)
210             throw new IllegalArgumentException("MemoryAddress expected as third param: " + handle.type());
211         if (cDesc.returnLayout().isEmpty())
212             throw new IllegalArgumentException("Return layout needed: " + cDesc);
213 
214         MethodHandle ret = identity(MemorySegment.class); // (MemorySegment) MemorySegment
215         handle = collectArguments(ret, 1, handle); // (MemorySegment, Addressable, SegmentAllocator, MemoryAddress, ...) MemorySegment
216         handle = collectArguments(handle, 3, MH_BASEADDRESS); // (MemorySegment, Addressable, SegmentAllocator, MemorySegment, ...) MemorySegment
217         handle = mergeArguments(handle, 0, 3);  // (MemorySegment, Addressable, SegmentAllocator, ...) MemorySegment
218         handle = collectArguments(handle, 0, insertArguments(MH_ALLOC_BUFFER, 1, cDesc.returnLayout().get())); // (SegmentAllocator, Addressable, SegmentAllocator, ...) MemoryAddress
219         handle = mergeArguments(handle, 0, 2);  // (SegmentAllocator, Addressable, ...) MemoryAddress
220         handle = swapArguments(handle, 0, 1); // (Addressable, SegmentAllocator, ...) MemoryAddress
221         return handle;
222     }
223 
224     /**
225      * Takes a MethodHandle that returns a MemorySegment, and adapts it to take an input buffer as a first argument
226      * (a MemoryAddress), and upon invocation, copies the contents of the returned MemorySegment into the input buffer
227      * passed as the first argument.
228      *
229      * @param target the target handle to adapt
230      * @return the adapted handle
231      */
232     public static MethodHandle adaptUpcallForIMR(MethodHandle target, boolean dropReturn) {
233         if (target.type().returnType() != MemorySegment.class)
234             throw new IllegalArgumentException("Must return MemorySegment for IMR");
235 
236         target = collectArguments(MH_BUFFER_COPY, 1, target); // (MemoryAddress, ...) MemoryAddress
237 
238         if (dropReturn) { // no handling for return value, need to drop it
239             target = dropReturn(target);
240         }
241 
242         return target;
243     }
244 
245     private static MemoryAddress bufferCopy(MemoryAddress dest, MemorySegment buffer) {
246         MemoryAddressImpl.ofLongUnchecked(dest.toRawLongValue(), buffer.byteSize()).copyFrom(buffer);
247         return dest;
248     }
249 
250     public static Class<?> primitiveCarrierForSize(long size, boolean useFloat) {
251         if (useFloat) {
252             if (size == 4) {
253                 return float.class;
254             } else if (size == 8) {
255                 return double.class;
256             }
257         } else {
258             if (size == 1) {
259                 return byte.class;
260             } else if (size == 2) {
261                 return short.class;
262             } else if (size <= 4) {
263                 return int.class;
264             } else if (size <= 8) {
265                 return long.class;
266             }
267         }
268 
269         throw new IllegalArgumentException("No type for size: " + size + " isFloat=" + useFloat);
270     }
271 
272     public static CLinker getSystemLinker() {
273         return switch (CABI.current()) {
274             case Win64 -> Windowsx64Linker.getInstance();
275             case SysV -> SysVx64Linker.getInstance();
276             case LinuxAArch64 -> LinuxAArch64Linker.getInstance();
277             case MacOsAArch64 -> MacOsAArch64Linker.getInstance();
278         };
279     }
280 
281     public static String toJavaStringInternal(MemorySegment segment, long start) {
282         int len = strlen(segment, start);
283         byte[] bytes = new byte[len];
284         MemorySegment.copy(segment, JAVA_BYTE, start, bytes, 0, len);
285         return new String(bytes, StandardCharsets.UTF_8);
286     }
287 
288     private static int strlen(MemorySegment segment, long start) {
289         // iterate until overflow (String can only hold a byte[], whose length can be expressed as an int)
290         for (int offset = 0; offset >= 0; offset++) {
291             byte curr = segment.get(JAVA_BYTE, start + offset);
292             if (curr == 0) {
293                 return offset;
294             }
295         }
296         throw new IllegalArgumentException("String too large");
297     }
298 
299     static long bufferCopySize(CallingSequence callingSequence) {
300         // FIXME: > 16 bytes alignment might need extra space since the
301         // starting address of the allocator might be un-aligned.
302         long size = 0;
303         for (int i = 0; i < callingSequence.argumentCount(); i++) {
304             List<Binding> bindings = callingSequence.argumentBindings(i);
305             for (Binding b : bindings) {
306                 if (b instanceof Binding.Copy) {
307                     Binding.Copy c = (Binding.Copy) b;
308                     size = Utils.alignUp(size, c.alignment());
309                     size += c.size();
310                 } else if (b instanceof Binding.Allocate) {
311                     Binding.Allocate c = (Binding.Allocate) b;
312                     size = Utils.alignUp(size, c.alignment());
313                     size += c.size();
314                 }
315             }
316         }
317         return size;
318     }
319 
320     static Map<VMStorage, Integer> indexMap(Binding.Move[] moves) {
321         return IntStream.range(0, moves.length)
322                         .boxed()
323                         .collect(Collectors.toMap(i -> moves[i].storage(), i -> i));
324     }
325 
326     static MethodHandle mergeArguments(MethodHandle mh, int sourceIndex, int destIndex) {
327         MethodType oldType = mh.type();
328         Class<?> sourceType = oldType.parameterType(sourceIndex);
329         Class<?> destType = oldType.parameterType(destIndex);
330         if (sourceType != destType) {
331             // TODO meet?
332             throw new IllegalArgumentException("Parameter types differ: " + sourceType + " != " + destType);
333         }
334         MethodType newType = oldType.dropParameterTypes(destIndex, destIndex + 1);
335         int[] reorder = new int[oldType.parameterCount()];
336         assert destIndex > sourceIndex;
337         for (int i = 0, index = 0; i < reorder.length; i++) {
338             if (i != destIndex) {
339                 reorder[i] = index++;
340             } else {
341                 reorder[i] = sourceIndex;
342             }
343         }
344         return permuteArguments(mh, newType, reorder);
345     }
346 
347 
348     static MethodHandle swapArguments(MethodHandle mh, int firstArg, int secondArg) {
349         MethodType mtype = mh.type();
350         int[] perms = new int[mtype.parameterCount()];
351         MethodType swappedType = MethodType.methodType(mtype.returnType());
352         for (int i = 0 ; i < perms.length ; i++) {
353             int dst = i;
354             if (i == firstArg) dst = secondArg;
355             if (i == secondArg) dst = firstArg;
356             perms[i] = dst;
357             swappedType = swappedType.appendParameterTypes(mtype.parameterType(dst));
358         }
359         return permuteArguments(mh, swappedType, perms);
360     }
361 
362     private static MethodHandle reachabilityFenceHandle(Class<?> type) {
363         return MH_REACHBILITY_FENCE.asType(MethodType.methodType(void.class, type));
364     }
365 
366     static void handleUncaughtException(Throwable t) {
367         if (t != null) {
368             t.printStackTrace();
369             JLA.exit(1);
370         }
371     }
372 
373     static MethodHandle wrapWithAllocator(MethodHandle specializedHandle,
374                                           int allocatorPos, long bufferCopySize,
375                                           boolean upcall) {
376         // insert try-finally to close the NativeScope used for Binding.Copy
377         MethodHandle closer;
378         int insertPos;
379         if (specializedHandle.type().returnType() == void.class) {
380             if (!upcall) {
381                 closer = empty(methodType(void.class, Throwable.class)); // (Throwable) -> void
382             } else {
383                 closer = MH_HANDLE_UNCAUGHT_EXCEPTION;
384             }
385             insertPos = 1;
386         } else {
387             closer = identity(specializedHandle.type().returnType()); // (V) -> V
388             if (!upcall) {
389                 closer = dropArguments(closer, 0, Throwable.class); // (Throwable, V) -> V
390             } else {
391                 closer = collectArguments(closer, 0, MH_HANDLE_UNCAUGHT_EXCEPTION); // (Throwable, V) -> V
392             }
393             insertPos = 2;
394         }
395 
396         // downcalls get the leading NativeSymbol/SegmentAllocator param as well
397         if (!upcall) {
398             closer = collectArguments(closer, insertPos++, reachabilityFenceHandle(NativeSymbol.class));
399             closer = dropArguments(closer, insertPos++, SegmentAllocator.class); // (Throwable, V?, NativeSymbol, SegmentAllocator) -> V/void
400         }
401 
402         closer = collectArguments(closer, insertPos++, MH_CLOSE_CONTEXT); // (Throwable, V?, NativeSymbol?, BindingContext) -> V/void
403 
404         MethodHandle contextFactory;
405 
406         if (bufferCopySize > 0) {
407             contextFactory = MethodHandles.insertArguments(MH_MAKE_CONTEXT_BOUNDED_ALLOCATOR, 0, bufferCopySize);
408         } else if (upcall) {
409             contextFactory = MH_MAKE_CONTEXT_NO_ALLOCATOR;
410         } else {
411             // this path is probably never used now, since ProgrammableInvoker never calls this routine with bufferCopySize == 0
412             contextFactory = constant(Binding.Context.class, Binding.Context.DUMMY);
413         }
414 
415         specializedHandle = tryFinally(specializedHandle, closer);
416         specializedHandle = collectArguments(specializedHandle, allocatorPos, contextFactory);
417         return specializedHandle;
418     }
419 
420     @ForceInline
421     @SuppressWarnings("fallthrough")
422     public static void acquire(Scoped[] args) {
423         ResourceScope scope4 = null;
424         ResourceScope scope3 = null;
425         ResourceScope scope2 = null;
426         ResourceScope scope1 = null;
427         ResourceScope scope0 = null;
428         switch (args.length) {
429             default:
430                 // slow path, acquire all remaining addressable parameters in isolation
431                 for (int i = 5 ; i < args.length ; i++) {
432                     acquire(args[i].scope());
433                 }
434             // fast path, acquire only scopes not seen in other parameters
435             case 5:
436                 scope4 = args[4].scope();
437                 acquire(scope4);
438             case 4:
439                 scope3 = args[3].scope();
440                 if (scope3 != scope4)
441                     acquire(scope3);
442             case 3:
443                 scope2 = args[2].scope();
444                 if (scope2 != scope3 && scope2 != scope4)
445                     acquire(scope2);
446             case 2:
447                 scope1 = args[1].scope();
448                 if (scope1 != scope2 && scope1 != scope3 && scope1 != scope4)
449                     acquire(scope1);
450             case 1:
451                 scope0 = args[0].scope();
452                 if (scope0 != scope1 && scope0 != scope2 && scope0 != scope3 && scope0 != scope4)
453                     acquire(scope0);
454             case 0: break;
455         }
456     }
457 
458     @ForceInline
459     @SuppressWarnings("fallthrough")
460     public static void release(Scoped[] args) {
461         ResourceScope scope4 = null;
462         ResourceScope scope3 = null;
463         ResourceScope scope2 = null;
464         ResourceScope scope1 = null;
465         ResourceScope scope0 = null;
466         switch (args.length) {
467             default:
468                 // slow path, release all remaining addressable parameters in isolation
469                 for (int i = 5 ; i < args.length ; i++) {
470                     release(args[i].scope());
471                 }
472             // fast path, release only scopes not seen in other parameters
473             case 5:
474                 scope4 = args[4].scope();
475                 release(scope4);
476             case 4:
477                 scope3 = args[3].scope();
478                 if (scope3 != scope4)
479                     release(scope3);
480             case 3:
481                 scope2 = args[2].scope();
482                 if (scope2 != scope3 && scope2 != scope4)
483                     release(scope2);
484             case 2:
485                 scope1 = args[1].scope();
486                 if (scope1 != scope2 && scope1 != scope3 && scope1 != scope4)
487                     release(scope1);
488             case 1:
489                 scope0 = args[0].scope();
490                 if (scope0 != scope1 && scope0 != scope2 && scope0 != scope3 && scope0 != scope4)
491                     release(scope0);
492             case 0: break;
493         }
494     }
495 
496     @ForceInline
497     private static void acquire(ResourceScope scope) {
498         ((ResourceScopeImpl)scope).acquire0();
499     }
500 
501     @ForceInline
502     private static void release(ResourceScope scope) {
503         ((ResourceScopeImpl)scope).release0();
504     }
505 
506     /*
507      * This method adds a try/finally block to a downcall method handle, to make sure that all by-reference
508      * parameters (including the target address of the native function) are kept alive for the duration of
509      * the downcall.
510      */
511     public static MethodHandle wrapDowncall(MethodHandle downcallHandle, FunctionDescriptor descriptor) {
512         boolean hasReturn = descriptor.returnLayout().isPresent();
513         MethodHandle tryBlock = downcallHandle;
514         MethodHandle cleanup = hasReturn ?
515                 MethodHandles.identity(downcallHandle.type().returnType()) :
516                 MethodHandles.empty(MethodType.methodType(void.class));
517         int addressableCount = 0;
518         List<UnaryOperator<MethodHandle>> adapters = new ArrayList<>();
519         for (int i = 0 ; i < downcallHandle.type().parameterCount() ; i++) {
520             Class<?> ptype = downcallHandle.type().parameterType(i);
521             if (ptype == Addressable.class || ptype == NativeSymbol.class) {
522                 addressableCount++;
523             } else {
524                 int pos = i;
525                 adapters.add(mh -> dropArguments(mh, pos, ptype));
526             }
527         }
528 
529         if (addressableCount > 0) {
530             cleanup = dropArguments(cleanup, 0, Throwable.class);
531 
532             MethodType adapterType = MethodType.methodType(void.class);
533             for (int i = 0 ; i < addressableCount ; i++) {
534                 adapterType = adapterType.appendParameterTypes(i == 0 ? NativeSymbol.class : Addressable.class);
535             }
536 
537             MethodHandle acquireHandle = ACQUIRE_MH.asCollector(Scoped[].class, addressableCount).asType(adapterType);
538             MethodHandle releaseHandle = RELEASE_MH.asCollector(Scoped[].class, addressableCount).asType(adapterType);
539 
540             for (UnaryOperator<MethodHandle> adapter : adapters) {
541                 acquireHandle = adapter.apply(acquireHandle);
542                 releaseHandle = adapter.apply(releaseHandle);
543             }
544 
545             tryBlock = foldArguments(tryBlock, acquireHandle);
546             cleanup = collectArguments(cleanup, hasReturn ? 2 : 1, releaseHandle);
547 
548             return tryFinally(tryBlock, cleanup);
549         } else {
550             return downcallHandle;
551         }
552     }
553 
554     public static void checkExceptions(MethodHandle target) {
555         Class<?>[] exceptions = JLIA.exceptionTypes(target);
556         if (exceptions != null && exceptions.length != 0) {
557             throw new IllegalArgumentException("Target handle may throw exceptions: " + Arrays.toString(exceptions));
558         }
559     }
560 
561     // lazy init MH_ALLOC and MH_FREE handles
562     private static class AllocHolder {
563 
564         private static final CLinker SYS_LINKER = getSystemLinker();
565 
566         static final MethodHandle MH_MALLOC = SYS_LINKER.downcallHandle(CLinker.systemCLinker().lookup("malloc").get(),
567                 FunctionDescriptor.of(ADDRESS, JAVA_LONG));
568 
569         static final MethodHandle MH_FREE = SYS_LINKER.downcallHandle(CLinker.systemCLinker().lookup("free").get(),
570                 FunctionDescriptor.ofVoid(ADDRESS));
571     }
572 
573     public static void checkSymbol(NativeSymbol symbol) {
574         checkAddressable(symbol, "Symbol is NULL");
575     }
576 
577     public static void checkAddress(MemoryAddress address) {
578         checkAddressable(address, "Address is NULL");
579     }
580 
581     private static void checkAddressable(Addressable symbol, String msg) {
582         Objects.requireNonNull(symbol);
583         if (symbol.address().toRawLongValue() == 0)
584             throw new IllegalArgumentException("Symbol is NULL: " + symbol);
585     }
586 
587     public static MemoryAddress allocateMemoryInternal(long size) {
588         try {
589             return (MemoryAddress) AllocHolder.MH_MALLOC.invokeExact(size);
590         } catch (Throwable th) {
591             throw new RuntimeException(th);
592         }
593     }
594 
595     public static void freeMemoryInternal(MemoryAddress addr) {
596         try {
597             AllocHolder.MH_FREE.invokeExact((Addressable)addr);
598         } catch (Throwable th) {
599             throw new RuntimeException(th);
600         }
601     }
602 
603     public static VaList newVaList(Consumer<VaList.Builder> actions, ResourceScope scope) {
604         return switch (CABI.current()) {
605             case Win64 -> Windowsx64Linker.newVaList(actions, scope);
606             case SysV -> SysVx64Linker.newVaList(actions, scope);
607             case LinuxAArch64 -> LinuxAArch64Linker.newVaList(actions, scope);
608             case MacOsAArch64 -> MacOsAArch64Linker.newVaList(actions, scope);
609         };
610     }
611 
612     public static VaList newVaListOfAddress(MemoryAddress ma, ResourceScope scope) {
613         return switch (CABI.current()) {
614             case Win64 -> Windowsx64Linker.newVaListOfAddress(ma, scope);
615             case SysV -> SysVx64Linker.newVaListOfAddress(ma, scope);
616             case LinuxAArch64 -> LinuxAArch64Linker.newVaListOfAddress(ma, scope);
617             case MacOsAArch64 -> MacOsAArch64Linker.newVaListOfAddress(ma, scope);
618         };
619     }
620 
621     public static VaList emptyVaList() {
622         return switch (CABI.current()) {
623             case Win64 -> Windowsx64Linker.emptyVaList();
624             case SysV -> SysVx64Linker.emptyVaList();
625             case LinuxAArch64 -> LinuxAArch64Linker.emptyVaList();
626             case MacOsAArch64 -> MacOsAArch64Linker.emptyVaList();
627         };
628     }
629 
630     static void checkType(Class<?> actualType, Class<?> expectedType) {
631         if (expectedType != actualType) {
632             throw new IllegalArgumentException(
633                     String.format("Invalid operand type: %s. %s expected", actualType, expectedType));
634         }
635     }
636 
637     public static boolean isTrivial(FunctionDescriptor cDesc) {
638         return false; // FIXME: use system property?
639     }
640 
641     public static boolean isVarargsIndex(FunctionDescriptor descriptor, int argIndex) {
642         int firstPos = descriptor.firstVariadicArgumentIndex();
643         return firstPos != -1 && argIndex >= firstPos;
644     }
645 
646     public static class SimpleVaArg {
647         public final Class<?> carrier;
648         public final MemoryLayout layout;
649         public final Object value;
650 
651         public SimpleVaArg(Class<?> carrier, MemoryLayout layout, Object value) {
652             this.carrier = carrier;
653             this.layout = layout;
654             this.value = value;
655         }
656 
657         public VarHandle varHandle() {
658             return layout.varHandle();
659         }
660     }
661 
662     public static non-sealed class EmptyVaList implements VaList, Scoped {
663 
664         private final MemoryAddress address;
665 
666         public EmptyVaList(MemoryAddress address) {
667             this.address = address;
668         }
669 
670         private static UnsupportedOperationException uoe() {
671             return new UnsupportedOperationException("Empty VaList");
672         }
673 
674         @Override
675         public int nextVarg(ValueLayout.OfInt layout) {
676             throw uoe();
677         }
678 
679         @Override
680         public long nextVarg(ValueLayout.OfLong layout) {
681             throw uoe();
682         }
683 
684         @Override
685         public double nextVarg(ValueLayout.OfDouble layout) {
686             throw uoe();
687         }
688 
689         @Override
690         public MemoryAddress nextVarg(ValueLayout.OfAddress layout) {
691             throw uoe();
692         }
693 
694         @Override
695         public MemorySegment nextVarg(GroupLayout layout, SegmentAllocator allocator) {
696             throw uoe();
697         }
698 
699         @Override
700         public void skip(MemoryLayout... layouts) {
701             throw uoe();
702         }
703 
704         @Override
705         public ResourceScope scope() {
706             return ResourceScope.globalScope();
707         }
708 
709         @Override
710         public VaList copy() {
711             return this;
712         }
713 
714         @Override
715         public MemoryAddress address() {
716             return address;
717         }
718     }
719 
720     static void writeOverSized(MemorySegment ptr, Class<?> type, Object o) {
721         // use VH_LONG for integers to zero out the whole register in the process
722         if (type == long.class) {
723             ptr.set(JAVA_LONG, 0, (long) o);
724         } else if (type == int.class) {
725             ptr.set(JAVA_LONG, 0, (int) o);
726         } else if (type == short.class) {
727             ptr.set(JAVA_LONG, 0, (short) o);
728         } else if (type == char.class) {
729             ptr.set(JAVA_LONG, 0, (char) o);
730         } else if (type == byte.class) {
731             ptr.set(JAVA_LONG, 0, (byte) o);
732         } else if (type == float.class) {
733             ptr.set(JAVA_FLOAT, 0, (float) o);
734         } else if (type == double.class) {
735             ptr.set(JAVA_DOUBLE, 0, (double) o);
736         } else if (type == boolean.class) {
737             ptr.set(JAVA_BOOLEAN, 0, (boolean) o);
738         } else {
739             throw new IllegalArgumentException("Unsupported carrier: " + type);
740         }
741     }
742 
743     static void write(MemorySegment ptr, Class<?> type, Object o) {
744         if (type == long.class) {
745             ptr.set(JAVA_LONG, 0, (long) o);
746         } else if (type == int.class) {
747             ptr.set(JAVA_INT, 0, (int) o);
748         } else if (type == short.class) {
749             ptr.set(JAVA_SHORT, 0, (short) o);
750         } else if (type == char.class) {
751             ptr.set(JAVA_CHAR, 0, (char) o);
752         } else if (type == byte.class) {
753             ptr.set(JAVA_BYTE, 0, (byte) o);
754         } else if (type == float.class) {
755             ptr.set(JAVA_FLOAT, 0, (float) o);
756         } else if (type == double.class) {
757             ptr.set(JAVA_DOUBLE, 0, (double) o);
758         } else if (type == boolean.class) {
759             ptr.set(JAVA_BOOLEAN, 0, (boolean) o);
760         } else {
761             throw new IllegalArgumentException("Unsupported carrier: " + type);
762         }
763     }
764 
765     static Object read(MemorySegment ptr, Class<?> type) {
766         if (type == long.class) {
767             return ptr.get(JAVA_LONG, 0);
768         } else if (type == int.class) {
769             return ptr.get(JAVA_INT, 0);
770         } else if (type == short.class) {
771             return ptr.get(JAVA_SHORT, 0);
772         } else if (type == char.class) {
773             return ptr.get(JAVA_CHAR, 0);
774         } else if (type == byte.class) {
775             return ptr.get(JAVA_BYTE, 0);
776         } else if (type == float.class) {
777             return ptr.get(JAVA_FLOAT, 0);
778         } else if (type == double.class) {
779             return ptr.get(JAVA_DOUBLE, 0);
780         } else if (type == boolean.class) {
781             return ptr.get(JAVA_BOOLEAN, 0);
782         } else {
783             throw new IllegalArgumentException("Unsupported carrier: " + type);
784         }
785     }
786 
787     public static MethodType inferMethodType(FunctionDescriptor descriptor, boolean upcall) {
788         MethodType type = MethodType.methodType(descriptor.returnLayout().isPresent() ?
789                 carrierFor(descriptor.returnLayout().get(), upcall) : void.class);
790         for (MemoryLayout argLayout : descriptor.argumentLayouts()) {
791             type = type.appendParameterTypes(carrierFor(argLayout, !upcall));
792         }
793         return type;
794     }
795 
796     static Class<?> carrierFor(MemoryLayout layout, boolean forArg) {
797         if (layout instanceof ValueLayout valueLayout) {
798             return (forArg && valueLayout.carrier().equals(MemoryAddress.class)) ?
799                     Addressable.class : valueLayout.carrier();
800         } else if (layout instanceof GroupLayout) {
801             return MemorySegment.class;
802         } else {
803             throw new IllegalArgumentException("Unsupported layout: " + layout);
804         }
805     }
806 }