1 /*
2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26 package jdk.internal.foreign.abi.x64.sysv;
27
28 import jdk.incubator.foreign.*;
29 import jdk.internal.foreign.ResourceScopeImpl;
30 import jdk.internal.foreign.Scoped;
31 import jdk.internal.foreign.Utils;
32 import jdk.internal.foreign.abi.SharedUtils;
33 import jdk.internal.misc.Unsafe;
34
35 import java.lang.invoke.VarHandle;
36 import java.lang.ref.Cleaner;
37 import java.util.ArrayList;
38 import java.util.List;
39 import java.util.Objects;
40
41 import static jdk.internal.foreign.PlatformLayouts.SysV;
42
43 import static jdk.incubator.foreign.MemoryLayout.PathElement.groupElement;
44 import static jdk.internal.foreign.abi.SharedUtils.SimpleVaArg;
45 import static jdk.internal.foreign.abi.SharedUtils.THROWING_ALLOCATOR;
46
47 // See https://software.intel.com/sites/default/files/article/402129/mpx-linux64-abi.pdf "3.5.7 Variable Argument Lists"
48 public non-sealed class SysVVaList implements VaList, Scoped {
49 private static final Unsafe U = Unsafe.getUnsafe();
50
51 static final Class<?> CARRIER = MemoryAddress.class;
52
53 // struct typedef __va_list_tag __va_list_tag {
54 // unsigned int gp_offset; /* 0 4 */
55 // unsigned int fp_offset; /* 4 4 */
56 // void * overflow_arg_area; /* 8 8 */
57 // void * reg_save_area; /* 16 8 */
58 //
59 // /* size: 24, cachelines: 1, members: 4 */
60 // /* last cacheline: 24 bytes */
61 // };
62 static final GroupLayout LAYOUT = MemoryLayout.structLayout(
63 SysV.C_INT.withName("gp_offset"),
64 SysV.C_INT.withName("fp_offset"),
65 SysV.C_POINTER.withName("overflow_arg_area"),
66 SysV.C_POINTER.withName("reg_save_area")
67 ).withName("__va_list_tag");
68
69 private static final MemoryLayout GP_REG = MemoryLayout.paddingLayout(64).withBitAlignment(64);
70 private static final MemoryLayout FP_REG = MemoryLayout.paddingLayout(128).withBitAlignment(128);
71
72 private static final GroupLayout LAYOUT_REG_SAVE_AREA = MemoryLayout.structLayout(
73 GP_REG.withName("%rdi"),
74 GP_REG.withName("%rsi"),
75 GP_REG.withName("%rdx"),
76 GP_REG.withName("%rcx"),
77 GP_REG.withName("%r8"),
78 GP_REG.withName("%r9"),
79 FP_REG.withName("%xmm0"),
80 FP_REG.withName("%xmm1"),
81 FP_REG.withName("%xmm2"),
82 FP_REG.withName("%xmm3"),
83 FP_REG.withName("%xmm4"),
84 FP_REG.withName("%xmm5"),
85 FP_REG.withName("%xmm6"),
86 FP_REG.withName("%xmm7")
87 // specification and implementation differ as to whether the following are part of a reg save area
88 // Let's go with the implementation, since then it actually works :)
89 // FP_REG.withName("%xmm8"),
90 // FP_REG.withName("%xmm9"),
91 // FP_REG.withName("%xmm10"),
92 // FP_REG.withName("%xmm11"),
93 // FP_REG.withName("%xmm12"),
94 // FP_REG.withName("%xmm13"),
95 // FP_REG.withName("%xmm14"),
96 // FP_REG.withName("%xmm15")
97 );
98
99 private static final long FP_OFFSET = LAYOUT_REG_SAVE_AREA.byteOffset(groupElement("%xmm0"));
100
101 private static final int GP_SLOT_SIZE = (int) GP_REG.byteSize();
102 private static final int FP_SLOT_SIZE = (int) FP_REG.byteSize();
103
104 private static final int MAX_GP_OFFSET = (int) FP_OFFSET; // 6 regs used
105 private static final int MAX_FP_OFFSET = (int) LAYOUT_REG_SAVE_AREA.byteSize(); // 8 16 byte regs
106
107 private static final VarHandle VH_fp_offset = LAYOUT.varHandle(groupElement("fp_offset"));
108 private static final VarHandle VH_gp_offset = LAYOUT.varHandle(groupElement("gp_offset"));
109 private static final VarHandle VH_overflow_arg_area = LAYOUT.varHandle(groupElement("overflow_arg_area"));
110 private static final VarHandle VH_reg_save_area = LAYOUT.varHandle(groupElement("reg_save_area"));
111
112 private static final VaList EMPTY = new SharedUtils.EmptyVaList(emptyListAddress());
113
114 private final MemorySegment segment;
115 private final MemorySegment regSaveArea;
116
117 private SysVVaList(MemorySegment segment, MemorySegment regSaveArea) {
118 this.segment = segment;
119 this.regSaveArea = regSaveArea;
120 }
121
122 private static SysVVaList readFromSegment(MemorySegment segment) {
123 MemorySegment regSaveArea = getRegSaveArea(segment);
124 return new SysVVaList(segment, regSaveArea);
125 }
126
127 private static MemoryAddress emptyListAddress() {
128 long ptr = U.allocateMemory(LAYOUT.byteSize());
129 ResourceScope scope = ResourceScope.newImplicitScope();
130 scope.addCloseAction(() -> U.freeMemory(ptr));
131 MemorySegment base = MemorySegment.ofAddressNative(MemoryAddress.ofLong(ptr),
132 LAYOUT.byteSize(), scope);
133 VH_gp_offset.set(base, MAX_GP_OFFSET);
134 VH_fp_offset.set(base, MAX_FP_OFFSET);
135 VH_overflow_arg_area.set(base, MemoryAddress.NULL);
136 VH_reg_save_area.set(base, MemoryAddress.NULL);
137 return base.address();
138 }
139
140 public static VaList empty() {
141 return EMPTY;
142 }
143
144 private int currentGPOffset() {
145 return (int) VH_gp_offset.get(segment);
146 }
147
148 private void currentGPOffset(int i) {
149 VH_gp_offset.set(segment, i);
150 }
151
152 private int currentFPOffset() {
153 return (int) VH_fp_offset.get(segment);
154 }
155
156 private void currentFPOffset(int i) {
157 VH_fp_offset.set(segment, i);
158 }
159
160 private MemoryAddress stackPtr() {
161 return (MemoryAddress) VH_overflow_arg_area.get(segment);
162 }
163
164 private void stackPtr(MemoryAddress ptr) {
165 VH_overflow_arg_area.set(segment, ptr);
166 }
167
168 private MemorySegment regSaveArea() {
169 return getRegSaveArea(segment);
170 }
171
172 private static MemorySegment getRegSaveArea(MemorySegment segment) {
173 return MemorySegment.ofAddressNative(((MemoryAddress)VH_reg_save_area.get(segment)),
174 LAYOUT_REG_SAVE_AREA.byteSize(), segment.scope());
175 }
176
177 private void preAlignStack(MemoryLayout layout) {
178 if (layout.byteAlignment() > 8) {
179 stackPtr(Utils.alignUp(stackPtr(), 16));
180 }
181 }
182
183 private void postAlignStack(MemoryLayout layout) {
184 stackPtr(Utils.alignUp(stackPtr().addOffset(layout.byteSize()), 8));
185 }
186
187 @Override
188 public int nextVarg(ValueLayout.OfInt layout) {
189 return (int) read(int.class, layout);
190 }
191
192 @Override
193 public long nextVarg(ValueLayout.OfLong layout) {
194 return (long) read(long.class, layout);
195 }
196
197 @Override
198 public double nextVarg(ValueLayout.OfDouble layout) {
199 return (double) read(double.class, layout);
200 }
201
202 @Override
203 public MemoryAddress nextVarg(ValueLayout.OfAddress layout) {
204 return (MemoryAddress) read(MemoryAddress.class, layout);
205 }
206
207 @Override
208 public MemorySegment nextVarg(GroupLayout layout, SegmentAllocator allocator) {
209 Objects.requireNonNull(allocator);
210 return (MemorySegment) read(MemorySegment.class, layout, allocator);
211 }
212
213 private Object read(Class<?> carrier, MemoryLayout layout) {
214 return read(carrier, layout, THROWING_ALLOCATOR);
215 }
216
217 private Object read(Class<?> carrier, MemoryLayout layout, SegmentAllocator allocator) {
218 Objects.requireNonNull(layout);
219 TypeClass typeClass = TypeClass.classifyLayout(layout);
220 if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)
221 || typeClass.inMemory()) {
222 preAlignStack(layout);
223 return switch (typeClass.kind()) {
224 case STRUCT -> {
225 MemorySegment slice = MemorySegment.ofAddressNative(stackPtr(), layout.byteSize(), scope());
226 MemorySegment seg = allocator.allocate(layout);
227 seg.copyFrom(slice);
228 postAlignStack(layout);
229 yield seg;
230 }
231 case POINTER, INTEGER, FLOAT -> {
232 VarHandle reader = layout.varHandle();
233 try (ResourceScope localScope = ResourceScope.newConfinedScope()) {
234 MemorySegment slice = MemorySegment.ofAddressNative(stackPtr(), layout.byteSize(), localScope);
235 Object res = reader.get(slice);
236 postAlignStack(layout);
237 yield res;
238 }
239 }
240 };
241 } else {
242 return switch (typeClass.kind()) {
243 case STRUCT -> {
244 MemorySegment value = allocator.allocate(layout);
245 int classIdx = 0;
246 long offset = 0;
247 while (offset < layout.byteSize()) {
248 final long copy = Math.min(layout.byteSize() - offset, 8);
249 boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE;
250 if (isSSE) {
251 MemorySegment.copy(regSaveArea, currentFPOffset(), value, offset, copy);
252 currentFPOffset(currentFPOffset() + FP_SLOT_SIZE);
253 } else {
254 MemorySegment.copy(regSaveArea, currentGPOffset(), value, offset, copy);
255 currentGPOffset(currentGPOffset() + GP_SLOT_SIZE);
256 }
257 offset += copy;
258 }
259 yield value;
260 }
261 case POINTER, INTEGER -> {
262 VarHandle reader = layout.varHandle();
263 Object res = reader.get(regSaveArea.asSlice(currentGPOffset()));
264 currentGPOffset(currentGPOffset() + GP_SLOT_SIZE);
265 yield res;
266 }
267 case FLOAT -> {
268 VarHandle reader = layout.varHandle();
269 Object res = reader.get(regSaveArea.asSlice(currentFPOffset()));
270 currentFPOffset(currentFPOffset() + FP_SLOT_SIZE);
271 yield res;
272 }
273 };
274 }
275 }
276
277 @Override
278 public void skip(MemoryLayout... layouts) {
279 Objects.requireNonNull(layouts);
280 ((ResourceScopeImpl)segment.scope()).checkValidStateSlow();
281 for (MemoryLayout layout : layouts) {
282 Objects.requireNonNull(layout);
283 TypeClass typeClass = TypeClass.classifyLayout(layout);
284 if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)) {
285 preAlignStack(layout);
286 postAlignStack(layout);
287 } else {
288 currentGPOffset(currentGPOffset() + (((int) typeClass.nIntegerRegs()) * GP_SLOT_SIZE));
289 currentFPOffset(currentFPOffset() + (((int) typeClass.nVectorRegs()) * FP_SLOT_SIZE));
290 }
291 }
292 }
293
294 static SysVVaList.Builder builder(ResourceScope scope) {
295 return new SysVVaList.Builder(scope);
296 }
297
298 public static VaList ofAddress(MemoryAddress ma, ResourceScope scope) {
299 return readFromSegment(MemorySegment.ofAddressNative(ma, LAYOUT.byteSize(), scope));
300 }
301
302 @Override
303 public ResourceScope scope() {
304 return segment.scope();
305 }
306
307 @Override
308 public VaList copy() {
309 MemorySegment copy = MemorySegment.allocateNative(LAYOUT, segment.scope());
310 copy.copyFrom(segment);
311 return new SysVVaList(copy, regSaveArea);
312 }
313
314 @Override
315 public MemoryAddress address() {
316 return segment.address();
317 }
318
319 private static boolean isRegOverflow(long currentGPOffset, long currentFPOffset, TypeClass typeClass) {
320 return currentGPOffset > MAX_GP_OFFSET - typeClass.nIntegerRegs() * GP_SLOT_SIZE
321 || currentFPOffset > MAX_FP_OFFSET - typeClass.nVectorRegs() * FP_SLOT_SIZE;
322 }
323
324 @Override
325 public String toString() {
326 return "SysVVaList{"
327 + "gp_offset=" + currentGPOffset()
328 + ", fp_offset=" + currentFPOffset()
329 + ", overflow_arg_area=" + stackPtr()
330 + ", reg_save_area=" + regSaveArea()
331 + '}';
332 }
333
334 public static non-sealed class Builder implements VaList.Builder {
335 private final ResourceScope scope;
336 private final MemorySegment reg_save_area;
337 private long currentGPOffset = 0;
338 private long currentFPOffset = FP_OFFSET;
339 private final List<SimpleVaArg> stackArgs = new ArrayList<>();
340
341 public Builder(ResourceScope scope) {
342 this.scope = scope;
343 this.reg_save_area = MemorySegment.allocateNative(LAYOUT_REG_SAVE_AREA, scope);
344 }
345
346 @Override
347 public Builder addVarg(ValueLayout.OfInt layout, int value) {
348 return arg(int.class, layout, value);
349 }
350
351 @Override
352 public Builder addVarg(ValueLayout.OfLong layout, long value) {
353 return arg(long.class, layout, value);
354 }
355
356 @Override
357 public Builder addVarg(ValueLayout.OfDouble layout, double value) {
358 return arg(double.class, layout, value);
359 }
360
361 @Override
362 public Builder addVarg(ValueLayout.OfAddress layout, Addressable value) {
363 return arg(MemoryAddress.class, layout, value.address());
364 }
365
366 @Override
367 public Builder addVarg(GroupLayout layout, MemorySegment value) {
368 return arg(MemorySegment.class, layout, value);
369 }
370
371 private Builder arg(Class<?> carrier, MemoryLayout layout, Object value) {
372 Objects.requireNonNull(layout);
373 Objects.requireNonNull(value);
374 TypeClass typeClass = TypeClass.classifyLayout(layout);
375 if (isRegOverflow(currentGPOffset, currentFPOffset, typeClass)
376 || typeClass.inMemory()) {
377 // stack it!
378 stackArgs.add(new SimpleVaArg(carrier, layout, value));
379 } else {
380 switch (typeClass.kind()) {
381 case STRUCT -> {
382 MemorySegment valueSegment = (MemorySegment) value;
383 int classIdx = 0;
384 long offset = 0;
385 while (offset < layout.byteSize()) {
386 final long copy = Math.min(layout.byteSize() - offset, 8);
387 boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE;
388 if (isSSE) {
389 MemorySegment.copy(valueSegment, offset, reg_save_area, currentFPOffset, copy);
390 currentFPOffset += FP_SLOT_SIZE;
391 } else {
392 MemorySegment.copy(valueSegment, offset, reg_save_area, currentGPOffset, copy);
393 currentGPOffset += GP_SLOT_SIZE;
394 }
395 offset += copy;
396 }
397 }
398 case POINTER, INTEGER -> {
399 VarHandle writer = layout.varHandle();
400 writer.set(reg_save_area.asSlice(currentGPOffset), value);
401 currentGPOffset += GP_SLOT_SIZE;
402 }
403 case FLOAT -> {
404 VarHandle writer = layout.varHandle();
405 writer.set(reg_save_area.asSlice(currentFPOffset), value);
406 currentFPOffset += FP_SLOT_SIZE;
407 }
408 }
409 }
410 return this;
411 }
412
413 private boolean isEmpty() {
414 return currentGPOffset == 0 && currentFPOffset == FP_OFFSET && stackArgs.isEmpty();
415 }
416
417 public VaList build() {
418 if (isEmpty()) {
419 return EMPTY;
420 }
421
422 SegmentAllocator allocator = SegmentAllocator.newNativeArena(scope);
423 MemorySegment vaListSegment = allocator.allocate(LAYOUT);
424 MemoryAddress stackArgsPtr = MemoryAddress.NULL;
425 if (!stackArgs.isEmpty()) {
426 long stackArgsSize = stackArgs.stream().reduce(0L, (acc, e) -> acc + e.layout.byteSize(), Long::sum);
427 MemorySegment stackArgsSegment = allocator.allocate(stackArgsSize, 16);
428 MemorySegment maOverflowArgArea = stackArgsSegment;
429 for (SimpleVaArg arg : stackArgs) {
430 if (arg.layout.byteSize() > 8) {
431 maOverflowArgArea = Utils.alignUp(maOverflowArgArea, Math.min(16, arg.layout.byteSize()));
432 }
433 if (arg.value instanceof MemorySegment) {
434 maOverflowArgArea.copyFrom((MemorySegment) arg.value);
435 } else {
436 VarHandle writer = arg.varHandle();
437 writer.set(maOverflowArgArea, arg.value);
438 }
439 maOverflowArgArea = maOverflowArgArea.asSlice(arg.layout.byteSize());
440 }
441 stackArgsPtr = stackArgsSegment.address();
442 }
443
444 VH_fp_offset.set(vaListSegment, (int) FP_OFFSET);
445 VH_overflow_arg_area.set(vaListSegment, stackArgsPtr);
446 VH_reg_save_area.set(vaListSegment, reg_save_area.address());
447 assert reg_save_area.scope().ownerThread() == vaListSegment.scope().ownerThread();
448 return new SysVVaList(vaListSegment, reg_save_area);
449 }
450 }
451 }