1 /* 2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. Oracle designates this 8 * particular file as subject to the "Classpath" exception as provided 9 * by Oracle in the LICENSE file that accompanied this code. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 package jdk.internal.foreign.abi.x64.sysv; 27 28 import jdk.incubator.foreign.*; 29 import jdk.internal.foreign.ResourceScopeImpl; 30 import jdk.internal.foreign.Scoped; 31 import jdk.internal.foreign.Utils; 32 import jdk.internal.foreign.abi.SharedUtils; 33 import jdk.internal.misc.Unsafe; 34 35 import java.lang.invoke.VarHandle; 36 import java.lang.ref.Cleaner; 37 import java.util.ArrayList; 38 import java.util.List; 39 import java.util.Objects; 40 41 import static jdk.internal.foreign.PlatformLayouts.SysV; 42 43 import static jdk.incubator.foreign.MemoryLayout.PathElement.groupElement; 44 import static jdk.internal.foreign.abi.SharedUtils.SimpleVaArg; 45 import static jdk.internal.foreign.abi.SharedUtils.THROWING_ALLOCATOR; 46 47 // See https://software.intel.com/sites/default/files/article/402129/mpx-linux64-abi.pdf "3.5.7 Variable Argument Lists" 48 public non-sealed class SysVVaList implements VaList, Scoped { 49 private static final Unsafe U = Unsafe.getUnsafe(); 50 51 static final Class<?> CARRIER = MemoryAddress.class; 52 53 // struct typedef __va_list_tag __va_list_tag { 54 // unsigned int gp_offset; /* 0 4 */ 55 // unsigned int fp_offset; /* 4 4 */ 56 // void * overflow_arg_area; /* 8 8 */ 57 // void * reg_save_area; /* 16 8 */ 58 // 59 // /* size: 24, cachelines: 1, members: 4 */ 60 // /* last cacheline: 24 bytes */ 61 // }; 62 static final GroupLayout LAYOUT = MemoryLayout.structLayout( 63 SysV.C_INT.withName("gp_offset"), 64 SysV.C_INT.withName("fp_offset"), 65 SysV.C_POINTER.withName("overflow_arg_area"), 66 SysV.C_POINTER.withName("reg_save_area") 67 ).withName("__va_list_tag"); 68 69 private static final MemoryLayout GP_REG = MemoryLayout.paddingLayout(64).withBitAlignment(64); 70 private static final MemoryLayout FP_REG = MemoryLayout.paddingLayout(128).withBitAlignment(128); 71 72 private static final GroupLayout LAYOUT_REG_SAVE_AREA = MemoryLayout.structLayout( 73 GP_REG.withName("%rdi"), 74 GP_REG.withName("%rsi"), 75 GP_REG.withName("%rdx"), 76 GP_REG.withName("%rcx"), 77 GP_REG.withName("%r8"), 78 GP_REG.withName("%r9"), 79 FP_REG.withName("%xmm0"), 80 FP_REG.withName("%xmm1"), 81 FP_REG.withName("%xmm2"), 82 FP_REG.withName("%xmm3"), 83 FP_REG.withName("%xmm4"), 84 FP_REG.withName("%xmm5"), 85 FP_REG.withName("%xmm6"), 86 FP_REG.withName("%xmm7") 87 // specification and implementation differ as to whether the following are part of a reg save area 88 // Let's go with the implementation, since then it actually works :) 89 // FP_REG.withName("%xmm8"), 90 // FP_REG.withName("%xmm9"), 91 // FP_REG.withName("%xmm10"), 92 // FP_REG.withName("%xmm11"), 93 // FP_REG.withName("%xmm12"), 94 // FP_REG.withName("%xmm13"), 95 // FP_REG.withName("%xmm14"), 96 // FP_REG.withName("%xmm15") 97 ); 98 99 private static final long FP_OFFSET = LAYOUT_REG_SAVE_AREA.byteOffset(groupElement("%xmm0")); 100 101 private static final int GP_SLOT_SIZE = (int) GP_REG.byteSize(); 102 private static final int FP_SLOT_SIZE = (int) FP_REG.byteSize(); 103 104 private static final int MAX_GP_OFFSET = (int) FP_OFFSET; // 6 regs used 105 private static final int MAX_FP_OFFSET = (int) LAYOUT_REG_SAVE_AREA.byteSize(); // 8 16 byte regs 106 107 private static final VarHandle VH_fp_offset = LAYOUT.varHandle(groupElement("fp_offset")); 108 private static final VarHandle VH_gp_offset = LAYOUT.varHandle(groupElement("gp_offset")); 109 private static final VarHandle VH_overflow_arg_area = LAYOUT.varHandle(groupElement("overflow_arg_area")); 110 private static final VarHandle VH_reg_save_area = LAYOUT.varHandle(groupElement("reg_save_area")); 111 112 private static final VaList EMPTY = new SharedUtils.EmptyVaList(emptyListAddress()); 113 114 private final MemorySegment segment; 115 private final MemorySegment regSaveArea; 116 117 private SysVVaList(MemorySegment segment, MemorySegment regSaveArea) { 118 this.segment = segment; 119 this.regSaveArea = regSaveArea; 120 } 121 122 private static SysVVaList readFromSegment(MemorySegment segment) { 123 MemorySegment regSaveArea = getRegSaveArea(segment); 124 return new SysVVaList(segment, regSaveArea); 125 } 126 127 private static MemoryAddress emptyListAddress() { 128 long ptr = U.allocateMemory(LAYOUT.byteSize()); 129 ResourceScope scope = ResourceScope.newImplicitScope(); 130 scope.addCloseAction(() -> U.freeMemory(ptr)); 131 MemorySegment base = MemorySegment.ofAddress(MemoryAddress.ofLong(ptr), 132 LAYOUT.byteSize(), scope); 133 VH_gp_offset.set(base, MAX_GP_OFFSET); 134 VH_fp_offset.set(base, MAX_FP_OFFSET); 135 VH_overflow_arg_area.set(base, MemoryAddress.NULL); 136 VH_reg_save_area.set(base, MemoryAddress.NULL); 137 return base.address(); 138 } 139 140 public static VaList empty() { 141 return EMPTY; 142 } 143 144 private int currentGPOffset() { 145 return (int) VH_gp_offset.get(segment); 146 } 147 148 private void currentGPOffset(int i) { 149 VH_gp_offset.set(segment, i); 150 } 151 152 private int currentFPOffset() { 153 return (int) VH_fp_offset.get(segment); 154 } 155 156 private void currentFPOffset(int i) { 157 VH_fp_offset.set(segment, i); 158 } 159 160 private MemoryAddress stackPtr() { 161 return (MemoryAddress) VH_overflow_arg_area.get(segment); 162 } 163 164 private void stackPtr(MemoryAddress ptr) { 165 VH_overflow_arg_area.set(segment, ptr); 166 } 167 168 private MemorySegment regSaveArea() { 169 return getRegSaveArea(segment); 170 } 171 172 private static MemorySegment getRegSaveArea(MemorySegment segment) { 173 return MemorySegment.ofAddress(((MemoryAddress)VH_reg_save_area.get(segment)), 174 LAYOUT_REG_SAVE_AREA.byteSize(), segment.scope()); 175 } 176 177 private void preAlignStack(MemoryLayout layout) { 178 if (layout.byteAlignment() > 8) { 179 stackPtr(Utils.alignUp(stackPtr(), 16)); 180 } 181 } 182 183 private void postAlignStack(MemoryLayout layout) { 184 stackPtr(Utils.alignUp(stackPtr().addOffset(layout.byteSize()), 8)); 185 } 186 187 @Override 188 public int nextVarg(ValueLayout.OfInt layout) { 189 return (int) read(int.class, layout); 190 } 191 192 @Override 193 public long nextVarg(ValueLayout.OfLong layout) { 194 return (long) read(long.class, layout); 195 } 196 197 @Override 198 public double nextVarg(ValueLayout.OfDouble layout) { 199 return (double) read(double.class, layout); 200 } 201 202 @Override 203 public MemoryAddress nextVarg(ValueLayout.OfAddress layout) { 204 return (MemoryAddress) read(MemoryAddress.class, layout); 205 } 206 207 @Override 208 public MemorySegment nextVarg(GroupLayout layout, SegmentAllocator allocator) { 209 Objects.requireNonNull(allocator); 210 return (MemorySegment) read(MemorySegment.class, layout, allocator); 211 } 212 213 private Object read(Class<?> carrier, MemoryLayout layout) { 214 return read(carrier, layout, THROWING_ALLOCATOR); 215 } 216 217 private Object read(Class<?> carrier, MemoryLayout layout, SegmentAllocator allocator) { 218 Objects.requireNonNull(layout); 219 TypeClass typeClass = TypeClass.classifyLayout(layout); 220 if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass) 221 || typeClass.inMemory()) { 222 preAlignStack(layout); 223 return switch (typeClass.kind()) { 224 case STRUCT -> { 225 MemorySegment slice = MemorySegment.ofAddress(stackPtr(), layout.byteSize(), scope()); 226 MemorySegment seg = allocator.allocate(layout); 227 seg.copyFrom(slice); 228 postAlignStack(layout); 229 yield seg; 230 } 231 case POINTER, INTEGER, FLOAT -> { 232 VarHandle reader = layout.varHandle(); 233 try (ResourceScope localScope = ResourceScope.newConfinedScope()) { 234 MemorySegment slice = MemorySegment.ofAddress(stackPtr(), layout.byteSize(), localScope); 235 Object res = reader.get(slice); 236 postAlignStack(layout); 237 yield res; 238 } 239 } 240 }; 241 } else { 242 return switch (typeClass.kind()) { 243 case STRUCT -> { 244 MemorySegment value = allocator.allocate(layout); 245 int classIdx = 0; 246 long offset = 0; 247 while (offset < layout.byteSize()) { 248 final long copy = Math.min(layout.byteSize() - offset, 8); 249 boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE; 250 if (isSSE) { 251 MemorySegment.copy(regSaveArea, currentFPOffset(), value, offset, copy); 252 currentFPOffset(currentFPOffset() + FP_SLOT_SIZE); 253 } else { 254 MemorySegment.copy(regSaveArea, currentGPOffset(), value, offset, copy); 255 currentGPOffset(currentGPOffset() + GP_SLOT_SIZE); 256 } 257 offset += copy; 258 } 259 yield value; 260 } 261 case POINTER, INTEGER -> { 262 VarHandle reader = layout.varHandle(); 263 Object res = reader.get(regSaveArea.asSlice(currentGPOffset())); 264 currentGPOffset(currentGPOffset() + GP_SLOT_SIZE); 265 yield res; 266 } 267 case FLOAT -> { 268 VarHandle reader = layout.varHandle(); 269 Object res = reader.get(regSaveArea.asSlice(currentFPOffset())); 270 currentFPOffset(currentFPOffset() + FP_SLOT_SIZE); 271 yield res; 272 } 273 }; 274 } 275 } 276 277 @Override 278 public void skip(MemoryLayout... layouts) { 279 Objects.requireNonNull(layouts); 280 ((ResourceScopeImpl)segment.scope()).checkValidStateSlow(); 281 for (MemoryLayout layout : layouts) { 282 Objects.requireNonNull(layout); 283 TypeClass typeClass = TypeClass.classifyLayout(layout); 284 if (isRegOverflow(currentGPOffset(), currentFPOffset(), typeClass)) { 285 preAlignStack(layout); 286 postAlignStack(layout); 287 } else { 288 currentGPOffset(currentGPOffset() + (((int) typeClass.nIntegerRegs()) * GP_SLOT_SIZE)); 289 currentFPOffset(currentFPOffset() + (((int) typeClass.nVectorRegs()) * FP_SLOT_SIZE)); 290 } 291 } 292 } 293 294 static SysVVaList.Builder builder(ResourceScope scope) { 295 return new SysVVaList.Builder(scope); 296 } 297 298 public static VaList ofAddress(MemoryAddress ma, ResourceScope scope) { 299 return readFromSegment(MemorySegment.ofAddress(ma, LAYOUT.byteSize(), scope)); 300 } 301 302 @Override 303 public ResourceScope scope() { 304 return segment.scope(); 305 } 306 307 @Override 308 public VaList copy() { 309 MemorySegment copy = MemorySegment.allocateNative(LAYOUT, segment.scope()); 310 copy.copyFrom(segment); 311 return new SysVVaList(copy, regSaveArea); 312 } 313 314 @Override 315 public MemoryAddress address() { 316 return segment.address(); 317 } 318 319 private static boolean isRegOverflow(long currentGPOffset, long currentFPOffset, TypeClass typeClass) { 320 return currentGPOffset > MAX_GP_OFFSET - typeClass.nIntegerRegs() * GP_SLOT_SIZE 321 || currentFPOffset > MAX_FP_OFFSET - typeClass.nVectorRegs() * FP_SLOT_SIZE; 322 } 323 324 @Override 325 public String toString() { 326 return "SysVVaList{" 327 + "gp_offset=" + currentGPOffset() 328 + ", fp_offset=" + currentFPOffset() 329 + ", overflow_arg_area=" + stackPtr() 330 + ", reg_save_area=" + regSaveArea() 331 + '}'; 332 } 333 334 public static non-sealed class Builder implements VaList.Builder { 335 private final ResourceScope scope; 336 private final MemorySegment reg_save_area; 337 private long currentGPOffset = 0; 338 private long currentFPOffset = FP_OFFSET; 339 private final List<SimpleVaArg> stackArgs = new ArrayList<>(); 340 341 public Builder(ResourceScope scope) { 342 this.scope = scope; 343 this.reg_save_area = MemorySegment.allocateNative(LAYOUT_REG_SAVE_AREA, scope); 344 } 345 346 @Override 347 public Builder addVarg(ValueLayout.OfInt layout, int value) { 348 return arg(int.class, layout, value); 349 } 350 351 @Override 352 public Builder addVarg(ValueLayout.OfLong layout, long value) { 353 return arg(long.class, layout, value); 354 } 355 356 @Override 357 public Builder addVarg(ValueLayout.OfDouble layout, double value) { 358 return arg(double.class, layout, value); 359 } 360 361 @Override 362 public Builder addVarg(ValueLayout.OfAddress layout, Addressable value) { 363 return arg(MemoryAddress.class, layout, value.address()); 364 } 365 366 @Override 367 public Builder addVarg(GroupLayout layout, MemorySegment value) { 368 return arg(MemorySegment.class, layout, value); 369 } 370 371 private Builder arg(Class<?> carrier, MemoryLayout layout, Object value) { 372 Objects.requireNonNull(layout); 373 Objects.requireNonNull(value); 374 TypeClass typeClass = TypeClass.classifyLayout(layout); 375 if (isRegOverflow(currentGPOffset, currentFPOffset, typeClass) 376 || typeClass.inMemory()) { 377 // stack it! 378 stackArgs.add(new SimpleVaArg(carrier, layout, value)); 379 } else { 380 switch (typeClass.kind()) { 381 case STRUCT -> { 382 MemorySegment valueSegment = (MemorySegment) value; 383 int classIdx = 0; 384 long offset = 0; 385 while (offset < layout.byteSize()) { 386 final long copy = Math.min(layout.byteSize() - offset, 8); 387 boolean isSSE = typeClass.classes.get(classIdx++) == ArgumentClassImpl.SSE; 388 if (isSSE) { 389 MemorySegment.copy(valueSegment, offset, reg_save_area, currentFPOffset, copy); 390 currentFPOffset += FP_SLOT_SIZE; 391 } else { 392 MemorySegment.copy(valueSegment, offset, reg_save_area, currentGPOffset, copy); 393 currentGPOffset += GP_SLOT_SIZE; 394 } 395 offset += copy; 396 } 397 } 398 case POINTER, INTEGER -> { 399 VarHandle writer = layout.varHandle(); 400 writer.set(reg_save_area.asSlice(currentGPOffset), value); 401 currentGPOffset += GP_SLOT_SIZE; 402 } 403 case FLOAT -> { 404 VarHandle writer = layout.varHandle(); 405 writer.set(reg_save_area.asSlice(currentFPOffset), value); 406 currentFPOffset += FP_SLOT_SIZE; 407 } 408 } 409 } 410 return this; 411 } 412 413 private boolean isEmpty() { 414 return currentGPOffset == 0 && currentFPOffset == FP_OFFSET && stackArgs.isEmpty(); 415 } 416 417 public VaList build() { 418 if (isEmpty()) { 419 return EMPTY; 420 } 421 422 SegmentAllocator allocator = SegmentAllocator.newNativeArena(scope); 423 MemorySegment vaListSegment = allocator.allocate(LAYOUT); 424 MemoryAddress stackArgsPtr = MemoryAddress.NULL; 425 if (!stackArgs.isEmpty()) { 426 long stackArgsSize = stackArgs.stream().reduce(0L, (acc, e) -> acc + e.layout.byteSize(), Long::sum); 427 MemorySegment stackArgsSegment = allocator.allocate(stackArgsSize, 16); 428 MemorySegment maOverflowArgArea = stackArgsSegment; 429 for (SimpleVaArg arg : stackArgs) { 430 if (arg.layout.byteSize() > 8) { 431 maOverflowArgArea = Utils.alignUp(maOverflowArgArea, Math.min(16, arg.layout.byteSize())); 432 } 433 if (arg.value instanceof MemorySegment) { 434 maOverflowArgArea.copyFrom((MemorySegment) arg.value); 435 } else { 436 VarHandle writer = arg.varHandle(); 437 writer.set(maOverflowArgArea, arg.value); 438 } 439 maOverflowArgArea = maOverflowArgArea.asSlice(arg.layout.byteSize()); 440 } 441 stackArgsPtr = stackArgsSegment.address(); 442 } 443 444 VH_fp_offset.set(vaListSegment, (int) FP_OFFSET); 445 VH_overflow_arg_area.set(vaListSegment, stackArgsPtr); 446 VH_reg_save_area.set(vaListSegment, reg_save_area.address()); 447 assert reg_save_area.scope().ownerThread() == vaListSegment.scope().ownerThread(); 448 return new SysVVaList(vaListSegment, reg_save_area); 449 } 450 } 451 }