< prev index next >

src/java.base/share/classes/java/lang/invoke/X-VarHandle.java.template

Print this page
@@ -1,7 +1,7 @@
  /*
-  * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+  * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   *
   * This code is free software; you can redistribute it and/or modify it
   * under the terms of the GNU General Public License version 2 only, as
   * published by the Free Software Foundation.  Oracle designates this

@@ -22,56 +22,68 @@
   * or visit www.oracle.com if you need additional information or have any
   * questions.
   */
  package java.lang.invoke;
  
+ #if[Object]
+ import jdk.internal.value.ValueClass;
+ #end[Object]
  import jdk.internal.util.Preconditions;
  import jdk.internal.vm.annotation.ForceInline;
  
  import java.lang.invoke.VarHandle.VarHandleDesc;
+ import java.lang.reflect.Field;
  import java.util.Objects;
  import java.util.Optional;
  
  import static java.lang.invoke.MethodHandleStatics.UNSAFE;
  
  #warn
  
- final class VarHandle$Type$s {
+ final class VarHandle$InputType$s {
  
      static sealed class FieldInstanceReadOnly extends VarHandle {
          final long fieldOffset;
          final Class<?> receiverType;
  #if[Object]
          final Class<?> fieldType;
+         final boolean nullRestricted;
  #end[Object]
+ #if[FlatValue]
+         final int layout; // Unsafe.fieldLayout
+ #end[FlatValue]
  
-         FieldInstanceReadOnly(Class<?> receiverType, long fieldOffset{#if[Object]?, Class<?> fieldType}) {
-             this(receiverType, fieldOffset{#if[Object]?, fieldType}, FieldInstanceReadOnly.FORM, false);
+         FieldInstanceReadOnly(Class<?> receiverType, long fieldOffset{#if[Object]?, Class<?> fieldType, boolean nullRestricted}{#if[FlatValue]?, int layout}) {
+             this(receiverType, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, FieldInstanceReadOnly.FORM, false);
          }
  
-         protected FieldInstanceReadOnly(Class<?> receiverType, long fieldOffset{#if[Object]?, Class<?> fieldType},
+         protected FieldInstanceReadOnly(Class<?> receiverType, long fieldOffset{#if[Object]?, Class<?> fieldType, boolean nullRestricted}{#if[FlatValue]?, int layout},
                                          VarForm form, boolean exact) {
              super(form, exact);
              this.fieldOffset = fieldOffset;
              this.receiverType = receiverType;
  #if[Object]
              this.fieldType = fieldType;
+             this.nullRestricted = nullRestricted;
  #end[Object]
+ #if[FlatValue]
+             this.layout = layout;
+ #end[FlatValue]
          }
  
          @Override
          public FieldInstanceReadOnly withInvokeExactBehavior() {
              return hasInvokeExactBehavior()
                  ? this
-                 : new FieldInstanceReadOnly(receiverType, fieldOffset{#if[Object]?, fieldType}, vform, true);
+                 : new FieldInstanceReadOnly(receiverType, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, vform, true);
          }
  
          @Override
          public FieldInstanceReadOnly withInvokeBehavior() {
              return !hasInvokeExactBehavior()
                  ? this
-                 : new FieldInstanceReadOnly(receiverType, fieldOffset{#if[Object]?, fieldType}, vform, false);
+                 : new FieldInstanceReadOnly(receiverType, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, vform, false);
          }
  
          @Override
          final MethodType accessModeTypeUncached(AccessType at) {
              return at.accessModeType(receiverType, {#if[Object]?fieldType:$type$.class});

@@ -91,190 +103,228 @@
          }
  
          @ForceInline
          static $type$ get(VarHandle ob, Object holder) {
              FieldInstanceReadOnly handle = (FieldInstanceReadOnly)ob;
-             return UNSAFE.get$Type$(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                  handle.fieldOffset);
+             $type$ value = UNSAFE.get$Type$(Objects.requireNonNull(handle.receiverType.cast(holder)),
+                                  handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType});
+ #if[Reference]
+             if (value == null && handle.nullRestricted) {
+                 throw new NullPointerException("Uninitialized null-restricted field");
+             }
+ #end[Reference]
+             return value;
          }
  
+ #if[NonPlainAccess]
          @ForceInline
          static $type$ getVolatile(VarHandle ob, Object holder) {
              FieldInstanceReadOnly handle = (FieldInstanceReadOnly)ob;
-             return UNSAFE.get$Type$Volatile(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                  handle.fieldOffset);
+             $type$ value = UNSAFE.get$Type$Volatile(Objects.requireNonNull(handle.receiverType.cast(holder)),
+                                  handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType});
+ #if[Reference]
+             if (value == null && handle.nullRestricted) {
+                 throw new NullPointerException("Uninitialized null-restricted field");
+             }
+ #end[Reference]
+             return value;
          }
  
          @ForceInline
          static $type$ getOpaque(VarHandle ob, Object holder) {
              FieldInstanceReadOnly handle = (FieldInstanceReadOnly)ob;
-             return UNSAFE.get$Type$Opaque(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                  handle.fieldOffset);
+             $type$ value = UNSAFE.get$Type$Opaque(Objects.requireNonNull(handle.receiverType.cast(holder)),
+                                  handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType});
+ #if[Reference]
+             if (value == null && handle.nullRestricted) {
+                 throw new NullPointerException("Uninitialized null-restricted field");
+ 
+             }
+ #end[Reference]
+             return value;
          }
  
          @ForceInline
          static $type$ getAcquire(VarHandle ob, Object holder) {
              FieldInstanceReadOnly handle = (FieldInstanceReadOnly)ob;
-             return UNSAFE.get$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                  handle.fieldOffset);
+             $type$ value = UNSAFE.get$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)),
+                                  handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType});
+ #if[Reference]
+             if (value == null && handle.nullRestricted) {
+                 throw new NullPointerException("Uninitialized null-restricted field");
+             }
+ #end[Reference]
+             return value;
          }
+ #end[NonPlainAccess]
  
          static final VarForm FORM = new VarForm(FieldInstanceReadOnly.class, Object.class, $type$.class);
      }
  
      static final class FieldInstanceReadWrite extends FieldInstanceReadOnly {
- 
-         FieldInstanceReadWrite(Class<?> receiverType, long fieldOffset{#if[Object]?, Class<?> fieldType}) {
-             this(receiverType, fieldOffset{#if[Object]?, fieldType}, false);
+         FieldInstanceReadWrite(Class<?> receiverType, long fieldOffset{#if[Object]?, Class<?> fieldType, boolean nullRestricted}{#if[FlatValue]?, int layout}) {
+             this(receiverType, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, false);
          }
  
-         private FieldInstanceReadWrite(Class<?> receiverType, long fieldOffset{#if[Object]?, Class<?> fieldType},
+         private FieldInstanceReadWrite(Class<?> receiverType, long fieldOffset{#if[Object]?, Class<?> fieldType, boolean nullRestricted}{#if[FlatValue]?, int layout},
                                         boolean exact) {
-             super(receiverType, fieldOffset{#if[Object]?, fieldType}, FieldInstanceReadWrite.FORM, exact);
+             super(receiverType, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, FieldInstanceReadWrite.FORM, exact);
          }
  
          @Override
          public FieldInstanceReadWrite withInvokeExactBehavior() {
              return hasInvokeExactBehavior()
                  ? this
-                 : new FieldInstanceReadWrite(receiverType, fieldOffset{#if[Object]?, fieldType}, true);
+                 : new FieldInstanceReadWrite(receiverType, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, true);
          }
  
          @Override
          public FieldInstanceReadWrite withInvokeBehavior() {
              return !hasInvokeExactBehavior()
                  ? this
-                 : new FieldInstanceReadWrite(receiverType, fieldOffset{#if[Object]?, fieldType}, false);
+                 : new FieldInstanceReadWrite(receiverType, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, false);
          }
  
+ #if[Object]
+         @ForceInline
+         static Object checkCast(FieldInstanceReadWrite handle, $type$ value) {
+ #if[Reference]
+             if (value == null && handle.nullRestricted)
+                 throw new NullPointerException("Uninitialized null-restricted field");
+ #end[Reference]
+             return handle.fieldType.cast(value);
+         }
+ #end[Object]
+ 
          @ForceInline
          static void set(VarHandle ob, Object holder, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              UNSAFE.put$Type$(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                              handle.fieldOffset,
-                              {#if[Object]?handle.fieldType.cast(value):value});
+                              handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType},
+                              {#if[Object]?checkCast(handle, value):value});
          }
  
+ #if[NonPlainAccess]
          @ForceInline
          static void setVolatile(VarHandle ob, Object holder, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              UNSAFE.put$Type$Volatile(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                      handle.fieldOffset,
-                                      {#if[Object]?handle.fieldType.cast(value):value});
+                                      handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType},
+                                      {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static void setOpaque(VarHandle ob, Object holder, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              UNSAFE.put$Type$Opaque(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                    handle.fieldOffset,
-                                    {#if[Object]?handle.fieldType.cast(value):value});
+                                    handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType},
+                                    {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static void setRelease(VarHandle ob, Object holder, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              UNSAFE.put$Type$Release(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                     handle.fieldOffset,
-                                     {#if[Object]?handle.fieldType.cast(value):value});
+                                     handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType},
+                                     {#if[Object]?checkCast(handle, value):value});
          }
  #if[CAS]
  
          @ForceInline
          static boolean compareAndSet(VarHandle ob, Object holder, $type$ expected, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              return UNSAFE.compareAndSet$Type$(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static $type$ compareAndExchange(VarHandle ob, Object holder, $type$ expected, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              return UNSAFE.compareAndExchange$Type$(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static $type$ compareAndExchangeAcquire(VarHandle ob, Object holder, $type$ expected, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              return UNSAFE.compareAndExchange$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static $type$ compareAndExchangeRelease(VarHandle ob, Object holder, $type$ expected, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              return UNSAFE.compareAndExchange$Type$Release(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static boolean weakCompareAndSetPlain(VarHandle ob, Object holder, $type$ expected, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              return UNSAFE.weakCompareAndSet$Type$Plain(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static boolean weakCompareAndSet(VarHandle ob, Object holder, $type$ expected, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              return UNSAFE.weakCompareAndSet$Type$(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static boolean weakCompareAndSetAcquire(VarHandle ob, Object holder, $type$ expected, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              return UNSAFE.weakCompareAndSet$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static boolean weakCompareAndSetRelease(VarHandle ob, Object holder, $type$ expected, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              return UNSAFE.weakCompareAndSet$Type$Release(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static $type$ getAndSet(VarHandle ob, Object holder, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              return UNSAFE.getAndSet$Type$(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                           handle.fieldOffset,
-                                           {#if[Object]?handle.fieldType.cast(value):value});
+                                           handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                           {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static $type$ getAndSetAcquire(VarHandle ob, Object holder, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              return UNSAFE.getAndSet$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                           handle.fieldOffset,
-                                           {#if[Object]?handle.fieldType.cast(value):value});
+                                           handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                           {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static $type$ getAndSetRelease(VarHandle ob, Object holder, $type$ value) {
              FieldInstanceReadWrite handle = (FieldInstanceReadWrite)ob;
              return UNSAFE.getAndSet$Type$Release(Objects.requireNonNull(handle.receiverType.cast(holder)),
-                                           handle.fieldOffset,
-                                           {#if[Object]?handle.fieldType.cast(value):value});
+                                           handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                           {#if[Object]?checkCast(handle, value):value});
          }
  #end[CAS]
  #if[AtomicAdd]
  
          @ForceInline

@@ -374,50 +424,59 @@
              return UNSAFE.getAndBitwiseXor$Type$Acquire(Objects.requireNonNull(handle.receiverType.cast(holder)),
                                         handle.fieldOffset,
                                         value);
          }
  #end[Bitwise]
+ #end[NonPlainAccess]
  
          static final VarForm FORM = new VarForm(FieldInstanceReadWrite.class, Object.class, $type$.class);
      }
  
- 
+ #if[Static]
      static sealed class FieldStaticReadOnly extends VarHandle {
          final Class<?> declaringClass;
          final Object base;
          final long fieldOffset;
  #if[Object]
          final Class<?> fieldType;
+         final boolean nullRestricted;
  #end[Object]
+ #if[FlatValue]
+         final int layout;
+ #end[FlatValue]
  
-         FieldStaticReadOnly(Class<?> declaringClass, Object base, long fieldOffset{#if[Object]?, Class<?> fieldType}) {
-             this(declaringClass, base, fieldOffset{#if[Object]?, fieldType}, FieldStaticReadOnly.FORM, false);
+         FieldStaticReadOnly(Class<?> declaringClass, Object base, long fieldOffset{#if[Object]?, Class<?> fieldType, boolean nullRestricted}{#if[FlatValue]?, int layout}) {
+             this(declaringClass, base, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, FieldStaticReadOnly.FORM, false);
          }
  
-         protected FieldStaticReadOnly(Class<?> declaringClass, Object base, long fieldOffset{#if[Object]?, Class<?> fieldType},
+         protected FieldStaticReadOnly(Class<?> declaringClass, Object base, long fieldOffset{#if[Object]?, Class<?> fieldType, boolean nullRestricted}{#if[FlatValue]?, int layout},
                                        VarForm form, boolean exact) {
              super(form, exact);
              this.declaringClass = declaringClass;
              this.base = base;
              this.fieldOffset = fieldOffset;
  #if[Object]
              this.fieldType = fieldType;
+             this.nullRestricted = nullRestricted;
  #end[Object]
+ #if[FlatValue]
+             this.layout = layout;
+ #end[FlatValue]
          }
  
          @Override
          public FieldStaticReadOnly withInvokeExactBehavior() {
              return hasInvokeExactBehavior()
                  ? this
-                 : new FieldStaticReadOnly(declaringClass, base, fieldOffset{#if[Object]?, fieldType}, vform, true);
+                 : new FieldStaticReadOnly(declaringClass, base, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, vform, true);
          }
  
          @Override
          public FieldStaticReadOnly withInvokeBehavior() {
              return !hasInvokeExactBehavior()
                  ? this
-                 : new FieldStaticReadOnly(declaringClass, base, fieldOffset{#if[Object]?, fieldType}, vform, false);
+                 : new FieldStaticReadOnly(declaringClass, base, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, vform, false);
          }
  
          @Override
          public Optional<VarHandleDesc> describeConstable() {
              var fieldTypeRef = {#if[Object]?fieldType:$type$.class}.describeConstable();

@@ -439,191 +498,225 @@
          }
  
          @ForceInline
          static $type$ get(VarHandle ob) {
              FieldStaticReadOnly handle = (FieldStaticReadOnly) ob.target();
-             return UNSAFE.get$Type$(handle.base,
-                                  handle.fieldOffset);
+             $type$ value = UNSAFE.get$Type$(handle.base,
+                                  handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType});
+ #if[Reference]
+             if (value == null && handle.nullRestricted) {
+                 throw new NullPointerException("Uninitialized null-restricted field");
+             }
+ #end[Reference]
+             return value;
          }
  
+ #if[NonPlainAccess]
          @ForceInline
          static $type$ getVolatile(VarHandle ob) {
              FieldStaticReadOnly handle = (FieldStaticReadOnly) ob.target();
-             return UNSAFE.get$Type$Volatile(handle.base,
-                                  handle.fieldOffset);
+             $type$ value = UNSAFE.get$Type$Volatile(handle.base,
+                                  handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType});
+ #if[Reference]
+             if (value == null && handle.nullRestricted) {
+                 throw new NullPointerException("Uninitialized null-restricted field");
+             }
+ #end[Reference]
+             return value;
          }
  
          @ForceInline
          static $type$ getOpaque(VarHandle ob) {
              FieldStaticReadOnly handle = (FieldStaticReadOnly) ob.target();
-             return UNSAFE.get$Type$Opaque(handle.base,
-                                  handle.fieldOffset);
+             $type$ value = UNSAFE.get$Type$Opaque(handle.base,
+                                  handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType});
+ #if[Reference]
+             if (value == null && handle.nullRestricted) {
+                 throw new NullPointerException("Uninitialized null-restricted field");
+             }
+ #end[Reference]
+             return value;
          }
  
          @ForceInline
          static $type$ getAcquire(VarHandle ob) {
              FieldStaticReadOnly handle = (FieldStaticReadOnly) ob.target();
-             return UNSAFE.get$Type$Acquire(handle.base,
-                                  handle.fieldOffset);
+             $type$ value = UNSAFE.get$Type$Acquire(handle.base,
+                                  handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType});
+ #if[Reference]
+             if (value == null && handle.nullRestricted) {
+                 throw new NullPointerException("Uninitialized null-restricted field");
+             }
+ #end[Reference]
+             return value;
          }
+ #end[NonPlainAccess]
  
          static final VarForm FORM = new VarForm(FieldStaticReadOnly.class, null, $type$.class);
      }
  
      static final class FieldStaticReadWrite extends FieldStaticReadOnly {
  
-         FieldStaticReadWrite(Class<?> declaringClass, Object base, long fieldOffset{#if[Object]?, Class<?> fieldType}) {
-             this(declaringClass, base, fieldOffset{#if[Object]?, fieldType}, false);
+         FieldStaticReadWrite(Class<?> declaringClass, Object base, long fieldOffset{#if[Object]?, Class<?> fieldType, boolean nullRestricted}{#if[FlatValue]?, int layout}) {
+             this(declaringClass, base, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, false);
          }
  
-         private FieldStaticReadWrite(Class<?> declaringClass, Object base, long fieldOffset{#if[Object]?, Class<?> fieldType},
+         private FieldStaticReadWrite(Class<?> declaringClass, Object base, long fieldOffset{#if[Object]?, Class<?> fieldType, boolean nullRestricted}{#if[FlatValue]?, int layout},
                                       boolean exact) {
-             super(declaringClass, base, fieldOffset{#if[Object]?, fieldType}, FieldStaticReadWrite.FORM, exact);
+             super(declaringClass, base, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, FieldStaticReadWrite.FORM, exact);
          }
  
          @Override
          public FieldStaticReadWrite withInvokeExactBehavior() {
              return hasInvokeExactBehavior()
                  ? this
-                 : new FieldStaticReadWrite(declaringClass, base, fieldOffset{#if[Object]?, fieldType}, true);
+                 : new FieldStaticReadWrite(declaringClass, base, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, true);
          }
  
          @Override
          public FieldStaticReadWrite withInvokeBehavior() {
              return !hasInvokeExactBehavior()
                  ? this
-                 : new FieldStaticReadWrite(declaringClass, base, fieldOffset{#if[Object]?, fieldType}, false);
+                 : new FieldStaticReadWrite(declaringClass, base, fieldOffset{#if[Object]?, fieldType, nullRestricted}{#if[FlatValue]?, layout}, false);
+         }
+ 
+ #if[Object]
+         @ForceInline
+         static Object checkCast(FieldStaticReadWrite handle, $type$ value) {
+             return handle.fieldType.cast(value);
          }
+ #end[Object]
  
          @ForceInline
          static void set(VarHandle ob, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              UNSAFE.put$Type$(handle.base,
-                              handle.fieldOffset,
-                              {#if[Object]?handle.fieldType.cast(value):value});
+                              handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType},
+                              {#if[Object]?checkCast(handle, value):value});
          }
  
+ #if[NonPlainAccess]
          @ForceInline
          static void setVolatile(VarHandle ob, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              UNSAFE.put$Type$Volatile(handle.base,
-                                      handle.fieldOffset,
-                                      {#if[Object]?handle.fieldType.cast(value):value});
+                                      handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType},
+                                      {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static void setOpaque(VarHandle ob, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              UNSAFE.put$Type$Opaque(handle.base,
-                                    handle.fieldOffset,
-                                    {#if[Object]?handle.fieldType.cast(value):value});
+                                    handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType},
+                                    {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static void setRelease(VarHandle ob, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              UNSAFE.put$Type$Release(handle.base,
-                                     handle.fieldOffset,
-                                     {#if[Object]?handle.fieldType.cast(value):value});
+                                     handle.fieldOffset{#if[FlatValue]?, handle.layout, handle.fieldType},
+                                     {#if[Object]?checkCast(handle, value):value});
          }
  #if[CAS]
  
          @ForceInline
          static boolean compareAndSet(VarHandle ob, $type$ expected, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              return UNSAFE.compareAndSet$Type$(handle.base,
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
  
          @ForceInline
          static $type$ compareAndExchange(VarHandle ob, $type$ expected, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              return UNSAFE.compareAndExchange$Type$(handle.base,
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static $type$ compareAndExchangeAcquire(VarHandle ob, $type$ expected, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              return UNSAFE.compareAndExchange$Type$Acquire(handle.base,
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static $type$ compareAndExchangeRelease(VarHandle ob, $type$ expected, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              return UNSAFE.compareAndExchange$Type$Release(handle.base,
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static boolean weakCompareAndSetPlain(VarHandle ob, $type$ expected, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              return UNSAFE.weakCompareAndSet$Type$Plain(handle.base,
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static boolean weakCompareAndSet(VarHandle ob, $type$ expected, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              return UNSAFE.weakCompareAndSet$Type$(handle.base,
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static boolean weakCompareAndSetAcquire(VarHandle ob, $type$ expected, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              return UNSAFE.weakCompareAndSet$Type$Acquire(handle.base,
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static boolean weakCompareAndSetRelease(VarHandle ob, $type$ expected, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              return UNSAFE.weakCompareAndSet$Type$Release(handle.base,
-                                                handle.fieldOffset,
-                                                {#if[Object]?handle.fieldType.cast(expected):expected},
-                                                {#if[Object]?handle.fieldType.cast(value):value});
+                                                handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                                {#if[Object]?checkCast(handle, expected):expected},
+                                                {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static $type$ getAndSet(VarHandle ob, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              return UNSAFE.getAndSet$Type$(handle.base,
-                                           handle.fieldOffset,
-                                           {#if[Object]?handle.fieldType.cast(value):value});
+                                           handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                           {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static $type$ getAndSetAcquire(VarHandle ob, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              return UNSAFE.getAndSet$Type$Acquire(handle.base,
-                                           handle.fieldOffset,
-                                           {#if[Object]?handle.fieldType.cast(value):value});
+                                           handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                           {#if[Object]?checkCast(handle, value):value});
          }
  
          @ForceInline
          static $type$ getAndSetRelease(VarHandle ob, $type$ value) {
              FieldStaticReadWrite handle = (FieldStaticReadWrite) ob.target();
              return UNSAFE.getAndSet$Type$Release(handle.base,
-                                           handle.fieldOffset,
-                                           {#if[Object]?handle.fieldType.cast(value):value});
+                                           handle.fieldOffset{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.fieldType},
+                                           {#if[Object]?checkCast(handle, value):value});
          }
  #end[CAS]
  #if[AtomicAdd]
  
          @ForceInline

@@ -722,49 +815,57 @@
              return UNSAFE.getAndBitwiseXor$Type$Acquire(handle.base,
                                         handle.fieldOffset,
                                         value);
          }
  #end[Bitwise]
+ #end[NonPlainAccess]
  
          static final VarForm FORM = new VarForm(FieldStaticReadWrite.class, null, $type$.class);
      }
+ #end[Static]
  
- 
+ #if[Array]
      static final class Array extends VarHandle {
          final int abase;
          final int ashift;
  #if[Object]
          final Class<{#if[Object]??:$type$[]}> arrayType;
          final Class<?> componentType;
  #end[Object]
+ #if[FlatValue]
+         final int layout;
+ #end[FlatValue]
  
-         Array(int abase, int ashift{#if[Object]?, Class<?> arrayType}) {
-             this(abase, ashift{#if[Object]?, arrayType}, false);
+         Array(int abase, int ashift{#if[Object]?, Class<?> arrayType}{#if[FlatValue]?, int layout}) {
+             this(abase, ashift{#if[Object]?, arrayType}{#if[FlatValue]?, layout}, false);
          }
  
-         private Array(int abase, int ashift{#if[Object]?, Class<?> arrayType}, boolean exact) {
+         private Array(int abase, int ashift{#if[Object]?, Class<?> arrayType}{#if[FlatValue]?, int layout}, boolean exact) {
              super(Array.FORM, exact);
              this.abase = abase;
              this.ashift = ashift;
  #if[Object]
              this.arrayType = {#if[Object]?arrayType:$type$[].class};
              this.componentType = arrayType.getComponentType();
  #end[Object]
+ #if[FlatValue]
+             this.layout = layout;
+ #end[FlatValue]
          }
  
          @Override
          public Array withInvokeExactBehavior() {
              return hasInvokeExactBehavior()
                  ? this
-                 : new Array(abase, ashift{#if[Object]?, arrayType}, true);
+                 : new Array(abase, ashift{#if[Object]?, arrayType}{#if[FlatValue]?, layout}, true);
          }
  
          @Override
          public Array withInvokeBehavior() {
              return !hasInvokeExactBehavior()
                  ? this
-                 : new Array(abase, ashift{#if[Object]?, arrayType}, false);
+                 : new Array(abase, ashift{#if[Object]?, arrayType}{#if[FlatValue]?, layout}, false);
          }
  
          @Override
          public Optional<VarHandleDesc> describeConstable() {
              var arrayTypeRef = {#if[Object]?arrayType:$type$[].class}.describeConstable();

@@ -774,16 +875,19 @@
              return Optional.of(VarHandleDesc.ofArray(arrayTypeRef.get()));
          }
  
          @Override
          final MethodType accessModeTypeUncached(AccessType at) {
-             return at.accessModeType({#if[Object]?arrayType:$type$[].class}, {#if[Object]?arrayType.getComponentType():$type$.class}, int.class);
+             return at.accessModeType({#if[Object]?arrayType:$type$[].class}, {#if[Object]?componentType:$type$.class}, int.class);
          }
  
  #if[Object]
          @ForceInline
          static Object runtimeTypeCheck(Array handle, Object[] oarray, Object value) {
+             if (value == null && ValueClass.isNullRestrictedArray(oarray)) {
+                 throw new NullPointerException("null not allowed for null-restricted array " + oarray.getClass().toGenericString());
+             }
              if (handle.arrayType == oarray.getClass()) {
                  // Fast path: static array type same as argument array type
                  return handle.componentType.cast(value);
              } else {
                  // Slow path: check value against argument array component type

@@ -818,21 +922,35 @@
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
-             array[index] = {#if[Object]?handle.componentType.cast(value):value};
+             array[index] = {#if[Object]?runtimeTypeCheck(handle, array, value):value};
          }
  
+ #if[NonPlainAccess]
          @ForceInline
          static $type$ getVolatile(VarHandle ob, Object oarray, int index) {
              Array handle = (Array)ob;
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.getFlatValueVolatile(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType());
+             }
+ #end[Reference]
              return UNSAFE.get$Type$Volatile(array,
                      (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase);
          }
  
          @ForceInline

@@ -841,63 +959,134 @@
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 UNSAFE.putFlatValueVolatile(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          runtimeTypeCheck(handle, array, value));
+                 return;
+             }
+ #end[Reference]
              UNSAFE.put$Type$Volatile(array,
                      (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
-                     {#if[Object]?runtimeTypeCheck(handle, array, value):value});
+                     {#if[FlatValue]?handle.layout, handle.componentType, }{#if[Object]?runtimeTypeCheck(handle, array, value):value});
          }
  
          @ForceInline
          static $type$ getOpaque(VarHandle ob, Object oarray, int index) {
              Array handle = (Array)ob;
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.getFlatValueOpaque(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType());
+             }
+ #end[Reference]
              return UNSAFE.get$Type$Opaque(array,
-                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase);
+                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase{#if[FlatValue]?, handle.layout, handle.componentType});
          }
  
          @ForceInline
          static void setOpaque(VarHandle ob, Object oarray, int index, $type$ value) {
              Array handle = (Array)ob;
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 UNSAFE.putFlatValueOpaque(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          runtimeTypeCheck(handle, array, value));
+                 return;
+             }
+ #end[Reference]
              UNSAFE.put$Type$Opaque(array,
                      (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
-                     {#if[Object]?runtimeTypeCheck(handle, array, value):value});
+                     {#if[FlatValue]?handle.layout, handle.componentType, }{#if[Object]?runtimeTypeCheck(handle, array, value):value});
          }
  
          @ForceInline
          static $type$ getAcquire(VarHandle ob, Object oarray, int index) {
              Array handle = (Array)ob;
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.getFlatValueAcquire(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType());
+             }
+ #end[Reference]
              return UNSAFE.get$Type$Acquire(array,
-                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase);
+                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase{#if[FlatValue]?, handle.layout, handle.componentType});
          }
  
          @ForceInline
          static void setRelease(VarHandle ob, Object oarray, int index, $type$ value) {
              Array handle = (Array)ob;
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 UNSAFE.putFlatValueRelease(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          runtimeTypeCheck(handle, array, value));
+                 return;
+             }
+ #end[Reference]
              UNSAFE.put$Type$Release(array,
                      (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
-                     {#if[Object]?runtimeTypeCheck(handle, array, value):value});
+                     {#if[FlatValue]?handle.layout, handle.componentType, }{#if[Object]?runtimeTypeCheck(handle, array, value):value});
          }
  #if[CAS]
  
          @ForceInline
          static boolean compareAndSet(VarHandle ob, Object oarray, int index, $type$ expected, $type$ value) {

@@ -905,12 +1094,27 @@
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.compareAndSetFlatValue(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          arrayType.componentType().cast(expected),
+                          runtimeTypeCheck(handle, array, value));
+             }
+ #end[Reference]
              return UNSAFE.compareAndSet$Type$(array,
-                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
+                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.componentType},
                      {#if[Object]?handle.componentType.cast(expected):expected},
                      {#if[Object]?runtimeTypeCheck(handle, array, value):value});
          }
  
          @ForceInline

@@ -919,12 +1123,27 @@
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.compareAndExchangeFlatValue(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          arrayType.componentType().cast(expected),
+                          runtimeTypeCheck(handle, array, value));
+             }
+ #end[Reference]
              return UNSAFE.compareAndExchange$Type$(array,
-                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
+                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.componentType},
                      {#if[Object]?handle.componentType.cast(expected):expected},
                      {#if[Object]?runtimeTypeCheck(handle, array, value):value});
          }
  
          @ForceInline

@@ -933,12 +1152,27 @@
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.compareAndExchangeFlatValueAcquire(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          arrayType.componentType().cast(expected),
+                          runtimeTypeCheck(handle, array, value));
+             }
+ #end[Reference]
              return UNSAFE.compareAndExchange$Type$Acquire(array,
-                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
+                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase{#if[FlatValue]?, handle.layout}{#if[Object]?, handle.componentType},
                      {#if[Object]?handle.componentType.cast(expected):expected},
                      {#if[Object]?runtimeTypeCheck(handle, array, value):value});
          }
  
          @ForceInline

@@ -947,12 +1181,27 @@
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.compareAndExchangeFlatValueRelease(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          arrayType.componentType().cast(expected),
+                          runtimeTypeCheck(handle, array, value));
+             }
+ #end[Reference]
              return UNSAFE.compareAndExchange$Type$Release(array,
-                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
+                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase{#if[Object]?, handle.componentType},
                      {#if[Object]?handle.componentType.cast(expected):expected},
                      {#if[Object]?runtimeTypeCheck(handle, array, value):value});
          }
  
          @ForceInline

@@ -961,12 +1210,27 @@
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.weakCompareAndSetFlatValuePlain(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          arrayType.componentType().cast(expected),
+                          runtimeTypeCheck(handle, array, value));
+             }
+ #end[Reference]
              return UNSAFE.weakCompareAndSet$Type$Plain(array,
-                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
+                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase{#if[Object]?, handle.componentType},
                      {#if[Object]?handle.componentType.cast(expected):expected},
                      {#if[Object]?runtimeTypeCheck(handle, array, value):value});
          }
  
          @ForceInline

@@ -975,12 +1239,27 @@
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.weakCompareAndSetFlatValue(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          arrayType.componentType().cast(expected),
+                          runtimeTypeCheck(handle, array, value));
+             }
+ #end[Reference]
              return UNSAFE.weakCompareAndSet$Type$(array,
-                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
+                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase{#if[Object]?, handle.componentType},
                      {#if[Object]?handle.componentType.cast(expected):expected},
                      {#if[Object]?runtimeTypeCheck(handle, array, value):value});
          }
  
          @ForceInline

@@ -989,12 +1268,27 @@
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.weakCompareAndSetFlatValueAcquire(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          arrayType.componentType().cast(expected),
+                          runtimeTypeCheck(handle, array, value));
+             }
+ #end[Reference]
              return UNSAFE.weakCompareAndSet$Type$Acquire(array,
-                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
+                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase{#if[Object]?, handle.componentType},
                      {#if[Object]?handle.componentType.cast(expected):expected},
                      {#if[Object]?runtimeTypeCheck(handle, array, value):value});
          }
  
          @ForceInline

@@ -1003,12 +1297,27 @@
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.weakCompareAndSetFlatValueRelease(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          arrayType.componentType().cast(expected),
+                          runtimeTypeCheck(handle, array, value));
+             }
+ #end[Reference]
              return UNSAFE.weakCompareAndSet$Type$Release(array,
-                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
+                     (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase{#if[Object]?, handle.componentType},
                      {#if[Object]?handle.componentType.cast(expected):expected},
                      {#if[Object]?runtimeTypeCheck(handle, array, value):value});
          }
  
          @ForceInline

@@ -1017,39 +1326,81 @@
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.getAndSetFlatValue(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          runtimeTypeCheck(handle, array, value));
+             }
+ #end[Reference]
              return UNSAFE.getAndSet$Type$(array,
                      (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
-                     {#if[Object]?runtimeTypeCheck(handle, array, value):value});
+                     {#if[Object]?handle.componentType, runtimeTypeCheck(handle, array, value):value});
          }
  
          @ForceInline
          static $type$ getAndSetAcquire(VarHandle ob, Object oarray, int index, $type$ value) {
              Array handle = (Array)ob;
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.getAndSetFlatValueAcquire(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          runtimeTypeCheck(handle, array, value));
+             }
+ #end[Reference]
              return UNSAFE.getAndSet$Type$Acquire(array,
                      (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
-                     {#if[Object]?runtimeTypeCheck(handle, array, value):value});
+                     {#if[Object]?handle.componentType, runtimeTypeCheck(handle, array, value):value});
          }
  
          @ForceInline
          static $type$ getAndSetRelease(VarHandle ob, Object oarray, int index, $type$ value) {
              Array handle = (Array)ob;
  #if[Object]
              Object[] array = (Object[]) handle.arrayType.cast(oarray);
  #else[Object]
              $type$[] array = ($type$[]) oarray;
  #end[Object]
+ #if[Reference]
+             Class<?> arrayType = oarray.getClass();
+             if (handle.arrayType != arrayType && ValueClass.isFlatArray(oarray)) {
+                 // delegate to flat access primitives
+                 VarHandles.checkAtomicFlatArray(array);
+                 int aoffset = (int) UNSAFE.arrayBaseOffset(arrayType);
+                 int ascale = UNSAFE.arrayIndexScale(arrayType);
+                 int ashift = 31 - Integer.numberOfLeadingZeros(ascale);
+                 int layout = UNSAFE.arrayLayout(arrayType);
+                 return UNSAFE.getAndSetFlatValueRelease(array,
+                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << ashift) + aoffset, layout, arrayType.componentType(),
+                          runtimeTypeCheck(handle, array, value));
+             }
+ #end[Reference]
              return UNSAFE.getAndSet$Type$Release(array,
                      (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
-                     {#if[Object]?runtimeTypeCheck(handle, array, value):value});
+                     {#if[Object]?handle.componentType, runtimeTypeCheck(handle, array, value):value});
          }
  #end[CAS]
  #if[AtomicAdd]
  
          @ForceInline

@@ -1160,9 +1511,11 @@
              return UNSAFE.getAndBitwiseXor$Type$Acquire(array,
                                         (((long) Preconditions.checkIndex(index, array.length, Preconditions.AIOOBE_FORMATTER)) << handle.ashift) + handle.abase,
                                         value);
          }
  #end[Bitwise]
+ #end[NonPlainAccess]
  
          static final VarForm FORM = new VarForm(Array.class, {#if[Object]?Object[].class:$type$[].class}, {#if[Object]?Object.class:$type$.class}, int.class);
      }
+ #end[Array]
  }
< prev index next >