1 /*
   2  * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.internal.misc;
  27 
  28 import jdk.internal.ref.Cleaner;
  29 import jdk.internal.value.ValueClass;
  30 import jdk.internal.vm.annotation.ForceInline;
  31 import jdk.internal.vm.annotation.IntrinsicCandidate;
  32 import sun.nio.ch.DirectBuffer;
  33 
  34 import java.lang.reflect.Field;
  35 import java.security.ProtectionDomain;
  36 
  37 import static jdk.internal.misc.UnsafeConstants.*;
  38 
  39 /**
  40  * A collection of methods for performing low-level, unsafe operations.
  41  * Although the class and all methods are public, use of this class is
  42  * limited because only trusted code can obtain instances of it.
  43  *
  44  * <em>Note:</em> It is the responsibility of the caller to make sure
  45  * arguments are checked before methods of this class are
  46  * called. While some rudimentary checks are performed on the input,
  47  * the checks are best effort and when performance is an overriding
  48  * priority, as when methods of this class are optimized by the
  49  * runtime compiler, some or all checks (if any) may be elided. Hence,
  50  * the caller must not rely on the checks and corresponding
  51  * exceptions!
  52  *
  53  * @author John R. Rose
  54  * @see #getUnsafe
  55  */
  56 
  57 public final class Unsafe {
  58 
  59     private static native void registerNatives();
  60     static {
  61         runtimeSetup();
  62     }
  63 
  64     // Called from JVM when loading an AOT cache
  65     private static void runtimeSetup() {
  66         registerNatives();
  67     }
  68 
  69     private Unsafe() {}
  70 
  71     private static final Unsafe theUnsafe = new Unsafe();
  72 
  73     /**
  74      * Provides the caller with the capability of performing unsafe
  75      * operations.
  76      *
  77      * <p>The returned {@code Unsafe} object should be carefully guarded
  78      * by the caller, since it can be used to read and write data at arbitrary
  79      * memory addresses.  It must never be passed to untrusted code.
  80      *
  81      * <p>Most methods in this class are very low-level, and correspond to a
  82      * small number of hardware instructions (on typical machines).  Compilers
  83      * are encouraged to optimize these methods accordingly.
  84      *
  85      * <p>Here is a suggested idiom for using unsafe operations:
  86      *
  87      * <pre> {@code
  88      * class MyTrustedClass {
  89      *   private static final Unsafe unsafe = Unsafe.getUnsafe();
  90      *   ...
  91      *   private long myCountAddress = ...;
  92      *   public int getCount() { return unsafe.getByte(myCountAddress); }
  93      * }}</pre>
  94      *
  95      * (It may assist compilers to make the local variable {@code final}.)
  96      */
  97     public static Unsafe getUnsafe() {
  98         return theUnsafe;
  99     }
 100 
 101     //--- peek and poke operations
 102     // (compilers should optimize these to memory ops)
 103 
 104     // These work on object fields in the Java heap.
 105     // They will not work on elements of packed arrays.
 106 
 107     /**
 108      * Fetches a value from a given Java variable.
 109      * More specifically, fetches a field or array element within the given
 110      * object {@code o} at the given offset, or (if {@code o} is null)
 111      * from the memory address whose numerical value is the given offset.
 112      * <p>
 113      * The results are undefined unless one of the following cases is true:
 114      * <ul>
 115      * <li>The offset was obtained from {@link #objectFieldOffset} on
 116      * the {@link java.lang.reflect.Field} of some Java field and the object
 117      * referred to by {@code o} is of a class compatible with that
 118      * field's class.
 119      *
 120      * <li>The offset and object reference {@code o} (either null or
 121      * non-null) were both obtained via {@link #staticFieldOffset}
 122      * and {@link #staticFieldBase} (respectively) from the
 123      * reflective {@link Field} representation of some Java field.
 124      *
 125      * <li>The object referred to by {@code o} is an array, and the offset
 126      * is an integer of the form {@code B+N*S}, where {@code N} is
 127      * a valid index into the array, and {@code B} and {@code S} are
 128      * the values obtained by {@link #arrayBaseOffset} and {@link
 129      * #arrayIndexScale} (respectively) from the array's class.  The value
 130      * referred to is the {@code N}<em>th</em> element of the array.
 131      *
 132      * </ul>
 133      * <p>
 134      * If one of the above cases is true, the call references a specific Java
 135      * variable (field or array element).  However, the results are undefined
 136      * if that variable is not in fact of the type returned by this method.
 137      * <p>
 138      * This method refers to a variable by means of two parameters, and so
 139      * it provides (in effect) a <em>double-register</em> addressing mode
 140      * for Java variables.  When the object reference is null, this method
 141      * uses its offset as an absolute address.  This is similar in operation
 142      * to methods such as {@link #getInt(long)}, which provide (in effect) a
 143      * <em>single-register</em> addressing mode for non-Java variables.
 144      * However, because Java variables may have a different layout in memory
 145      * from non-Java variables, programmers should not assume that these
 146      * two addressing modes are ever equivalent.  Also, programmers should
 147      * remember that offsets from the double-register addressing mode cannot
 148      * be portably confused with longs used in the single-register addressing
 149      * mode.
 150      *
 151      * @param o Java heap object in which the variable resides, if any, else
 152      *        null
 153      * @param offset indication of where the variable resides in a Java heap
 154      *        object, if any, else a memory address locating the variable
 155      *        statically
 156      * @return the value fetched from the indicated Java variable
 157      * @throws RuntimeException No defined exceptions are thrown, not even
 158      *         {@link NullPointerException}
 159      */
 160     @IntrinsicCandidate
 161     public native int getInt(Object o, long offset);
 162 
 163     /**
 164      * Stores a value into a given Java variable.
 165      * <p>
 166      * The first two parameters are interpreted exactly as with
 167      * {@link #getInt(Object, long)} to refer to a specific
 168      * Java variable (field or array element).  The given value
 169      * is stored into that variable.
 170      * <p>
 171      * The variable must be of the same type as the method
 172      * parameter {@code x}.
 173      *
 174      * @param o Java heap object in which the variable resides, if any, else
 175      *        null
 176      * @param offset indication of where the variable resides in a Java heap
 177      *        object, if any, else a memory address locating the variable
 178      *        statically
 179      * @param x the value to store into the indicated Java variable
 180      * @throws RuntimeException No defined exceptions are thrown, not even
 181      *         {@link NullPointerException}
 182      */
 183     @IntrinsicCandidate
 184     public native void putInt(Object o, long offset, int x);
 185 
 186 
 187     /**
 188      * Returns true if the given field is flattened.
 189      */
 190     public boolean isFlatField(Field f) {
 191         if (f == null) {
 192             throw new NullPointerException();
 193         }
 194         return isFlatField0(f);
 195     }
 196 
 197     private native boolean isFlatField0(Object o);
 198 
 199     /* Returns true if the given field has a null marker
 200      * <p>
 201      * Nullable flat fields are stored in a flattened representation
 202      * and have an associated null marker to indicate if the the field value is
 203      * null or the one stored with the flat representation
 204      */
 205 
 206     public boolean hasNullMarker(Field f) {
 207         if (f == null) {
 208             throw new NullPointerException();
 209         }
 210         return hasNullMarker0(f);
 211     }
 212 
 213     private native boolean hasNullMarker0(Object o);
 214 
 215     /* Returns the offset of the null marker of the field,
 216     * or -1 if the field doesn't have a null marker
 217     */
 218 
 219     public int nullMarkerOffset(Field f) {
 220         if (f == null) {
 221             throw new NullPointerException();
 222         }
 223         return nullMarkerOffset0(f);
 224     }
 225 
 226     private native int nullMarkerOffset0(Object o);
 227 
 228     public static final int NON_FLAT_LAYOUT = 0;
 229 
 230     /* Reports the kind of layout used for an element in the storage
 231      * allocation of the given array. Do not expect to perform any logic
 232      * or layout control with this value, it is just an opaque token
 233      * used for performance reasons.
 234      *
 235      * A layout of 0 indicates this array is not flat.
 236      */
 237     public int arrayLayout(Object[] array) {
 238         if (array == null) {
 239             throw new NullPointerException();
 240         }
 241         return arrayLayout0(array);
 242     }
 243 
 244     private native int arrayLayout0(Object[] array);
 245 
 246 
 247     /* Reports the kind of layout used for a given field in the storage
 248      * allocation of its class.  Do not expect to perform any logic
 249      * or layout control with this value, it is just an opaque token
 250      * used for performance reasons.
 251      *
 252      * A layout of 0 indicates this field is not flat.
 253      */
 254     public int fieldLayout(Field f) {
 255         if (f == null) {
 256             throw new NullPointerException();
 257         }
 258         return fieldLayout0(f);
 259     }
 260 
 261     private native int fieldLayout0(Object o);
 262 
 263     public native Object[] newSpecialArray(Class<?> componentType,
 264                                                   int length, int layoutKind);
 265 
 266     /**
 267      * Fetches a reference value from a given Java variable.
 268      * This method can return a reference to either an object or value
 269      * or a null reference.
 270      *
 271      * @see #getInt(Object, long)
 272      */
 273     @IntrinsicCandidate
 274     public native Object getReference(Object o, long offset);
 275 
 276     /**
 277      * Stores a reference value into a given Java variable.
 278      * This method can store a reference to either an object or value
 279      * or a null reference.
 280      * <p>
 281      * Unless the reference {@code x} being stored is either null
 282      * or matches the field type, the results are undefined.
 283      * If the reference {@code o} is non-null, card marks or
 284      * other store barriers for that object (if the VM requires them)
 285      * are updated.
 286      * @see #putInt(Object, long, int)
 287      */
 288     @IntrinsicCandidate
 289     public native void putReference(Object o, long offset, Object x);
 290 
 291     /**
 292      * Fetches a value of type {@code <V>} from a given Java variable.
 293      * More specifically, fetches a field or array element within the given
 294      * {@code o} object at the given offset, or (if {@code o} is null)
 295      * from the memory address whose numerical value is the given offset.
 296      *
 297      * @param o Java heap object in which the variable resides, if any, else
 298      *        null
 299      * @param offset indication of where the variable resides in a Java heap
 300      *        object, if any, else a memory address locating the variable
 301      *        statically
 302      * @param valueType value type
 303      * @param <V> the type of a value
 304      * @return the value fetched from the indicated Java variable
 305      * @throws RuntimeException No defined exceptions are thrown, not even
 306      *         {@link NullPointerException}
 307      */
 308     @IntrinsicCandidate
 309     public native <V> V getValue(Object o, long offset, Class<?> valueType);
 310 
 311     /**
 312      * Fetches a value of type {@code <V>} from a given Java variable.
 313      * More specifically, fetches a field or array element within the given
 314      * {@code o} object at the given offset, or (if {@code o} is null)
 315      * from the memory address whose numerical value is the given offset.
 316      *
 317      * @param o Java heap object in which the variable resides, if any, else
 318      *        null
 319      * @param offset indication of where the variable resides in a Java heap
 320      *        object, if any, else a memory address locating the variable
 321      *        statically
 322      * @param layoutKind opaque value used by the VM to know the layout
 323      *        the field or array element. This value must be retrieved with
 324      *        {@link #fieldLayout} or {@link #arrayLayout}.
 325      * @param valueType value type
 326      * @param <V> the type of a value
 327      * @return the value fetched from the indicated Java variable
 328      * @throws RuntimeException No defined exceptions are thrown, not even
 329      *         {@link NullPointerException}
 330      */
 331     @IntrinsicCandidate
 332     public native <V> V getFlatValue(Object o, long offset, int layoutKind, Class<?> valueType);
 333 
 334 
 335     /**
 336      * Stores the given value into a given Java variable.
 337      *
 338      * Unless the reference {@code o} being stored is either null
 339      * or matches the field type, the results are undefined.
 340      *
 341      * @param o Java heap object in which the variable resides, if any, else
 342      *        null
 343      * @param offset indication of where the variable resides in a Java heap
 344      *        object, if any, else a memory address locating the variable
 345      *        statically
 346      * @param valueType value type
 347      * @param v the value to store into the indicated Java variable
 348      * @param <V> the type of a value
 349      * @throws RuntimeException No defined exceptions are thrown, not even
 350      *         {@link NullPointerException}
 351      */
 352     @IntrinsicCandidate
 353     public native <V> void putValue(Object o, long offset, Class<?> valueType, V v);
 354 
 355     /**
 356      * Stores the given value into a given Java variable.
 357      *
 358      * Unless the reference {@code o} being stored is either null
 359      * or matches the field type, the results are undefined.
 360      *
 361      * @param o Java heap object in which the variable resides, if any, else
 362      *        null
 363      * @param offset indication of where the variable resides in a Java heap
 364      *        object, if any, else a memory address locating the variable
 365      *        statically
 366      * @param layoutKind opaque value used by the VM to know the layout
 367      *        the field or array element. This value must be retrieved with
 368      *        {@link #fieldLayout} or {@link #arrayLayout}.
 369      * @param valueType value type
 370      * @param v the value to store into the indicated Java variable
 371      * @param <V> the type of a value
 372      * @throws RuntimeException No defined exceptions are thrown, not even
 373      *         {@link NullPointerException}
 374      */
 375     @IntrinsicCandidate
 376     public native <V> void putFlatValue(Object o, long offset, int layoutKind, Class<?> valueType, V v);
 377 
 378     /**
 379      * Returns an object instance with a private buffered value whose layout
 380      * and contents is exactly the given value instance.  The return object
 381      * is in the larval state that can be updated using the unsafe put operation.
 382      *
 383      * @param value a value instance
 384      * @param <V> the type of the given value instance
 385      */
 386     @IntrinsicCandidate
 387     public native <V> V makePrivateBuffer(V value);
 388 
 389     /**
 390      * Exits the larval state and returns a value instance.
 391      *
 392      * @param value a value instance
 393      * @param <V> the type of the given value instance
 394      */
 395     @IntrinsicCandidate
 396     public native <V> V finishPrivateBuffer(V value);
 397 
 398     /**
 399      * Returns the header size of the given value type.
 400      *
 401      * @param valueType value type
 402      * @return the header size of the value type
 403      */
 404     public native <V> long valueHeaderSize(Class<V> valueType);
 405 
 406     /** @see #getInt(Object, long) */
 407     @IntrinsicCandidate
 408     public native boolean getBoolean(Object o, long offset);
 409 
 410     /** @see #putInt(Object, long, int) */
 411     @IntrinsicCandidate
 412     public native void    putBoolean(Object o, long offset, boolean x);
 413 
 414     /** @see #getInt(Object, long) */
 415     @IntrinsicCandidate
 416     public native byte    getByte(Object o, long offset);
 417 
 418     /** @see #putInt(Object, long, int) */
 419     @IntrinsicCandidate
 420     public native void    putByte(Object o, long offset, byte x);
 421 
 422     /** @see #getInt(Object, long) */
 423     @IntrinsicCandidate
 424     public native short   getShort(Object o, long offset);
 425 
 426     /** @see #putInt(Object, long, int) */
 427     @IntrinsicCandidate
 428     public native void    putShort(Object o, long offset, short x);
 429 
 430     /** @see #getInt(Object, long) */
 431     @IntrinsicCandidate
 432     public native char    getChar(Object o, long offset);
 433 
 434     /** @see #putInt(Object, long, int) */
 435     @IntrinsicCandidate
 436     public native void    putChar(Object o, long offset, char x);
 437 
 438     /** @see #getInt(Object, long) */
 439     @IntrinsicCandidate
 440     public native long    getLong(Object o, long offset);
 441 
 442     /** @see #putInt(Object, long, int) */
 443     @IntrinsicCandidate
 444     public native void    putLong(Object o, long offset, long x);
 445 
 446     /** @see #getInt(Object, long) */
 447     @IntrinsicCandidate
 448     public native float   getFloat(Object o, long offset);
 449 
 450     /** @see #putInt(Object, long, int) */
 451     @IntrinsicCandidate
 452     public native void    putFloat(Object o, long offset, float x);
 453 
 454     /** @see #getInt(Object, long) */
 455     @IntrinsicCandidate
 456     public native double  getDouble(Object o, long offset);
 457 
 458     /** @see #putInt(Object, long, int) */
 459     @IntrinsicCandidate
 460     public native void    putDouble(Object o, long offset, double x);
 461 
 462     /**
 463      * Fetches a native pointer from a given memory address.  If the address is
 464      * zero, or does not point into a block obtained from {@link
 465      * #allocateMemory}, the results are undefined.
 466      *
 467      * <p>If the native pointer is less than 64 bits wide, it is extended as
 468      * an unsigned number to a Java long.  The pointer may be indexed by any
 469      * given byte offset, simply by adding that offset (as a simple integer) to
 470      * the long representing the pointer.  The number of bytes actually read
 471      * from the target address may be determined by consulting {@link
 472      * #addressSize}.
 473      *
 474      * @see #allocateMemory
 475      * @see #getInt(Object, long)
 476      */
 477     @ForceInline
 478     public long getAddress(Object o, long offset) {
 479         if (ADDRESS_SIZE == 4) {
 480             return Integer.toUnsignedLong(getInt(o, offset));
 481         } else {
 482             return getLong(o, offset);
 483         }
 484     }
 485 
 486     /**
 487      * Stores a native pointer into a given memory address.  If the address is
 488      * zero, or does not point into a block obtained from {@link
 489      * #allocateMemory}, the results are undefined.
 490      *
 491      * <p>The number of bytes actually written at the target address may be
 492      * determined by consulting {@link #addressSize}.
 493      *
 494      * @see #allocateMemory
 495      * @see #putInt(Object, long, int)
 496      */
 497     @ForceInline
 498     public void putAddress(Object o, long offset, long x) {
 499         if (ADDRESS_SIZE == 4) {
 500             putInt(o, offset, (int)x);
 501         } else {
 502             putLong(o, offset, x);
 503         }
 504     }
 505 
 506     // These read VM internal data.
 507 
 508     /**
 509      * Fetches an uncompressed reference value from a given native variable
 510      * ignoring the VM's compressed references mode.
 511      *
 512      * @param address a memory address locating the variable
 513      * @return the value fetched from the indicated native variable
 514      */
 515     public native Object getUncompressedObject(long address);
 516 
 517     // These work on values in the C heap.
 518 
 519     /**
 520      * Fetches a value from a given memory address.  If the address is zero, or
 521      * does not point into a block obtained from {@link #allocateMemory}, the
 522      * results are undefined.
 523      *
 524      * @see #allocateMemory
 525      */
 526     @ForceInline
 527     public byte getByte(long address) {
 528         return getByte(null, address);
 529     }
 530 
 531     /**
 532      * Stores a value into a given memory address.  If the address is zero, or
 533      * does not point into a block obtained from {@link #allocateMemory}, the
 534      * results are undefined.
 535      *
 536      * @see #getByte(long)
 537      */
 538     @ForceInline
 539     public void putByte(long address, byte x) {
 540         putByte(null, address, x);
 541     }
 542 
 543     /** @see #getByte(long) */
 544     @ForceInline
 545     public short getShort(long address) {
 546         return getShort(null, address);
 547     }
 548 
 549     /** @see #putByte(long, byte) */
 550     @ForceInline
 551     public void putShort(long address, short x) {
 552         putShort(null, address, x);
 553     }
 554 
 555     /** @see #getByte(long) */
 556     @ForceInline
 557     public char getChar(long address) {
 558         return getChar(null, address);
 559     }
 560 
 561     /** @see #putByte(long, byte) */
 562     @ForceInline
 563     public void putChar(long address, char x) {
 564         putChar(null, address, x);
 565     }
 566 
 567     /** @see #getByte(long) */
 568     @ForceInline
 569     public int getInt(long address) {
 570         return getInt(null, address);
 571     }
 572 
 573     /** @see #putByte(long, byte) */
 574     @ForceInline
 575     public void putInt(long address, int x) {
 576         putInt(null, address, x);
 577     }
 578 
 579     /** @see #getByte(long) */
 580     @ForceInline
 581     public long getLong(long address) {
 582         return getLong(null, address);
 583     }
 584 
 585     /** @see #putByte(long, byte) */
 586     @ForceInline
 587     public void putLong(long address, long x) {
 588         putLong(null, address, x);
 589     }
 590 
 591     /** @see #getByte(long) */
 592     @ForceInline
 593     public float getFloat(long address) {
 594         return getFloat(null, address);
 595     }
 596 
 597     /** @see #putByte(long, byte) */
 598     @ForceInline
 599     public void putFloat(long address, float x) {
 600         putFloat(null, address, x);
 601     }
 602 
 603     /** @see #getByte(long) */
 604     @ForceInline
 605     public double getDouble(long address) {
 606         return getDouble(null, address);
 607     }
 608 
 609     /** @see #putByte(long, byte) */
 610     @ForceInline
 611     public void putDouble(long address, double x) {
 612         putDouble(null, address, x);
 613     }
 614 
 615     /** @see #getAddress(Object, long) */
 616     @ForceInline
 617     public long getAddress(long address) {
 618         return getAddress(null, address);
 619     }
 620 
 621     /** @see #putAddress(Object, long, long) */
 622     @ForceInline
 623     public void putAddress(long address, long x) {
 624         putAddress(null, address, x);
 625     }
 626 
 627 
 628 
 629     //--- helper methods for validating various types of objects/values
 630 
 631     /**
 632      * Create an exception reflecting that some of the input was invalid
 633      *
 634      * <em>Note:</em> It is the responsibility of the caller to make
 635      * sure arguments are checked before the methods are called. While
 636      * some rudimentary checks are performed on the input, the checks
 637      * are best effort and when performance is an overriding priority,
 638      * as when methods of this class are optimized by the runtime
 639      * compiler, some or all checks (if any) may be elided. Hence, the
 640      * caller must not rely on the checks and corresponding
 641      * exceptions!
 642      *
 643      * @return an exception object
 644      */
 645     private RuntimeException invalidInput() {
 646         return new IllegalArgumentException();
 647     }
 648 
 649     /**
 650      * Check if a value is 32-bit clean (32 MSB are all zero)
 651      *
 652      * @param value the 64-bit value to check
 653      *
 654      * @return true if the value is 32-bit clean
 655      */
 656     private boolean is32BitClean(long value) {
 657         return value >>> 32 == 0;
 658     }
 659 
 660     /**
 661      * Check the validity of a size (the equivalent of a size_t)
 662      *
 663      * @throws RuntimeException if the size is invalid
 664      *         (<em>Note:</em> after optimization, invalid inputs may
 665      *         go undetected, which will lead to unpredictable
 666      *         behavior)
 667      */
 668     private void checkSize(long size) {
 669         if (ADDRESS_SIZE == 4) {
 670             // Note: this will also check for negative sizes
 671             if (!is32BitClean(size)) {
 672                 throw invalidInput();
 673             }
 674         } else if (size < 0) {
 675             throw invalidInput();
 676         }
 677     }
 678 
 679     /**
 680      * Check the validity of a native address (the equivalent of void*)
 681      *
 682      * @throws RuntimeException if the address is invalid
 683      *         (<em>Note:</em> after optimization, invalid inputs may
 684      *         go undetected, which will lead to unpredictable
 685      *         behavior)
 686      */
 687     private void checkNativeAddress(long address) {
 688         if (ADDRESS_SIZE == 4) {
 689             // Accept both zero and sign extended pointers. A valid
 690             // pointer will, after the +1 below, either have produced
 691             // the value 0x0 or 0x1. Masking off the low bit allows
 692             // for testing against 0.
 693             if ((((address >> 32) + 1) & ~1) != 0) {
 694                 throw invalidInput();
 695             }
 696         }
 697     }
 698 
 699     /**
 700      * Check the validity of an offset, relative to a base object
 701      *
 702      * @param o the base object
 703      * @param offset the offset to check
 704      *
 705      * @throws RuntimeException if the size is invalid
 706      *         (<em>Note:</em> after optimization, invalid inputs may
 707      *         go undetected, which will lead to unpredictable
 708      *         behavior)
 709      */
 710     private void checkOffset(Object o, long offset) {
 711         if (ADDRESS_SIZE == 4) {
 712             // Note: this will also check for negative offsets
 713             if (!is32BitClean(offset)) {
 714                 throw invalidInput();
 715             }
 716         } else if (offset < 0) {
 717             throw invalidInput();
 718         }
 719     }
 720 
 721     /**
 722      * Check the validity of a double-register pointer
 723      *
 724      * Note: This code deliberately does *not* check for NPE for (at
 725      * least) three reasons:
 726      *
 727      * 1) NPE is not just NULL/0 - there is a range of values all
 728      * resulting in an NPE, which is not trivial to check for
 729      *
 730      * 2) It is the responsibility of the callers of Unsafe methods
 731      * to verify the input, so throwing an exception here is not really
 732      * useful - passing in a NULL pointer is a critical error and the
 733      * must not expect an exception to be thrown anyway.
 734      *
 735      * 3) the actual operations will detect NULL pointers anyway by
 736      * means of traps and signals (like SIGSEGV).
 737      *
 738      * @param o Java heap object, or null
 739      * @param offset indication of where the variable resides in a Java heap
 740      *        object, if any, else a memory address locating the variable
 741      *        statically
 742      *
 743      * @throws RuntimeException if the pointer is invalid
 744      *         (<em>Note:</em> after optimization, invalid inputs may
 745      *         go undetected, which will lead to unpredictable
 746      *         behavior)
 747      */
 748     private void checkPointer(Object o, long offset) {
 749         if (o == null) {
 750             checkNativeAddress(offset);
 751         } else {
 752             checkOffset(o, offset);
 753         }
 754     }
 755 
 756     /**
 757      * Check if a type is a primitive array type
 758      *
 759      * @param c the type to check
 760      *
 761      * @return true if the type is a primitive array type
 762      */
 763     private void checkPrimitiveArray(Class<?> c) {
 764         Class<?> componentType = c.getComponentType();
 765         if (componentType == null || !componentType.isPrimitive()) {
 766             throw invalidInput();
 767         }
 768     }
 769 
 770     /**
 771      * Check that a pointer is a valid primitive array type pointer
 772      *
 773      * Note: pointers off-heap are considered to be primitive arrays
 774      *
 775      * @throws RuntimeException if the pointer is invalid
 776      *         (<em>Note:</em> after optimization, invalid inputs may
 777      *         go undetected, which will lead to unpredictable
 778      *         behavior)
 779      */
 780     private void checkPrimitivePointer(Object o, long offset) {
 781         checkPointer(o, offset);
 782 
 783         if (o != null) {
 784             // If on heap, it must be a primitive array
 785             checkPrimitiveArray(o.getClass());
 786         }
 787     }
 788 
 789 
 790     //--- wrappers for malloc, realloc, free:
 791 
 792     /**
 793      * Round up allocation size to a multiple of HeapWordSize.
 794      */
 795     private long alignToHeapWordSize(long bytes) {
 796         if (bytes >= 0) {
 797             return (bytes + ADDRESS_SIZE - 1) & ~(ADDRESS_SIZE - 1);
 798         } else {
 799             throw invalidInput();
 800         }
 801     }
 802 
 803     /**
 804      * Allocates a new block of native memory, of the given size in bytes.  The
 805      * contents of the memory are uninitialized; they will generally be
 806      * garbage.  The resulting native pointer will be zero if and only if the
 807      * requested size is zero.  The resulting native pointer will be aligned for
 808      * all value types.   Dispose of this memory by calling {@link #freeMemory}
 809      * or resize it with {@link #reallocateMemory}.
 810      *
 811      * <em>Note:</em> It is the responsibility of the caller to make
 812      * sure arguments are checked before the methods are called. While
 813      * some rudimentary checks are performed on the input, the checks
 814      * are best effort and when performance is an overriding priority,
 815      * as when methods of this class are optimized by the runtime
 816      * compiler, some or all checks (if any) may be elided. Hence, the
 817      * caller must not rely on the checks and corresponding
 818      * exceptions!
 819      *
 820      * @throws RuntimeException if the size is negative or too large
 821      *         for the native size_t type
 822      *
 823      * @throws OutOfMemoryError if the allocation is refused by the system
 824      *
 825      * @see #getByte(long)
 826      * @see #putByte(long, byte)
 827      */
 828     public long allocateMemory(long bytes) {
 829         bytes = alignToHeapWordSize(bytes);
 830 
 831         allocateMemoryChecks(bytes);
 832 
 833         if (bytes == 0) {
 834             return 0;
 835         }
 836 
 837         long p = allocateMemory0(bytes);
 838         if (p == 0) {
 839             throw new OutOfMemoryError("Unable to allocate " + bytes + " bytes");
 840         }
 841 
 842         return p;
 843     }
 844 
 845     /**
 846      * Validate the arguments to allocateMemory
 847      *
 848      * @throws RuntimeException if the arguments are invalid
 849      *         (<em>Note:</em> after optimization, invalid inputs may
 850      *         go undetected, which will lead to unpredictable
 851      *         behavior)
 852      */
 853     private void allocateMemoryChecks(long bytes) {
 854         checkSize(bytes);
 855     }
 856 
 857     /**
 858      * Resizes a new block of native memory, to the given size in bytes.  The
 859      * contents of the new block past the size of the old block are
 860      * uninitialized; they will generally be garbage.  The resulting native
 861      * pointer will be zero if and only if the requested size is zero.  The
 862      * resulting native pointer will be aligned for all value types.  Dispose
 863      * of this memory by calling {@link #freeMemory}, or resize it with {@link
 864      * #reallocateMemory}.  The address passed to this method may be null, in
 865      * which case an allocation will be performed.
 866      *
 867      * <em>Note:</em> It is the responsibility of the caller to make
 868      * sure arguments are checked before the methods are called. While
 869      * some rudimentary checks are performed on the input, the checks
 870      * are best effort and when performance is an overriding priority,
 871      * as when methods of this class are optimized by the runtime
 872      * compiler, some or all checks (if any) may be elided. Hence, the
 873      * caller must not rely on the checks and corresponding
 874      * exceptions!
 875      *
 876      * @throws RuntimeException if the size is negative or too large
 877      *         for the native size_t type
 878      *
 879      * @throws OutOfMemoryError if the allocation is refused by the system
 880      *
 881      * @see #allocateMemory
 882      */
 883     public long reallocateMemory(long address, long bytes) {
 884         bytes = alignToHeapWordSize(bytes);
 885 
 886         reallocateMemoryChecks(address, bytes);
 887 
 888         if (bytes == 0) {
 889             freeMemory(address);
 890             return 0;
 891         }
 892 
 893         long p = (address == 0) ? allocateMemory0(bytes) : reallocateMemory0(address, bytes);
 894         if (p == 0) {
 895             throw new OutOfMemoryError("Unable to allocate " + bytes + " bytes");
 896         }
 897 
 898         return p;
 899     }
 900 
 901     /**
 902      * Validate the arguments to reallocateMemory
 903      *
 904      * @throws RuntimeException if the arguments are invalid
 905      *         (<em>Note:</em> after optimization, invalid inputs may
 906      *         go undetected, which will lead to unpredictable
 907      *         behavior)
 908      */
 909     private void reallocateMemoryChecks(long address, long bytes) {
 910         checkPointer(null, address);
 911         checkSize(bytes);
 912     }
 913 
 914     /**
 915      * Sets all bytes in a given block of memory to a fixed value
 916      * (usually zero).
 917      *
 918      * <p>This method determines a block's base address by means of two parameters,
 919      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 920      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 921      * the offset supplies an absolute base address.
 922      *
 923      * <p>The stores are in coherent (atomic) units of a size determined
 924      * by the address and length parameters.  If the effective address and
 925      * length are all even modulo 8, the stores take place in 'long' units.
 926      * If the effective address and length are (resp.) even modulo 4 or 2,
 927      * the stores take place in units of 'int' or 'short'.
 928      *
 929      * <em>Note:</em> It is the responsibility of the caller to make
 930      * sure arguments are checked before the methods are called. While
 931      * some rudimentary checks are performed on the input, the checks
 932      * are best effort and when performance is an overriding priority,
 933      * as when methods of this class are optimized by the runtime
 934      * compiler, some or all checks (if any) may be elided. Hence, the
 935      * caller must not rely on the checks and corresponding
 936      * exceptions!
 937      *
 938      * @throws RuntimeException if any of the arguments is invalid
 939      *
 940      * @since 1.7
 941      */
 942     public void setMemory(Object o, long offset, long bytes, byte value) {
 943         setMemoryChecks(o, offset, bytes, value);
 944 
 945         if (bytes == 0) {
 946             return;
 947         }
 948 
 949         setMemory0(o, offset, bytes, value);
 950     }
 951 
 952     /**
 953      * Sets all bytes in a given block of memory to a fixed value
 954      * (usually zero).  This provides a <em>single-register</em> addressing mode,
 955      * as discussed in {@link #getInt(Object,long)}.
 956      *
 957      * <p>Equivalent to {@code setMemory(null, address, bytes, value)}.
 958      */
 959     public void setMemory(long address, long bytes, byte value) {
 960         setMemory(null, address, bytes, value);
 961     }
 962 
 963     /**
 964      * Validate the arguments to setMemory
 965      *
 966      * @throws RuntimeException if the arguments are invalid
 967      *         (<em>Note:</em> after optimization, invalid inputs may
 968      *         go undetected, which will lead to unpredictable
 969      *         behavior)
 970      */
 971     private void setMemoryChecks(Object o, long offset, long bytes, byte value) {
 972         checkPrimitivePointer(o, offset);
 973         checkSize(bytes);
 974     }
 975 
 976     /**
 977      * Sets all bytes in a given block of memory to a copy of another
 978      * block.
 979      *
 980      * <p>This method determines each block's base address by means of two parameters,
 981      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 982      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 983      * the offset supplies an absolute base address.
 984      *
 985      * <p>The transfers are in coherent (atomic) units of a size determined
 986      * by the address and length parameters.  If the effective addresses and
 987      * length are all even modulo 8, the transfer takes place in 'long' units.
 988      * If the effective addresses and length are (resp.) even modulo 4 or 2,
 989      * the transfer takes place in units of 'int' or 'short'.
 990      *
 991      * <em>Note:</em> It is the responsibility of the caller to make
 992      * sure arguments are checked before the methods are called. While
 993      * some rudimentary checks are performed on the input, the checks
 994      * are best effort and when performance is an overriding priority,
 995      * as when methods of this class are optimized by the runtime
 996      * compiler, some or all checks (if any) may be elided. Hence, the
 997      * caller must not rely on the checks and corresponding
 998      * exceptions!
 999      *
1000      * @throws RuntimeException if any of the arguments is invalid
1001      *
1002      * @since 1.7
1003      */
1004     public void copyMemory(Object srcBase, long srcOffset,
1005                            Object destBase, long destOffset,
1006                            long bytes) {
1007         copyMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes);
1008 
1009         if (bytes == 0) {
1010             return;
1011         }
1012 
1013         copyMemory0(srcBase, srcOffset, destBase, destOffset, bytes);
1014     }
1015 
1016     /**
1017      * Sets all bytes in a given block of memory to a copy of another
1018      * block.  This provides a <em>single-register</em> addressing mode,
1019      * as discussed in {@link #getInt(Object,long)}.
1020      *
1021      * Equivalent to {@code copyMemory(null, srcAddress, null, destAddress, bytes)}.
1022      */
1023     public void copyMemory(long srcAddress, long destAddress, long bytes) {
1024         copyMemory(null, srcAddress, null, destAddress, bytes);
1025     }
1026 
1027     /**
1028      * Validate the arguments to copyMemory
1029      *
1030      * @throws RuntimeException if any of the arguments is invalid
1031      *         (<em>Note:</em> after optimization, invalid inputs may
1032      *         go undetected, which will lead to unpredictable
1033      *         behavior)
1034      */
1035     private void copyMemoryChecks(Object srcBase, long srcOffset,
1036                                   Object destBase, long destOffset,
1037                                   long bytes) {
1038         checkSize(bytes);
1039         checkPrimitivePointer(srcBase, srcOffset);
1040         checkPrimitivePointer(destBase, destOffset);
1041     }
1042 
1043     /**
1044      * Copies all elements from one block of memory to another block,
1045      * *unconditionally* byte swapping the elements on the fly.
1046      *
1047      * <p>This method determines each block's base address by means of two parameters,
1048      * and so it provides (in effect) a <em>double-register</em> addressing mode,
1049      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
1050      * the offset supplies an absolute base address.
1051      *
1052      * <em>Note:</em> It is the responsibility of the caller to make
1053      * sure arguments are checked before the methods are called. While
1054      * some rudimentary checks are performed on the input, the checks
1055      * are best effort and when performance is an overriding priority,
1056      * as when methods of this class are optimized by the runtime
1057      * compiler, some or all checks (if any) may be elided. Hence, the
1058      * caller must not rely on the checks and corresponding
1059      * exceptions!
1060      *
1061      * @throws RuntimeException if any of the arguments is invalid
1062      *
1063      * @since 9
1064      */
1065     public void copySwapMemory(Object srcBase, long srcOffset,
1066                                Object destBase, long destOffset,
1067                                long bytes, long elemSize) {
1068         copySwapMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
1069 
1070         if (bytes == 0) {
1071             return;
1072         }
1073 
1074         copySwapMemory0(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
1075     }
1076 
1077     private void copySwapMemoryChecks(Object srcBase, long srcOffset,
1078                                       Object destBase, long destOffset,
1079                                       long bytes, long elemSize) {
1080         checkSize(bytes);
1081 
1082         if (elemSize != 2 && elemSize != 4 && elemSize != 8) {
1083             throw invalidInput();
1084         }
1085         if (bytes % elemSize != 0) {
1086             throw invalidInput();
1087         }
1088 
1089         checkPrimitivePointer(srcBase, srcOffset);
1090         checkPrimitivePointer(destBase, destOffset);
1091     }
1092 
1093     /**
1094      * Copies all elements from one block of memory to another block, byte swapping the
1095      * elements on the fly.
1096      *
1097      * This provides a <em>single-register</em> addressing mode, as
1098      * discussed in {@link #getInt(Object,long)}.
1099      *
1100      * Equivalent to {@code copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize)}.
1101      */
1102     public void copySwapMemory(long srcAddress, long destAddress, long bytes, long elemSize) {
1103         copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize);
1104     }
1105 
1106     /**
1107      * Disposes of a block of native memory, as obtained from {@link
1108      * #allocateMemory} or {@link #reallocateMemory}.  The address passed to
1109      * this method may be null, in which case no action is taken.
1110      *
1111      * <em>Note:</em> It is the responsibility of the caller to make
1112      * sure arguments are checked before the methods are called. While
1113      * some rudimentary checks are performed on the input, the checks
1114      * are best effort and when performance is an overriding priority,
1115      * as when methods of this class are optimized by the runtime
1116      * compiler, some or all checks (if any) may be elided. Hence, the
1117      * caller must not rely on the checks and corresponding
1118      * exceptions!
1119      *
1120      * @throws RuntimeException if any of the arguments is invalid
1121      *
1122      * @see #allocateMemory
1123      */
1124     public void freeMemory(long address) {
1125         freeMemoryChecks(address);
1126 
1127         if (address == 0) {
1128             return;
1129         }
1130 
1131         freeMemory0(address);
1132     }
1133 
1134     /**
1135      * Validate the arguments to freeMemory
1136      *
1137      * @throws RuntimeException if the arguments are invalid
1138      *         (<em>Note:</em> after optimization, invalid inputs may
1139      *         go undetected, which will lead to unpredictable
1140      *         behavior)
1141      */
1142     private void freeMemoryChecks(long address) {
1143         checkPointer(null, address);
1144     }
1145 
1146     /**
1147      * Ensure writeback of a specified virtual memory address range
1148      * from cache to physical memory. All bytes in the address range
1149      * are guaranteed to have been written back to physical memory on
1150      * return from this call i.e. subsequently executed store
1151      * instructions are guaranteed not to be visible before the
1152      * writeback is completed.
1153      *
1154      * @param address
1155      *        the lowest byte address that must be guaranteed written
1156      *        back to memory. bytes at lower addresses may also be
1157      *        written back.
1158      *
1159      * @param length
1160      *        the length in bytes of the region starting at address
1161      *        that must be guaranteed written back to memory.
1162      *
1163      * @throws RuntimeException if memory writeback is not supported
1164      *         on the current hardware of if the arguments are invalid.
1165      *         (<em>Note:</em> after optimization, invalid inputs may
1166      *         go undetected, which will lead to unpredictable
1167      *         behavior)
1168      *
1169      * @since 14
1170      */
1171 
1172     public void writebackMemory(long address, long length) {
1173         checkWritebackEnabled();
1174         checkWritebackMemory(address, length);
1175 
1176         // perform any required pre-writeback barrier
1177         writebackPreSync0();
1178 
1179         // write back one cache line at a time
1180         long line = dataCacheLineAlignDown(address);
1181         long end = address + length;
1182         while (line < end) {
1183             writeback0(line);
1184             line += dataCacheLineFlushSize();
1185         }
1186 
1187         // perform any required post-writeback barrier
1188         writebackPostSync0();
1189     }
1190 
1191     /**
1192      * Validate the arguments to writebackMemory
1193      *
1194      * @throws RuntimeException if the arguments are invalid
1195      *         (<em>Note:</em> after optimization, invalid inputs may
1196      *         go undetected, which will lead to unpredictable
1197      *         behavior)
1198      */
1199     private void checkWritebackMemory(long address, long length) {
1200         checkNativeAddress(address);
1201         checkSize(length);
1202     }
1203 
1204     /**
1205      * Validate that the current hardware supports memory writeback.
1206      * (<em>Note:</em> this is a belt and braces check.  Clients are
1207      * expected to test whether writeback is enabled by calling
1208      * ({@link isWritebackEnabled #isWritebackEnabled} and avoid
1209      * calling method {@link writeback #writeback} if it is disabled).
1210      *
1211      *
1212      * @throws RuntimeException if memory writeback is not supported
1213      */
1214     private void checkWritebackEnabled() {
1215         if (!isWritebackEnabled()) {
1216             throw new RuntimeException("writebackMemory not enabled!");
1217         }
1218     }
1219 
1220     /**
1221      * force writeback of an individual cache line.
1222      *
1223      * @param address
1224      *        the start address of the cache line to be written back
1225      */
1226     @IntrinsicCandidate
1227     private native void writeback0(long address);
1228 
1229      /**
1230       * Serialize writeback operations relative to preceding memory writes.
1231       */
1232     @IntrinsicCandidate
1233     private native void writebackPreSync0();
1234 
1235      /**
1236       * Serialize writeback operations relative to following memory writes.
1237       */
1238     @IntrinsicCandidate
1239     private native void writebackPostSync0();
1240 
1241     //--- random queries
1242 
1243     /**
1244      * This constant differs from all results that will ever be returned from
1245      * {@link #staticFieldOffset}, {@link #objectFieldOffset},
1246      * or {@link #arrayBaseOffset}.
1247      * <p>
1248      * The static type is @code long} to emphasize that long arithmetic should
1249      * always be used for offset calculations to avoid overflows.
1250      */
1251     public static final long INVALID_FIELD_OFFSET = -1;
1252 
1253     /**
1254      * Reports the location of a given field in the storage allocation of its
1255      * class.  Do not expect to perform any sort of arithmetic on this offset;
1256      * it is just a cookie which is passed to the unsafe heap memory accessors.
1257      *
1258      * <p>Any given field will always have the same offset and base, and no
1259      * two distinct fields of the same class will ever have the same offset
1260      * and base.
1261      *
1262      * <p>As of 1.4.1, offsets for fields are represented as long values,
1263      * although the Sun JVM does not use the most significant 32 bits.
1264      * However, JVM implementations which store static fields at absolute
1265      * addresses can use long offsets and null base pointers to express
1266      * the field locations in a form usable by {@link #getInt(Object,long)}.
1267      * Therefore, code which will be ported to such JVMs on 64-bit platforms
1268      * must preserve all bits of static field offsets.
1269      * @see #getInt(Object, long)
1270      */
1271     public long objectFieldOffset(Field f) {
1272         if (f == null) {
1273             throw new NullPointerException();
1274         }
1275 
1276         return objectFieldOffset0(f);
1277     }
1278 
1279     /**
1280      * Reports the location of the field with a given name in the storage
1281      * allocation of its class.
1282      *
1283      * @throws NullPointerException if any parameter is {@code null}.
1284      * @throws InternalError if there is no field named {@code name} declared
1285      *         in class {@code c}, i.e., if {@code c.getDeclaredField(name)}
1286      *         would throw {@code java.lang.NoSuchFieldException}.
1287      *
1288      * @see #objectFieldOffset(Field)
1289      */
1290     public long objectFieldOffset(Class<?> c, String name) {
1291         if (c == null || name == null) {
1292             throw new NullPointerException();
1293         }
1294 
1295         return objectFieldOffset1(c, name);
1296     }
1297 
1298     /**
1299      * Reports the location of a given static field, in conjunction with {@link
1300      * #staticFieldBase}.
1301      * <p>Do not expect to perform any sort of arithmetic on this offset;
1302      * it is just a cookie which is passed to the unsafe heap memory accessors.
1303      *
1304      * <p>Any given field will always have the same offset, and no two distinct
1305      * fields of the same class will ever have the same offset.
1306      *
1307      * <p>As of 1.4.1, offsets for fields are represented as long values,
1308      * although the Sun JVM does not use the most significant 32 bits.
1309      * It is hard to imagine a JVM technology which needs more than
1310      * a few bits to encode an offset within a non-array object,
1311      * However, for consistency with other methods in this class,
1312      * this method reports its result as a long value.
1313      * @see #getInt(Object, long)
1314      */
1315     public long staticFieldOffset(Field f) {
1316         if (f == null) {
1317             throw new NullPointerException();
1318         }
1319 
1320         return staticFieldOffset0(f);
1321     }
1322 
1323     /**
1324      * Reports the location of a given static field, in conjunction with {@link
1325      * #staticFieldOffset}.
1326      * <p>Fetch the base "Object", if any, with which static fields of the
1327      * given class can be accessed via methods like {@link #getInt(Object,
1328      * long)}.  This value may be null.  This value may refer to an object
1329      * which is a "cookie", not guaranteed to be a real Object, and it should
1330      * not be used in any way except as argument to the get and put routines in
1331      * this class.
1332      */
1333     public Object staticFieldBase(Field f) {
1334         if (f == null) {
1335             throw new NullPointerException();
1336         }
1337 
1338         return staticFieldBase0(f);
1339     }
1340 
1341     /**
1342      * Detects if the given class may need to be initialized. This is often
1343      * needed in conjunction with obtaining the static field base of a
1344      * class.
1345      * @return false only if a call to {@code ensureClassInitialized} would have no effect
1346      */
1347     public boolean shouldBeInitialized(Class<?> c) {
1348         if (c == null) {
1349             throw new NullPointerException();
1350         }
1351 
1352         return shouldBeInitialized0(c);
1353     }
1354 
1355     /**
1356      * Ensures the given class has been initialized (see JVMS-5.5 for details).
1357      * This is often needed in conjunction with obtaining the static field base
1358      * of a class.
1359      *
1360      * The call returns when either class {@code c} is fully initialized or
1361      * class {@code c} is being initialized and the call is performed from
1362      * the initializing thread. In the latter case a subsequent call to
1363      * {@link #shouldBeInitialized} will return {@code true}.
1364      */
1365     public void ensureClassInitialized(Class<?> c) {
1366         if (c == null) {
1367             throw new NullPointerException();
1368         }
1369 
1370         ensureClassInitialized0(c);
1371     }
1372 
1373     /**
1374      * The reading or writing of strict static fields may require
1375      * special processing.  Notify the VM that such an event is about
1376      * to happen.  The VM may respond by throwing an exception, in the
1377      * case of a read of an uninitialized field.  If the VM allows the
1378      * method to return normally, no further calls are needed, with
1379      * the same arguments.
1380      */
1381     public void notifyStrictStaticAccess(Class<?> c, long staticFieldOffset, boolean writing) {
1382         if (c == null) {
1383             throw new NullPointerException();
1384         }
1385         notifyStrictStaticAccess0(c, staticFieldOffset, writing);
1386     }
1387 
1388     /**
1389      * Reports the offset of the first element in the storage allocation of a
1390      * given array class.  If {@link #arrayIndexScale} returns a non-zero value
1391      * for the same class, you may use that scale factor, together with this
1392      * base offset, to form new offsets to access elements of arrays of the
1393      * given class.
1394      * <p>
1395      * The return value is in the range of a {@code int}.  The return type is
1396      * {@code long} to emphasize that long arithmetic should always be used
1397      * for offset calculations to avoid overflows.
1398      *
1399      * @see #getInt(Object, long)
1400      * @see #putInt(Object, long, int)
1401      */
1402     public long arrayBaseOffset(Class<?> arrayClass) {
1403         if (arrayClass == null) {
1404             throw new NullPointerException();
1405         }
1406 
1407         return arrayBaseOffset0(arrayClass);
1408     }
1409 
1410     public long arrayBaseOffset(Object[] array) {
1411         if (array == null) {
1412             throw new NullPointerException();
1413         }
1414 
1415         return arrayBaseOffset1(array);
1416     }
1417 
1418     /** The value of {@code arrayBaseOffset(boolean[].class)} */
1419     public static final long ARRAY_BOOLEAN_BASE_OFFSET
1420             = theUnsafe.arrayBaseOffset(boolean[].class);
1421 
1422     /** The value of {@code arrayBaseOffset(byte[].class)} */
1423     public static final long ARRAY_BYTE_BASE_OFFSET
1424             = theUnsafe.arrayBaseOffset(byte[].class);
1425 
1426     /** The value of {@code arrayBaseOffset(short[].class)} */
1427     public static final long ARRAY_SHORT_BASE_OFFSET
1428             = theUnsafe.arrayBaseOffset(short[].class);
1429 
1430     /** The value of {@code arrayBaseOffset(char[].class)} */
1431     public static final long ARRAY_CHAR_BASE_OFFSET
1432             = theUnsafe.arrayBaseOffset(char[].class);
1433 
1434     /** The value of {@code arrayBaseOffset(int[].class)} */
1435     public static final long ARRAY_INT_BASE_OFFSET
1436             = theUnsafe.arrayBaseOffset(int[].class);
1437 
1438     /** The value of {@code arrayBaseOffset(long[].class)} */
1439     public static final long ARRAY_LONG_BASE_OFFSET
1440             = theUnsafe.arrayBaseOffset(long[].class);
1441 
1442     /** The value of {@code arrayBaseOffset(float[].class)} */
1443     public static final long ARRAY_FLOAT_BASE_OFFSET
1444             = theUnsafe.arrayBaseOffset(float[].class);
1445 
1446     /** The value of {@code arrayBaseOffset(double[].class)} */
1447     public static final long ARRAY_DOUBLE_BASE_OFFSET
1448             = theUnsafe.arrayBaseOffset(double[].class);
1449 
1450     /** The value of {@code arrayBaseOffset(Object[].class)} */
1451     public static final long ARRAY_OBJECT_BASE_OFFSET
1452             = theUnsafe.arrayBaseOffset(Object[].class);
1453 
1454     /**
1455      * Reports the scale factor for addressing elements in the storage
1456      * allocation of a given array class.  However, arrays of "narrow" types
1457      * will generally not work properly with accessors like {@link
1458      * #getByte(Object, long)}, so the scale factor for such classes is reported
1459      * as zero.
1460      * <p>
1461      * The computation of the actual memory offset should always use {@code
1462      * long} arithmetic to avoid overflows.
1463      *
1464      * @see #arrayBaseOffset
1465      * @see #getInt(Object, long)
1466      * @see #putInt(Object, long, int)
1467      */
1468     public int arrayIndexScale(Class<?> arrayClass) {
1469         if (arrayClass == null) {
1470             throw new NullPointerException();
1471         }
1472 
1473         return arrayIndexScale0(arrayClass);
1474     }
1475 
1476     public int arrayIndexScale(Object[] array) {
1477         if (array == null) {
1478             throw new NullPointerException();
1479         }
1480 
1481         return arrayIndexScale1(array);
1482     }
1483 
1484     /**
1485      * Return the size of the object in the heap.
1486      * @param o an object
1487      * @return the objects's size
1488      * @since Valhalla
1489      */
1490     public long getObjectSize(Object o) {
1491         if (o == null)
1492             throw new NullPointerException();
1493         return getObjectSize0(o);
1494     }
1495 
1496     /** The value of {@code arrayIndexScale(boolean[].class)} */
1497     public static final int ARRAY_BOOLEAN_INDEX_SCALE
1498             = theUnsafe.arrayIndexScale(boolean[].class);
1499 
1500     /** The value of {@code arrayIndexScale(byte[].class)} */
1501     public static final int ARRAY_BYTE_INDEX_SCALE
1502             = theUnsafe.arrayIndexScale(byte[].class);
1503 
1504     /** The value of {@code arrayIndexScale(short[].class)} */
1505     public static final int ARRAY_SHORT_INDEX_SCALE
1506             = theUnsafe.arrayIndexScale(short[].class);
1507 
1508     /** The value of {@code arrayIndexScale(char[].class)} */
1509     public static final int ARRAY_CHAR_INDEX_SCALE
1510             = theUnsafe.arrayIndexScale(char[].class);
1511 
1512     /** The value of {@code arrayIndexScale(int[].class)} */
1513     public static final int ARRAY_INT_INDEX_SCALE
1514             = theUnsafe.arrayIndexScale(int[].class);
1515 
1516     /** The value of {@code arrayIndexScale(long[].class)} */
1517     public static final int ARRAY_LONG_INDEX_SCALE
1518             = theUnsafe.arrayIndexScale(long[].class);
1519 
1520     /** The value of {@code arrayIndexScale(float[].class)} */
1521     public static final int ARRAY_FLOAT_INDEX_SCALE
1522             = theUnsafe.arrayIndexScale(float[].class);
1523 
1524     /** The value of {@code arrayIndexScale(double[].class)} */
1525     public static final int ARRAY_DOUBLE_INDEX_SCALE
1526             = theUnsafe.arrayIndexScale(double[].class);
1527 
1528     /** The value of {@code arrayIndexScale(Object[].class)} */
1529     public static final int ARRAY_OBJECT_INDEX_SCALE
1530             = theUnsafe.arrayIndexScale(Object[].class);
1531 
1532     /**
1533      * Reports the size in bytes of a native pointer, as stored via {@link
1534      * #putAddress}.  This value will be either 4 or 8.  Note that the sizes of
1535      * other primitive types (as stored in native memory blocks) is determined
1536      * fully by their information content.
1537      */
1538     public int addressSize() {
1539         return ADDRESS_SIZE;
1540     }
1541 
1542     /** The value of {@code addressSize()} */
1543     public static final int ADDRESS_SIZE = ADDRESS_SIZE0;
1544 
1545     /**
1546      * Reports the size in bytes of a native memory page (whatever that is).
1547      * This value will always be a power of two.
1548      */
1549     public int pageSize() { return PAGE_SIZE; }
1550 
1551     /**
1552      * Reports the size in bytes of a data cache line written back by
1553      * the hardware cache line flush operation available to the JVM or
1554      * 0 if data cache line flushing is not enabled.
1555      */
1556     public int dataCacheLineFlushSize() { return DATA_CACHE_LINE_FLUSH_SIZE; }
1557 
1558     /**
1559      * Rounds down address to a data cache line boundary as
1560      * determined by {@link #dataCacheLineFlushSize}
1561      * @return the rounded down address
1562      */
1563     public long dataCacheLineAlignDown(long address) {
1564         return (address & ~(DATA_CACHE_LINE_FLUSH_SIZE - 1));
1565     }
1566 
1567     /**
1568      * Returns true if data cache line writeback
1569      */
1570     public static boolean isWritebackEnabled() { return DATA_CACHE_LINE_FLUSH_SIZE != 0; }
1571 
1572     //--- random trusted operations from JNI:
1573 
1574     /**
1575      * Tells the VM to define a class, without security checks.  By default, the
1576      * class loader and protection domain come from the caller's class.
1577      */
1578     public Class<?> defineClass(String name, byte[] b, int off, int len,
1579                                 ClassLoader loader,
1580                                 ProtectionDomain protectionDomain) {
1581         if (b == null) {
1582             throw new NullPointerException();
1583         }
1584         if (len < 0) {
1585             throw new ArrayIndexOutOfBoundsException();
1586         }
1587 
1588         return defineClass0(name, b, off, len, loader, protectionDomain);
1589     }
1590 
1591     public native Class<?> defineClass0(String name, byte[] b, int off, int len,
1592                                         ClassLoader loader,
1593                                         ProtectionDomain protectionDomain);
1594 
1595     /**
1596      * Allocates an instance but does not run any constructor.
1597      * Initializes the class if it has not yet been.
1598      */
1599     @IntrinsicCandidate
1600     public native Object allocateInstance(Class<?> cls)
1601         throws InstantiationException;
1602 
1603     /**
1604      * Allocates an array of a given type, but does not do zeroing.
1605      * <p>
1606      * This method should only be used in the very rare cases where a high-performance code
1607      * overwrites the destination array completely, and compilers cannot assist in zeroing elimination.
1608      * In an overwhelming majority of cases, a normal Java allocation should be used instead.
1609      * <p>
1610      * Users of this method are <b>required</b> to overwrite the initial (garbage) array contents
1611      * before allowing untrusted code, or code in other threads, to observe the reference
1612      * to the newly allocated array. In addition, the publication of the array reference must be
1613      * safe according to the Java Memory Model requirements.
1614      * <p>
1615      * The safest approach to deal with an uninitialized array is to keep the reference to it in local
1616      * variable at least until the initialization is complete, and then publish it <b>once</b>, either
1617      * by writing it to a <em>volatile</em> field, or storing it into a <em>final</em> field in constructor,
1618      * or issuing a {@link #storeFence} before publishing the reference.
1619      * <p>
1620      * @implnote This method can only allocate primitive arrays, to avoid garbage reference
1621      * elements that could break heap integrity.
1622      *
1623      * @param componentType array component type to allocate
1624      * @param length array size to allocate
1625      * @throws IllegalArgumentException if component type is null, or not a primitive class;
1626      *                                  or the length is negative
1627      */
1628     public Object allocateUninitializedArray(Class<?> componentType, int length) {
1629        if (componentType == null) {
1630            throw new IllegalArgumentException("Component type is null");
1631        }
1632        if (!componentType.isPrimitive()) {
1633            throw new IllegalArgumentException("Component type is not primitive");
1634        }
1635        if (length < 0) {
1636            throw new IllegalArgumentException("Negative length");
1637        }
1638        return allocateUninitializedArray0(componentType, length);
1639     }
1640 
1641     @IntrinsicCandidate
1642     private Object allocateUninitializedArray0(Class<?> componentType, int length) {
1643        // These fallbacks provide zeroed arrays, but intrinsic is not required to
1644        // return the zeroed arrays.
1645        if (componentType == byte.class)    return new byte[length];
1646        if (componentType == boolean.class) return new boolean[length];
1647        if (componentType == short.class)   return new short[length];
1648        if (componentType == char.class)    return new char[length];
1649        if (componentType == int.class)     return new int[length];
1650        if (componentType == float.class)   return new float[length];
1651        if (componentType == long.class)    return new long[length];
1652        if (componentType == double.class)  return new double[length];
1653        return null;
1654     }
1655 
1656     /** Throws the exception without telling the verifier. */
1657     public native void throwException(Throwable ee);
1658 
1659     /**
1660      * Atomically updates Java variable to {@code x} if it is currently
1661      * holding {@code expected}.
1662      *
1663      * <p>This operation has memory semantics of a {@code volatile} read
1664      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1665      *
1666      * @return {@code true} if successful
1667      */
1668     @IntrinsicCandidate
1669     public final native boolean compareAndSetReference(Object o, long offset,
1670                                                        Object expected,
1671                                                        Object x);
1672 
1673     private final boolean isValueObject(Object o) {
1674         return o != null && o.getClass().isValue();
1675     }
1676 
1677     /*
1678      * For value type, CAS should do substitutability test as opposed
1679      * to two pointers comparison.
1680      */
1681     @ForceInline
1682     public final <V> boolean compareAndSetReference(Object o, long offset,
1683                                                     Class<?> type,
1684                                                     V expected,
1685                                                     V x) {
1686         if (type.isValue() || isValueObject(expected)) {
1687             while (true) {
1688                 Object witness = getReferenceVolatile(o, offset);
1689                 if (witness != expected) {
1690                     return false;
1691                 }
1692                 if (compareAndSetReference(o, offset, witness, x)) {
1693                     return true;
1694                 }
1695             }
1696         } else {
1697             return compareAndSetReference(o, offset, expected, x);
1698         }
1699     }
1700 
1701     @ForceInline
1702     public final <V> boolean compareAndSetFlatValue(Object o, long offset,
1703                                                 int layout,
1704                                                 Class<?> valueType,
1705                                                 V expected,
1706                                                 V x) {
1707         while (true) {
1708             Object witness = getFlatValueVolatile(o, offset, layout, valueType);
1709             if (witness != expected) {
1710                 return false;
1711             }
1712             if (compareAndSetFlatValueAsBytes(o, offset, layout, valueType, witness, x)) {
1713                 return true;
1714             }
1715         }
1716     }
1717 
1718     @IntrinsicCandidate
1719     public final native Object compareAndExchangeReference(Object o, long offset,
1720                                                            Object expected,
1721                                                            Object x);
1722 
1723     @ForceInline
1724     public final <V> Object compareAndExchangeReference(Object o, long offset,
1725                                                         Class<?> valueType,
1726                                                         V expected,
1727                                                         V x) {
1728         if (valueType.isValue() || isValueObject(expected)) {
1729             while (true) {
1730                 Object witness = getReferenceVolatile(o, offset);
1731                 if (witness != expected) {
1732                     return witness;
1733                 }
1734                 if (compareAndSetReference(o, offset, witness, x)) {
1735                     return witness;
1736                 }
1737             }
1738         } else {
1739             return compareAndExchangeReference(o, offset, expected, x);
1740         }
1741     }
1742 
1743     @ForceInline
1744     public final <V> Object compareAndExchangeFlatValue(Object o, long offset,
1745                                                     int layout,
1746                                                     Class<?> valueType,
1747                                                     V expected,
1748                                                     V x) {
1749         while (true) {
1750             Object witness = getFlatValueVolatile(o, offset, layout, valueType);
1751             if (witness != expected) {
1752                 return witness;
1753             }
1754             if (compareAndSetFlatValueAsBytes(o, offset, layout, valueType, witness, x)) {
1755                 return witness;
1756             }
1757         }
1758     }
1759 
1760     @IntrinsicCandidate
1761     public final Object compareAndExchangeReferenceAcquire(Object o, long offset,
1762                                                            Object expected,
1763                                                            Object x) {
1764         return compareAndExchangeReference(o, offset, expected, x);
1765     }
1766 
1767     public final <V> Object compareAndExchangeReferenceAcquire(Object o, long offset,
1768                                                                Class<?> valueType,
1769                                                                V expected,
1770                                                                V x) {
1771         return compareAndExchangeReference(o, offset, valueType, expected, x);
1772     }
1773 
1774     @ForceInline
1775     public final <V> Object compareAndExchangeFlatValueAcquire(Object o, long offset,
1776                                                            int layout,
1777                                                            Class<?> valueType,
1778                                                            V expected,
1779                                                            V x) {
1780         return compareAndExchangeFlatValue(o, offset, layout, valueType, expected, x);
1781     }
1782 
1783     @IntrinsicCandidate
1784     public final Object compareAndExchangeReferenceRelease(Object o, long offset,
1785                                                            Object expected,
1786                                                            Object x) {
1787         return compareAndExchangeReference(o, offset, expected, x);
1788     }
1789 
1790     public final <V> Object compareAndExchangeReferenceRelease(Object o, long offset,
1791                                                                Class<?> valueType,
1792                                                                V expected,
1793                                                                V x) {
1794         return compareAndExchangeReference(o, offset, valueType, expected, x);
1795     }
1796 
1797     @ForceInline
1798     public final <V> Object compareAndExchangeFlatValueRelease(Object o, long offset,
1799                                                            int layout,
1800                                                            Class<?> valueType,
1801                                                            V expected,
1802                                                            V x) {
1803         return compareAndExchangeFlatValue(o, offset, layout, valueType, expected, x);
1804     }
1805 
1806     @IntrinsicCandidate
1807     public final boolean weakCompareAndSetReferencePlain(Object o, long offset,
1808                                                          Object expected,
1809                                                          Object x) {
1810         return compareAndSetReference(o, offset, expected, x);
1811     }
1812 
1813     public final <V> boolean weakCompareAndSetReferencePlain(Object o, long offset,
1814                                                              Class<?> valueType,
1815                                                              V expected,
1816                                                              V x) {
1817         if (valueType.isValue() || isValueObject(expected)) {
1818             return compareAndSetReference(o, offset, valueType, expected, x);
1819         } else {
1820             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1821         }
1822     }
1823 
1824     @ForceInline
1825     public final <V> boolean weakCompareAndSetFlatValuePlain(Object o, long offset,
1826                                                          int layout,
1827                                                          Class<?> valueType,
1828                                                          V expected,
1829                                                          V x) {
1830         return compareAndSetFlatValue(o, offset, layout, valueType, expected, x);
1831     }
1832 
1833     @IntrinsicCandidate
1834     public final boolean weakCompareAndSetReferenceAcquire(Object o, long offset,
1835                                                            Object expected,
1836                                                            Object x) {
1837         return compareAndSetReference(o, offset, expected, x);
1838     }
1839 
1840     public final <V> boolean weakCompareAndSetReferenceAcquire(Object o, long offset,
1841                                                                Class<?> valueType,
1842                                                                V expected,
1843                                                                V x) {
1844         if (valueType.isValue() || isValueObject(expected)) {
1845             return compareAndSetReference(o, offset, valueType, expected, x);
1846         } else {
1847             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1848         }
1849     }
1850 
1851     @ForceInline
1852     public final <V> boolean weakCompareAndSetFlatValueAcquire(Object o, long offset,
1853                                                            int layout,
1854                                                            Class<?> valueType,
1855                                                            V expected,
1856                                                            V x) {
1857         return compareAndSetFlatValue(o, offset, layout, valueType, expected, x);
1858     }
1859 
1860     @IntrinsicCandidate
1861     public final boolean weakCompareAndSetReferenceRelease(Object o, long offset,
1862                                                            Object expected,
1863                                                            Object x) {
1864         return compareAndSetReference(o, offset, expected, x);
1865     }
1866 
1867     public final <V> boolean weakCompareAndSetReferenceRelease(Object o, long offset,
1868                                                                Class<?> valueType,
1869                                                                V expected,
1870                                                                V x) {
1871         if (valueType.isValue() || isValueObject(expected)) {
1872             return compareAndSetReference(o, offset, valueType, expected, x);
1873         } else {
1874             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1875         }
1876     }
1877 
1878     @ForceInline
1879     public final <V> boolean weakCompareAndSetFlatValueRelease(Object o, long offset,
1880                                                            int layout,
1881                                                            Class<?> valueType,
1882                                                            V expected,
1883                                                            V x) {
1884         return compareAndSetFlatValue(o, offset, layout, valueType, expected, x);
1885     }
1886 
1887     @IntrinsicCandidate
1888     public final boolean weakCompareAndSetReference(Object o, long offset,
1889                                                     Object expected,
1890                                                     Object x) {
1891         return compareAndSetReference(o, offset, expected, x);
1892     }
1893 
1894     public final <V> boolean weakCompareAndSetReference(Object o, long offset,
1895                                                         Class<?> valueType,
1896                                                         V expected,
1897                                                         V x) {
1898         if (valueType.isValue() || isValueObject(expected)) {
1899             return compareAndSetReference(o, offset, valueType, expected, x);
1900         } else {
1901             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1902         }
1903     }
1904 
1905     @ForceInline
1906     public final <V> boolean weakCompareAndSetFlatValue(Object o, long offset,
1907                                                     int layout,
1908                                                     Class<?> valueType,
1909                                                     V expected,
1910                                                     V x) {
1911         return compareAndSetFlatValue(o, offset, layout, valueType, expected, x);
1912     }
1913 
1914     /**
1915      * Atomically updates Java variable to {@code x} if it is currently
1916      * holding {@code expected}.
1917      *
1918      * <p>This operation has memory semantics of a {@code volatile} read
1919      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1920      *
1921      * @return {@code true} if successful
1922      */
1923     @IntrinsicCandidate
1924     public final native boolean compareAndSetInt(Object o, long offset,
1925                                                  int expected,
1926                                                  int x);
1927 
1928     @IntrinsicCandidate
1929     public final native int compareAndExchangeInt(Object o, long offset,
1930                                                   int expected,
1931                                                   int x);
1932 
1933     @IntrinsicCandidate
1934     public final int compareAndExchangeIntAcquire(Object o, long offset,
1935                                                          int expected,
1936                                                          int x) {
1937         return compareAndExchangeInt(o, offset, expected, x);
1938     }
1939 
1940     @IntrinsicCandidate
1941     public final int compareAndExchangeIntRelease(Object o, long offset,
1942                                                          int expected,
1943                                                          int x) {
1944         return compareAndExchangeInt(o, offset, expected, x);
1945     }
1946 
1947     @IntrinsicCandidate
1948     public final boolean weakCompareAndSetIntPlain(Object o, long offset,
1949                                                    int expected,
1950                                                    int x) {
1951         return compareAndSetInt(o, offset, expected, x);
1952     }
1953 
1954     @IntrinsicCandidate
1955     public final boolean weakCompareAndSetIntAcquire(Object o, long offset,
1956                                                      int expected,
1957                                                      int x) {
1958         return compareAndSetInt(o, offset, expected, x);
1959     }
1960 
1961     @IntrinsicCandidate
1962     public final boolean weakCompareAndSetIntRelease(Object o, long offset,
1963                                                      int expected,
1964                                                      int x) {
1965         return compareAndSetInt(o, offset, expected, x);
1966     }
1967 
1968     @IntrinsicCandidate
1969     public final boolean weakCompareAndSetInt(Object o, long offset,
1970                                               int expected,
1971                                               int x) {
1972         return compareAndSetInt(o, offset, expected, x);
1973     }
1974 
1975     @IntrinsicCandidate
1976     public final byte compareAndExchangeByte(Object o, long offset,
1977                                              byte expected,
1978                                              byte x) {
1979         long wordOffset = offset & ~3;
1980         int shift = (int) (offset & 3) << 3;
1981         if (BIG_ENDIAN) {
1982             shift = 24 - shift;
1983         }
1984         int mask           = 0xFF << shift;
1985         int maskedExpected = (expected & 0xFF) << shift;
1986         int maskedX        = (x & 0xFF) << shift;
1987         int fullWord;
1988         do {
1989             fullWord = getIntVolatile(o, wordOffset);
1990             if ((fullWord & mask) != maskedExpected)
1991                 return (byte) ((fullWord & mask) >> shift);
1992         } while (!weakCompareAndSetInt(o, wordOffset,
1993                                                 fullWord, (fullWord & ~mask) | maskedX));
1994         return expected;
1995     }
1996 
1997     @IntrinsicCandidate
1998     public final boolean compareAndSetByte(Object o, long offset,
1999                                            byte expected,
2000                                            byte x) {
2001         return compareAndExchangeByte(o, offset, expected, x) == expected;
2002     }
2003 
2004     @IntrinsicCandidate
2005     public final boolean weakCompareAndSetByte(Object o, long offset,
2006                                                byte expected,
2007                                                byte x) {
2008         return compareAndSetByte(o, offset, expected, x);
2009     }
2010 
2011     @IntrinsicCandidate
2012     public final boolean weakCompareAndSetByteAcquire(Object o, long offset,
2013                                                       byte expected,
2014                                                       byte x) {
2015         return weakCompareAndSetByte(o, offset, expected, x);
2016     }
2017 
2018     @IntrinsicCandidate
2019     public final boolean weakCompareAndSetByteRelease(Object o, long offset,
2020                                                       byte expected,
2021                                                       byte x) {
2022         return weakCompareAndSetByte(o, offset, expected, x);
2023     }
2024 
2025     @IntrinsicCandidate
2026     public final boolean weakCompareAndSetBytePlain(Object o, long offset,
2027                                                     byte expected,
2028                                                     byte x) {
2029         return weakCompareAndSetByte(o, offset, expected, x);
2030     }
2031 
2032     @IntrinsicCandidate
2033     public final byte compareAndExchangeByteAcquire(Object o, long offset,
2034                                                     byte expected,
2035                                                     byte x) {
2036         return compareAndExchangeByte(o, offset, expected, x);
2037     }
2038 
2039     @IntrinsicCandidate
2040     public final byte compareAndExchangeByteRelease(Object o, long offset,
2041                                                     byte expected,
2042                                                     byte x) {
2043         return compareAndExchangeByte(o, offset, expected, x);
2044     }
2045 
2046     @IntrinsicCandidate
2047     public final short compareAndExchangeShort(Object o, long offset,
2048                                                short expected,
2049                                                short x) {
2050         if ((offset & 3) == 3) {
2051             throw new IllegalArgumentException("Update spans the word, not supported");
2052         }
2053         long wordOffset = offset & ~3;
2054         int shift = (int) (offset & 3) << 3;
2055         if (BIG_ENDIAN) {
2056             shift = 16 - shift;
2057         }
2058         int mask           = 0xFFFF << shift;
2059         int maskedExpected = (expected & 0xFFFF) << shift;
2060         int maskedX        = (x & 0xFFFF) << shift;
2061         int fullWord;
2062         do {
2063             fullWord = getIntVolatile(o, wordOffset);
2064             if ((fullWord & mask) != maskedExpected) {
2065                 return (short) ((fullWord & mask) >> shift);
2066             }
2067         } while (!weakCompareAndSetInt(o, wordOffset,
2068                                                 fullWord, (fullWord & ~mask) | maskedX));
2069         return expected;
2070     }
2071 
2072     @IntrinsicCandidate
2073     public final boolean compareAndSetShort(Object o, long offset,
2074                                             short expected,
2075                                             short x) {
2076         return compareAndExchangeShort(o, offset, expected, x) == expected;
2077     }
2078 
2079     @IntrinsicCandidate
2080     public final boolean weakCompareAndSetShort(Object o, long offset,
2081                                                 short expected,
2082                                                 short x) {
2083         return compareAndSetShort(o, offset, expected, x);
2084     }
2085 
2086     @IntrinsicCandidate
2087     public final boolean weakCompareAndSetShortAcquire(Object o, long offset,
2088                                                        short expected,
2089                                                        short x) {
2090         return weakCompareAndSetShort(o, offset, expected, x);
2091     }
2092 
2093     @IntrinsicCandidate
2094     public final boolean weakCompareAndSetShortRelease(Object o, long offset,
2095                                                        short expected,
2096                                                        short x) {
2097         return weakCompareAndSetShort(o, offset, expected, x);
2098     }
2099 
2100     @IntrinsicCandidate
2101     public final boolean weakCompareAndSetShortPlain(Object o, long offset,
2102                                                      short expected,
2103                                                      short x) {
2104         return weakCompareAndSetShort(o, offset, expected, x);
2105     }
2106 
2107 
2108     @IntrinsicCandidate
2109     public final short compareAndExchangeShortAcquire(Object o, long offset,
2110                                                      short expected,
2111                                                      short x) {
2112         return compareAndExchangeShort(o, offset, expected, x);
2113     }
2114 
2115     @IntrinsicCandidate
2116     public final short compareAndExchangeShortRelease(Object o, long offset,
2117                                                     short expected,
2118                                                     short x) {
2119         return compareAndExchangeShort(o, offset, expected, x);
2120     }
2121 
2122     @ForceInline
2123     private char s2c(short s) {
2124         return (char) s;
2125     }
2126 
2127     @ForceInline
2128     private short c2s(char s) {
2129         return (short) s;
2130     }
2131 
2132     @ForceInline
2133     public final boolean compareAndSetChar(Object o, long offset,
2134                                            char expected,
2135                                            char x) {
2136         return compareAndSetShort(o, offset, c2s(expected), c2s(x));
2137     }
2138 
2139     @ForceInline
2140     public final char compareAndExchangeChar(Object o, long offset,
2141                                              char expected,
2142                                              char x) {
2143         return s2c(compareAndExchangeShort(o, offset, c2s(expected), c2s(x)));
2144     }
2145 
2146     @ForceInline
2147     public final char compareAndExchangeCharAcquire(Object o, long offset,
2148                                             char expected,
2149                                             char x) {
2150         return s2c(compareAndExchangeShortAcquire(o, offset, c2s(expected), c2s(x)));
2151     }
2152 
2153     @ForceInline
2154     public final char compareAndExchangeCharRelease(Object o, long offset,
2155                                             char expected,
2156                                             char x) {
2157         return s2c(compareAndExchangeShortRelease(o, offset, c2s(expected), c2s(x)));
2158     }
2159 
2160     @ForceInline
2161     public final boolean weakCompareAndSetChar(Object o, long offset,
2162                                                char expected,
2163                                                char x) {
2164         return weakCompareAndSetShort(o, offset, c2s(expected), c2s(x));
2165     }
2166 
2167     @ForceInline
2168     public final boolean weakCompareAndSetCharAcquire(Object o, long offset,
2169                                                       char expected,
2170                                                       char x) {
2171         return weakCompareAndSetShortAcquire(o, offset, c2s(expected), c2s(x));
2172     }
2173 
2174     @ForceInline
2175     public final boolean weakCompareAndSetCharRelease(Object o, long offset,
2176                                                       char expected,
2177                                                       char x) {
2178         return weakCompareAndSetShortRelease(o, offset, c2s(expected), c2s(x));
2179     }
2180 
2181     @ForceInline
2182     public final boolean weakCompareAndSetCharPlain(Object o, long offset,
2183                                                     char expected,
2184                                                     char x) {
2185         return weakCompareAndSetShortPlain(o, offset, c2s(expected), c2s(x));
2186     }
2187 
2188     /**
2189      * The JVM converts integral values to boolean values using two
2190      * different conventions, byte testing against zero and truncation
2191      * to least-significant bit.
2192      *
2193      * <p>The JNI documents specify that, at least for returning
2194      * values from native methods, a Java boolean value is converted
2195      * to the value-set 0..1 by first truncating to a byte (0..255 or
2196      * maybe -128..127) and then testing against zero. Thus, Java
2197      * booleans in non-Java data structures are by convention
2198      * represented as 8-bit containers containing either zero (for
2199      * false) or any non-zero value (for true).
2200      *
2201      * <p>Java booleans in the heap are also stored in bytes, but are
2202      * strongly normalized to the value-set 0..1 (i.e., they are
2203      * truncated to the least-significant bit).
2204      *
2205      * <p>The main reason for having different conventions for
2206      * conversion is performance: Truncation to the least-significant
2207      * bit can be usually implemented with fewer (machine)
2208      * instructions than byte testing against zero.
2209      *
2210      * <p>A number of Unsafe methods load boolean values from the heap
2211      * as bytes. Unsafe converts those values according to the JNI
2212      * rules (i.e, using the "testing against zero" convention). The
2213      * method {@code byte2bool} implements that conversion.
2214      *
2215      * @param b the byte to be converted to boolean
2216      * @return the result of the conversion
2217      */
2218     @ForceInline
2219     private boolean byte2bool(byte b) {
2220         return b != 0;
2221     }
2222 
2223     /**
2224      * Convert a boolean value to a byte. The return value is strongly
2225      * normalized to the value-set 0..1 (i.e., the value is truncated
2226      * to the least-significant bit). See {@link #byte2bool(byte)} for
2227      * more details on conversion conventions.
2228      *
2229      * @param b the boolean to be converted to byte (and then normalized)
2230      * @return the result of the conversion
2231      */
2232     @ForceInline
2233     private byte bool2byte(boolean b) {
2234         return b ? (byte)1 : (byte)0;
2235     }
2236 
2237     @ForceInline
2238     public final boolean compareAndSetBoolean(Object o, long offset,
2239                                               boolean expected,
2240                                               boolean x) {
2241         return compareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
2242     }
2243 
2244     @ForceInline
2245     public final boolean compareAndExchangeBoolean(Object o, long offset,
2246                                                    boolean expected,
2247                                                    boolean x) {
2248         return byte2bool(compareAndExchangeByte(o, offset, bool2byte(expected), bool2byte(x)));
2249     }
2250 
2251     @ForceInline
2252     public final boolean compareAndExchangeBooleanAcquire(Object o, long offset,
2253                                                     boolean expected,
2254                                                     boolean x) {
2255         return byte2bool(compareAndExchangeByteAcquire(o, offset, bool2byte(expected), bool2byte(x)));
2256     }
2257 
2258     @ForceInline
2259     public final boolean compareAndExchangeBooleanRelease(Object o, long offset,
2260                                                        boolean expected,
2261                                                        boolean x) {
2262         return byte2bool(compareAndExchangeByteRelease(o, offset, bool2byte(expected), bool2byte(x)));
2263     }
2264 
2265     @ForceInline
2266     public final boolean weakCompareAndSetBoolean(Object o, long offset,
2267                                                   boolean expected,
2268                                                   boolean x) {
2269         return weakCompareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
2270     }
2271 
2272     @ForceInline
2273     public final boolean weakCompareAndSetBooleanAcquire(Object o, long offset,
2274                                                          boolean expected,
2275                                                          boolean x) {
2276         return weakCompareAndSetByteAcquire(o, offset, bool2byte(expected), bool2byte(x));
2277     }
2278 
2279     @ForceInline
2280     public final boolean weakCompareAndSetBooleanRelease(Object o, long offset,
2281                                                          boolean expected,
2282                                                          boolean x) {
2283         return weakCompareAndSetByteRelease(o, offset, bool2byte(expected), bool2byte(x));
2284     }
2285 
2286     @ForceInline
2287     public final boolean weakCompareAndSetBooleanPlain(Object o, long offset,
2288                                                        boolean expected,
2289                                                        boolean x) {
2290         return weakCompareAndSetBytePlain(o, offset, bool2byte(expected), bool2byte(x));
2291     }
2292 
2293     /**
2294      * Atomically updates Java variable to {@code x} if it is currently
2295      * holding {@code expected}.
2296      *
2297      * <p>This operation has memory semantics of a {@code volatile} read
2298      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2299      *
2300      * @return {@code true} if successful
2301      */
2302     @ForceInline
2303     public final boolean compareAndSetFloat(Object o, long offset,
2304                                             float expected,
2305                                             float x) {
2306         return compareAndSetInt(o, offset,
2307                                  Float.floatToRawIntBits(expected),
2308                                  Float.floatToRawIntBits(x));
2309     }
2310 
2311     @ForceInline
2312     public final float compareAndExchangeFloat(Object o, long offset,
2313                                                float expected,
2314                                                float x) {
2315         int w = compareAndExchangeInt(o, offset,
2316                                       Float.floatToRawIntBits(expected),
2317                                       Float.floatToRawIntBits(x));
2318         return Float.intBitsToFloat(w);
2319     }
2320 
2321     @ForceInline
2322     public final float compareAndExchangeFloatAcquire(Object o, long offset,
2323                                                   float expected,
2324                                                   float x) {
2325         int w = compareAndExchangeIntAcquire(o, offset,
2326                                              Float.floatToRawIntBits(expected),
2327                                              Float.floatToRawIntBits(x));
2328         return Float.intBitsToFloat(w);
2329     }
2330 
2331     @ForceInline
2332     public final float compareAndExchangeFloatRelease(Object o, long offset,
2333                                                   float expected,
2334                                                   float x) {
2335         int w = compareAndExchangeIntRelease(o, offset,
2336                                              Float.floatToRawIntBits(expected),
2337                                              Float.floatToRawIntBits(x));
2338         return Float.intBitsToFloat(w);
2339     }
2340 
2341     @ForceInline
2342     public final boolean weakCompareAndSetFloatPlain(Object o, long offset,
2343                                                      float expected,
2344                                                      float x) {
2345         return weakCompareAndSetIntPlain(o, offset,
2346                                      Float.floatToRawIntBits(expected),
2347                                      Float.floatToRawIntBits(x));
2348     }
2349 
2350     @ForceInline
2351     public final boolean weakCompareAndSetFloatAcquire(Object o, long offset,
2352                                                        float expected,
2353                                                        float x) {
2354         return weakCompareAndSetIntAcquire(o, offset,
2355                                             Float.floatToRawIntBits(expected),
2356                                             Float.floatToRawIntBits(x));
2357     }
2358 
2359     @ForceInline
2360     public final boolean weakCompareAndSetFloatRelease(Object o, long offset,
2361                                                        float expected,
2362                                                        float x) {
2363         return weakCompareAndSetIntRelease(o, offset,
2364                                             Float.floatToRawIntBits(expected),
2365                                             Float.floatToRawIntBits(x));
2366     }
2367 
2368     @ForceInline
2369     public final boolean weakCompareAndSetFloat(Object o, long offset,
2370                                                 float expected,
2371                                                 float x) {
2372         return weakCompareAndSetInt(o, offset,
2373                                              Float.floatToRawIntBits(expected),
2374                                              Float.floatToRawIntBits(x));
2375     }
2376 
2377     /**
2378      * Atomically updates Java variable to {@code x} if it is currently
2379      * holding {@code expected}.
2380      *
2381      * <p>This operation has memory semantics of a {@code volatile} read
2382      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2383      *
2384      * @return {@code true} if successful
2385      */
2386     @ForceInline
2387     public final boolean compareAndSetDouble(Object o, long offset,
2388                                              double expected,
2389                                              double x) {
2390         return compareAndSetLong(o, offset,
2391                                  Double.doubleToRawLongBits(expected),
2392                                  Double.doubleToRawLongBits(x));
2393     }
2394 
2395     @ForceInline
2396     public final double compareAndExchangeDouble(Object o, long offset,
2397                                                  double expected,
2398                                                  double x) {
2399         long w = compareAndExchangeLong(o, offset,
2400                                         Double.doubleToRawLongBits(expected),
2401                                         Double.doubleToRawLongBits(x));
2402         return Double.longBitsToDouble(w);
2403     }
2404 
2405     @ForceInline
2406     public final double compareAndExchangeDoubleAcquire(Object o, long offset,
2407                                                         double expected,
2408                                                         double x) {
2409         long w = compareAndExchangeLongAcquire(o, offset,
2410                                                Double.doubleToRawLongBits(expected),
2411                                                Double.doubleToRawLongBits(x));
2412         return Double.longBitsToDouble(w);
2413     }
2414 
2415     @ForceInline
2416     public final double compareAndExchangeDoubleRelease(Object o, long offset,
2417                                                         double expected,
2418                                                         double x) {
2419         long w = compareAndExchangeLongRelease(o, offset,
2420                                                Double.doubleToRawLongBits(expected),
2421                                                Double.doubleToRawLongBits(x));
2422         return Double.longBitsToDouble(w);
2423     }
2424 
2425     @ForceInline
2426     public final boolean weakCompareAndSetDoublePlain(Object o, long offset,
2427                                                       double expected,
2428                                                       double x) {
2429         return weakCompareAndSetLongPlain(o, offset,
2430                                      Double.doubleToRawLongBits(expected),
2431                                      Double.doubleToRawLongBits(x));
2432     }
2433 
2434     @ForceInline
2435     public final boolean weakCompareAndSetDoubleAcquire(Object o, long offset,
2436                                                         double expected,
2437                                                         double x) {
2438         return weakCompareAndSetLongAcquire(o, offset,
2439                                              Double.doubleToRawLongBits(expected),
2440                                              Double.doubleToRawLongBits(x));
2441     }
2442 
2443     @ForceInline
2444     public final boolean weakCompareAndSetDoubleRelease(Object o, long offset,
2445                                                         double expected,
2446                                                         double x) {
2447         return weakCompareAndSetLongRelease(o, offset,
2448                                              Double.doubleToRawLongBits(expected),
2449                                              Double.doubleToRawLongBits(x));
2450     }
2451 
2452     @ForceInline
2453     public final boolean weakCompareAndSetDouble(Object o, long offset,
2454                                                  double expected,
2455                                                  double x) {
2456         return weakCompareAndSetLong(o, offset,
2457                                               Double.doubleToRawLongBits(expected),
2458                                               Double.doubleToRawLongBits(x));
2459     }
2460 
2461     /**
2462      * Atomically updates Java variable to {@code x} if it is currently
2463      * holding {@code expected}.
2464      *
2465      * <p>This operation has memory semantics of a {@code volatile} read
2466      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2467      *
2468      * @return {@code true} if successful
2469      */
2470     @IntrinsicCandidate
2471     public final native boolean compareAndSetLong(Object o, long offset,
2472                                                   long expected,
2473                                                   long x);
2474 
2475     @IntrinsicCandidate
2476     public final native long compareAndExchangeLong(Object o, long offset,
2477                                                     long expected,
2478                                                     long x);
2479 
2480     @IntrinsicCandidate
2481     public final long compareAndExchangeLongAcquire(Object o, long offset,
2482                                                            long expected,
2483                                                            long x) {
2484         return compareAndExchangeLong(o, offset, expected, x);
2485     }
2486 
2487     @IntrinsicCandidate
2488     public final long compareAndExchangeLongRelease(Object o, long offset,
2489                                                            long expected,
2490                                                            long x) {
2491         return compareAndExchangeLong(o, offset, expected, x);
2492     }
2493 
2494     @IntrinsicCandidate
2495     public final boolean weakCompareAndSetLongPlain(Object o, long offset,
2496                                                     long expected,
2497                                                     long x) {
2498         return compareAndSetLong(o, offset, expected, x);
2499     }
2500 
2501     @IntrinsicCandidate
2502     public final boolean weakCompareAndSetLongAcquire(Object o, long offset,
2503                                                       long expected,
2504                                                       long x) {
2505         return compareAndSetLong(o, offset, expected, x);
2506     }
2507 
2508     @IntrinsicCandidate
2509     public final boolean weakCompareAndSetLongRelease(Object o, long offset,
2510                                                       long expected,
2511                                                       long x) {
2512         return compareAndSetLong(o, offset, expected, x);
2513     }
2514 
2515     @IntrinsicCandidate
2516     public final boolean weakCompareAndSetLong(Object o, long offset,
2517                                                long expected,
2518                                                long x) {
2519         return compareAndSetLong(o, offset, expected, x);
2520     }
2521 
2522     /**
2523      * Fetches a reference value from a given Java variable, with volatile
2524      * load semantics. Otherwise identical to {@link #getReference(Object, long)}
2525      */
2526     @IntrinsicCandidate
2527     public native Object getReferenceVolatile(Object o, long offset);
2528 
2529     @ForceInline
2530     public final <V> Object getFlatValueVolatile(Object o, long offset, int layout, Class<?> valueType) {
2531         // we translate using fences (see: https://gee.cs.oswego.edu/dl/html/j9mm.html)
2532         Object res = getFlatValue(o, offset, layout, valueType);
2533         fullFence();
2534         return res;
2535     }
2536 
2537     /**
2538      * Stores a reference value into a given Java variable, with
2539      * volatile store semantics. Otherwise identical to {@link #putReference(Object, long, Object)}
2540      */
2541     @IntrinsicCandidate
2542     public native void putReferenceVolatile(Object o, long offset, Object x);
2543 
2544     @ForceInline
2545     public final <V> void putFlatValueVolatile(Object o, long offset, int layout, Class<?> valueType, V x) {
2546         // we translate using fences (see: https://gee.cs.oswego.edu/dl/html/j9mm.html)
2547         putFlatValueRelease(o, offset, layout, valueType, x);
2548         fullFence();
2549     }
2550 
2551     /** Volatile version of {@link #getInt(Object, long)}  */
2552     @IntrinsicCandidate
2553     public native int     getIntVolatile(Object o, long offset);
2554 
2555     /** Volatile version of {@link #putInt(Object, long, int)}  */
2556     @IntrinsicCandidate
2557     public native void    putIntVolatile(Object o, long offset, int x);
2558 
2559     /** Volatile version of {@link #getBoolean(Object, long)}  */
2560     @IntrinsicCandidate
2561     public native boolean getBooleanVolatile(Object o, long offset);
2562 
2563     /** Volatile version of {@link #putBoolean(Object, long, boolean)}  */
2564     @IntrinsicCandidate
2565     public native void    putBooleanVolatile(Object o, long offset, boolean x);
2566 
2567     /** Volatile version of {@link #getByte(Object, long)}  */
2568     @IntrinsicCandidate
2569     public native byte    getByteVolatile(Object o, long offset);
2570 
2571     /** Volatile version of {@link #putByte(Object, long, byte)}  */
2572     @IntrinsicCandidate
2573     public native void    putByteVolatile(Object o, long offset, byte x);
2574 
2575     /** Volatile version of {@link #getShort(Object, long)}  */
2576     @IntrinsicCandidate
2577     public native short   getShortVolatile(Object o, long offset);
2578 
2579     /** Volatile version of {@link #putShort(Object, long, short)}  */
2580     @IntrinsicCandidate
2581     public native void    putShortVolatile(Object o, long offset, short x);
2582 
2583     /** Volatile version of {@link #getChar(Object, long)}  */
2584     @IntrinsicCandidate
2585     public native char    getCharVolatile(Object o, long offset);
2586 
2587     /** Volatile version of {@link #putChar(Object, long, char)}  */
2588     @IntrinsicCandidate
2589     public native void    putCharVolatile(Object o, long offset, char x);
2590 
2591     /** Volatile version of {@link #getLong(Object, long)}  */
2592     @IntrinsicCandidate
2593     public native long    getLongVolatile(Object o, long offset);
2594 
2595     /** Volatile version of {@link #putLong(Object, long, long)}  */
2596     @IntrinsicCandidate
2597     public native void    putLongVolatile(Object o, long offset, long x);
2598 
2599     /** Volatile version of {@link #getFloat(Object, long)}  */
2600     @IntrinsicCandidate
2601     public native float   getFloatVolatile(Object o, long offset);
2602 
2603     /** Volatile version of {@link #putFloat(Object, long, float)}  */
2604     @IntrinsicCandidate
2605     public native void    putFloatVolatile(Object o, long offset, float x);
2606 
2607     /** Volatile version of {@link #getDouble(Object, long)}  */
2608     @IntrinsicCandidate
2609     public native double  getDoubleVolatile(Object o, long offset);
2610 
2611     /** Volatile version of {@link #putDouble(Object, long, double)}  */
2612     @IntrinsicCandidate
2613     public native void    putDoubleVolatile(Object o, long offset, double x);
2614 
2615 
2616 
2617     /** Acquire version of {@link #getReferenceVolatile(Object, long)} */
2618     @IntrinsicCandidate
2619     public final Object getReferenceAcquire(Object o, long offset) {
2620         return getReferenceVolatile(o, offset);
2621     }
2622 
2623     @ForceInline
2624     public final <V> Object getFlatValueAcquire(Object o, long offset, int layout, Class<?> valueType) {
2625         // we translate using fences (see: https://gee.cs.oswego.edu/dl/html/j9mm.html)
2626         Object res = getFlatValue(o, offset, layout, valueType);
2627         loadFence();
2628         return res;
2629     }
2630 
2631     /** Acquire version of {@link #getBooleanVolatile(Object, long)} */
2632     @IntrinsicCandidate
2633     public final boolean getBooleanAcquire(Object o, long offset) {
2634         return getBooleanVolatile(o, offset);
2635     }
2636 
2637     /** Acquire version of {@link #getByteVolatile(Object, long)} */
2638     @IntrinsicCandidate
2639     public final byte getByteAcquire(Object o, long offset) {
2640         return getByteVolatile(o, offset);
2641     }
2642 
2643     /** Acquire version of {@link #getShortVolatile(Object, long)} */
2644     @IntrinsicCandidate
2645     public final short getShortAcquire(Object o, long offset) {
2646         return getShortVolatile(o, offset);
2647     }
2648 
2649     /** Acquire version of {@link #getCharVolatile(Object, long)} */
2650     @IntrinsicCandidate
2651     public final char getCharAcquire(Object o, long offset) {
2652         return getCharVolatile(o, offset);
2653     }
2654 
2655     /** Acquire version of {@link #getIntVolatile(Object, long)} */
2656     @IntrinsicCandidate
2657     public final int getIntAcquire(Object o, long offset) {
2658         return getIntVolatile(o, offset);
2659     }
2660 
2661     /** Acquire version of {@link #getFloatVolatile(Object, long)} */
2662     @IntrinsicCandidate
2663     public final float getFloatAcquire(Object o, long offset) {
2664         return getFloatVolatile(o, offset);
2665     }
2666 
2667     /** Acquire version of {@link #getLongVolatile(Object, long)} */
2668     @IntrinsicCandidate
2669     public final long getLongAcquire(Object o, long offset) {
2670         return getLongVolatile(o, offset);
2671     }
2672 
2673     /** Acquire version of {@link #getDoubleVolatile(Object, long)} */
2674     @IntrinsicCandidate
2675     public final double getDoubleAcquire(Object o, long offset) {
2676         return getDoubleVolatile(o, offset);
2677     }
2678 
2679     /*
2680      * Versions of {@link #putReferenceVolatile(Object, long, Object)}
2681      * that do not guarantee immediate visibility of the store to
2682      * other threads. This method is generally only useful if the
2683      * underlying field is a Java volatile (or if an array cell, one
2684      * that is otherwise only accessed using volatile accesses).
2685      *
2686      * Corresponds to C11 atomic_store_explicit(..., memory_order_release).
2687      */
2688 
2689     /** Release version of {@link #putReferenceVolatile(Object, long, Object)} */
2690     @IntrinsicCandidate
2691     public final void putReferenceRelease(Object o, long offset, Object x) {
2692         putReferenceVolatile(o, offset, x);
2693     }
2694 
2695     @ForceInline
2696     public final <V> void putFlatValueRelease(Object o, long offset, int layout, Class<?> valueType, V x) {
2697         // we translate using fences (see: https://gee.cs.oswego.edu/dl/html/j9mm.html)
2698         storeFence();
2699         putFlatValue(o, offset, layout, valueType, x);
2700     }
2701 
2702     /** Release version of {@link #putBooleanVolatile(Object, long, boolean)} */
2703     @IntrinsicCandidate
2704     public final void putBooleanRelease(Object o, long offset, boolean x) {
2705         putBooleanVolatile(o, offset, x);
2706     }
2707 
2708     /** Release version of {@link #putByteVolatile(Object, long, byte)} */
2709     @IntrinsicCandidate
2710     public final void putByteRelease(Object o, long offset, byte x) {
2711         putByteVolatile(o, offset, x);
2712     }
2713 
2714     /** Release version of {@link #putShortVolatile(Object, long, short)} */
2715     @IntrinsicCandidate
2716     public final void putShortRelease(Object o, long offset, short x) {
2717         putShortVolatile(o, offset, x);
2718     }
2719 
2720     /** Release version of {@link #putCharVolatile(Object, long, char)} */
2721     @IntrinsicCandidate
2722     public final void putCharRelease(Object o, long offset, char x) {
2723         putCharVolatile(o, offset, x);
2724     }
2725 
2726     /** Release version of {@link #putIntVolatile(Object, long, int)} */
2727     @IntrinsicCandidate
2728     public final void putIntRelease(Object o, long offset, int x) {
2729         putIntVolatile(o, offset, x);
2730     }
2731 
2732     /** Release version of {@link #putFloatVolatile(Object, long, float)} */
2733     @IntrinsicCandidate
2734     public final void putFloatRelease(Object o, long offset, float x) {
2735         putFloatVolatile(o, offset, x);
2736     }
2737 
2738     /** Release version of {@link #putLongVolatile(Object, long, long)} */
2739     @IntrinsicCandidate
2740     public final void putLongRelease(Object o, long offset, long x) {
2741         putLongVolatile(o, offset, x);
2742     }
2743 
2744     /** Release version of {@link #putDoubleVolatile(Object, long, double)} */
2745     @IntrinsicCandidate
2746     public final void putDoubleRelease(Object o, long offset, double x) {
2747         putDoubleVolatile(o, offset, x);
2748     }
2749 
2750     // ------------------------------ Opaque --------------------------------------
2751 
2752     /** Opaque version of {@link #getReferenceVolatile(Object, long)} */
2753     @IntrinsicCandidate
2754     public final Object getReferenceOpaque(Object o, long offset) {
2755         return getReferenceVolatile(o, offset);
2756     }
2757 
2758     @ForceInline
2759     public final <V> Object getFlatValueOpaque(Object o, long offset, int layout, Class<?> valueType) {
2760         // this is stronger than opaque semantics
2761         return getFlatValueAcquire(o, offset, layout, valueType);
2762     }
2763 
2764     /** Opaque version of {@link #getBooleanVolatile(Object, long)} */
2765     @IntrinsicCandidate
2766     public final boolean getBooleanOpaque(Object o, long offset) {
2767         return getBooleanVolatile(o, offset);
2768     }
2769 
2770     /** Opaque version of {@link #getByteVolatile(Object, long)} */
2771     @IntrinsicCandidate
2772     public final byte getByteOpaque(Object o, long offset) {
2773         return getByteVolatile(o, offset);
2774     }
2775 
2776     /** Opaque version of {@link #getShortVolatile(Object, long)} */
2777     @IntrinsicCandidate
2778     public final short getShortOpaque(Object o, long offset) {
2779         return getShortVolatile(o, offset);
2780     }
2781 
2782     /** Opaque version of {@link #getCharVolatile(Object, long)} */
2783     @IntrinsicCandidate
2784     public final char getCharOpaque(Object o, long offset) {
2785         return getCharVolatile(o, offset);
2786     }
2787 
2788     /** Opaque version of {@link #getIntVolatile(Object, long)} */
2789     @IntrinsicCandidate
2790     public final int getIntOpaque(Object o, long offset) {
2791         return getIntVolatile(o, offset);
2792     }
2793 
2794     /** Opaque version of {@link #getFloatVolatile(Object, long)} */
2795     @IntrinsicCandidate
2796     public final float getFloatOpaque(Object o, long offset) {
2797         return getFloatVolatile(o, offset);
2798     }
2799 
2800     /** Opaque version of {@link #getLongVolatile(Object, long)} */
2801     @IntrinsicCandidate
2802     public final long getLongOpaque(Object o, long offset) {
2803         return getLongVolatile(o, offset);
2804     }
2805 
2806     /** Opaque version of {@link #getDoubleVolatile(Object, long)} */
2807     @IntrinsicCandidate
2808     public final double getDoubleOpaque(Object o, long offset) {
2809         return getDoubleVolatile(o, offset);
2810     }
2811 
2812     /** Opaque version of {@link #putReferenceVolatile(Object, long, Object)} */
2813     @IntrinsicCandidate
2814     public final void putReferenceOpaque(Object o, long offset, Object x) {
2815         putReferenceVolatile(o, offset, x);
2816     }
2817 
2818     @ForceInline
2819     public final <V> void putFlatValueOpaque(Object o, long offset, int layout, Class<?> valueType, V x) {
2820         // this is stronger than opaque semantics
2821         putFlatValueRelease(o, offset, layout, valueType, x);
2822     }
2823 
2824     /** Opaque version of {@link #putBooleanVolatile(Object, long, boolean)} */
2825     @IntrinsicCandidate
2826     public final void putBooleanOpaque(Object o, long offset, boolean x) {
2827         putBooleanVolatile(o, offset, x);
2828     }
2829 
2830     /** Opaque version of {@link #putByteVolatile(Object, long, byte)} */
2831     @IntrinsicCandidate
2832     public final void putByteOpaque(Object o, long offset, byte x) {
2833         putByteVolatile(o, offset, x);
2834     }
2835 
2836     /** Opaque version of {@link #putShortVolatile(Object, long, short)} */
2837     @IntrinsicCandidate
2838     public final void putShortOpaque(Object o, long offset, short x) {
2839         putShortVolatile(o, offset, x);
2840     }
2841 
2842     /** Opaque version of {@link #putCharVolatile(Object, long, char)} */
2843     @IntrinsicCandidate
2844     public final void putCharOpaque(Object o, long offset, char x) {
2845         putCharVolatile(o, offset, x);
2846     }
2847 
2848     /** Opaque version of {@link #putIntVolatile(Object, long, int)} */
2849     @IntrinsicCandidate
2850     public final void putIntOpaque(Object o, long offset, int x) {
2851         putIntVolatile(o, offset, x);
2852     }
2853 
2854     /** Opaque version of {@link #putFloatVolatile(Object, long, float)} */
2855     @IntrinsicCandidate
2856     public final void putFloatOpaque(Object o, long offset, float x) {
2857         putFloatVolatile(o, offset, x);
2858     }
2859 
2860     /** Opaque version of {@link #putLongVolatile(Object, long, long)} */
2861     @IntrinsicCandidate
2862     public final void putLongOpaque(Object o, long offset, long x) {
2863         putLongVolatile(o, offset, x);
2864     }
2865 
2866     /** Opaque version of {@link #putDoubleVolatile(Object, long, double)} */
2867     @IntrinsicCandidate
2868     public final void putDoubleOpaque(Object o, long offset, double x) {
2869         putDoubleVolatile(o, offset, x);
2870     }
2871 
2872     @ForceInline
2873     private boolean compareAndSetFlatValueAsBytes(Object o, long offset, int layout, Class<?> valueType, Object expected, Object x) {
2874         // We turn the payload of an atomic value into a numeric value (of suitable type)
2875         // by storing the value into an array element (of matching layout) and by reading
2876         // back the array element as an integral value. After which we can implement the CAS
2877         // as a plain numeric CAS. Note: this only works if the payload contains no oops
2878         // (see VarHandles::isAtomicFlat).
2879         Object[] expectedArray = newSpecialArray(valueType, 1, layout);
2880         Object xArray = newSpecialArray(valueType, 1, layout);
2881         long base = arrayBaseOffset(expectedArray);
2882         int scale = arrayIndexScale(expectedArray);
2883         putFlatValue(expectedArray, base, layout, valueType, expected);
2884         putFlatValue(xArray, base, layout, valueType, x);
2885         switch (scale) {
2886             case 1: {
2887                 byte expectedByte = getByte(expectedArray, base);
2888                 byte xByte = getByte(xArray, base);
2889                 return compareAndSetByte(o, offset, expectedByte, xByte);
2890             }
2891             case 2: {
2892                 short expectedShort = getShort(expectedArray, base);
2893                 short xShort = getShort(xArray, base);
2894                 return compareAndSetShort(o, offset, expectedShort, xShort);
2895             }
2896             case 4: {
2897                 int expectedInt = getInt(expectedArray, base);
2898                 int xInt = getInt(xArray, base);
2899                 return compareAndSetInt(o, offset, expectedInt, xInt);
2900             }
2901             case 8: {
2902                 long expectedLong = getLong(expectedArray, base);
2903                 long xLong = getLong(xArray, base);
2904                 return compareAndSetLong(o, offset, expectedLong, xLong);
2905             }
2906             default: {
2907                 throw new UnsupportedOperationException();
2908             }
2909         }
2910     }
2911 
2912     /**
2913      * Unblocks the given thread blocked on {@code park}, or, if it is
2914      * not blocked, causes the subsequent call to {@code park} not to
2915      * block.  Note: this operation is "unsafe" solely because the
2916      * caller must somehow ensure that the thread has not been
2917      * destroyed. Nothing special is usually required to ensure this
2918      * when called from Java (in which there will ordinarily be a live
2919      * reference to the thread) but this is not nearly-automatically
2920      * so when calling from native code.
2921      *
2922      * @param thread the thread to unpark.
2923      */
2924     @IntrinsicCandidate
2925     public native void unpark(Object thread);
2926 
2927     /**
2928      * Blocks current thread, returning when a balancing
2929      * {@code unpark} occurs, or a balancing {@code unpark} has
2930      * already occurred, or the thread is interrupted, or, if not
2931      * absolute and time is not zero, the given time nanoseconds have
2932      * elapsed, or if absolute, the given deadline in milliseconds
2933      * since Epoch has passed, or spuriously (i.e., returning for no
2934      * "reason"). Note: This operation is in the Unsafe class only
2935      * because {@code unpark} is, so it would be strange to place it
2936      * elsewhere.
2937      */
2938     @IntrinsicCandidate
2939     public native void park(boolean isAbsolute, long time);
2940 
2941     /**
2942      * Gets the load average in the system run queue assigned
2943      * to the available processors averaged over various periods of time.
2944      * This method retrieves the given {@code nelem} samples and
2945      * assigns to the elements of the given {@code loadavg} array.
2946      * The system imposes a maximum of 3 samples, representing
2947      * averages over the last 1,  5,  and  15 minutes, respectively.
2948      *
2949      * @param loadavg an array of double of size nelems
2950      * @param nelems the number of samples to be retrieved and
2951      *        must be 1 to 3.
2952      *
2953      * @return the number of samples actually retrieved; or -1
2954      *         if the load average is unobtainable.
2955      */
2956     public int getLoadAverage(double[] loadavg, int nelems) {
2957         if (nelems < 0 || nelems > 3 || nelems > loadavg.length) {
2958             throw new ArrayIndexOutOfBoundsException();
2959         }
2960 
2961         return getLoadAverage0(loadavg, nelems);
2962     }
2963 
2964     // The following contain CAS-based Java implementations used on
2965     // platforms not supporting native instructions
2966 
2967     /**
2968      * Atomically adds the given value to the current value of a field
2969      * or array element within the given object {@code o}
2970      * at the given {@code offset}.
2971      *
2972      * @param o object/array to update the field/element in
2973      * @param offset field/element offset
2974      * @param delta the value to add
2975      * @return the previous value
2976      * @since 1.8
2977      */
2978     @IntrinsicCandidate
2979     public final int getAndAddInt(Object o, long offset, int delta) {
2980         int v;
2981         do {
2982             v = getIntVolatile(o, offset);
2983         } while (!weakCompareAndSetInt(o, offset, v, v + delta));
2984         return v;
2985     }
2986 
2987     @ForceInline
2988     public final int getAndAddIntRelease(Object o, long offset, int delta) {
2989         int v;
2990         do {
2991             v = getInt(o, offset);
2992         } while (!weakCompareAndSetIntRelease(o, offset, v, v + delta));
2993         return v;
2994     }
2995 
2996     @ForceInline
2997     public final int getAndAddIntAcquire(Object o, long offset, int delta) {
2998         int v;
2999         do {
3000             v = getIntAcquire(o, offset);
3001         } while (!weakCompareAndSetIntAcquire(o, offset, v, v + delta));
3002         return v;
3003     }
3004 
3005     /**
3006      * Atomically adds the given value to the current value of a field
3007      * or array element within the given object {@code o}
3008      * at the given {@code offset}.
3009      *
3010      * @param o object/array to update the field/element in
3011      * @param offset field/element offset
3012      * @param delta the value to add
3013      * @return the previous value
3014      * @since 1.8
3015      */
3016     @IntrinsicCandidate
3017     public final long getAndAddLong(Object o, long offset, long delta) {
3018         long v;
3019         do {
3020             v = getLongVolatile(o, offset);
3021         } while (!weakCompareAndSetLong(o, offset, v, v + delta));
3022         return v;
3023     }
3024 
3025     @ForceInline
3026     public final long getAndAddLongRelease(Object o, long offset, long delta) {
3027         long v;
3028         do {
3029             v = getLong(o, offset);
3030         } while (!weakCompareAndSetLongRelease(o, offset, v, v + delta));
3031         return v;
3032     }
3033 
3034     @ForceInline
3035     public final long getAndAddLongAcquire(Object o, long offset, long delta) {
3036         long v;
3037         do {
3038             v = getLongAcquire(o, offset);
3039         } while (!weakCompareAndSetLongAcquire(o, offset, v, v + delta));
3040         return v;
3041     }
3042 
3043     @IntrinsicCandidate
3044     public final byte getAndAddByte(Object o, long offset, byte delta) {
3045         byte v;
3046         do {
3047             v = getByteVolatile(o, offset);
3048         } while (!weakCompareAndSetByte(o, offset, v, (byte) (v + delta)));
3049         return v;
3050     }
3051 
3052     @ForceInline
3053     public final byte getAndAddByteRelease(Object o, long offset, byte delta) {
3054         byte v;
3055         do {
3056             v = getByte(o, offset);
3057         } while (!weakCompareAndSetByteRelease(o, offset, v, (byte) (v + delta)));
3058         return v;
3059     }
3060 
3061     @ForceInline
3062     public final byte getAndAddByteAcquire(Object o, long offset, byte delta) {
3063         byte v;
3064         do {
3065             v = getByteAcquire(o, offset);
3066         } while (!weakCompareAndSetByteAcquire(o, offset, v, (byte) (v + delta)));
3067         return v;
3068     }
3069 
3070     @IntrinsicCandidate
3071     public final short getAndAddShort(Object o, long offset, short delta) {
3072         short v;
3073         do {
3074             v = getShortVolatile(o, offset);
3075         } while (!weakCompareAndSetShort(o, offset, v, (short) (v + delta)));
3076         return v;
3077     }
3078 
3079     @ForceInline
3080     public final short getAndAddShortRelease(Object o, long offset, short delta) {
3081         short v;
3082         do {
3083             v = getShort(o, offset);
3084         } while (!weakCompareAndSetShortRelease(o, offset, v, (short) (v + delta)));
3085         return v;
3086     }
3087 
3088     @ForceInline
3089     public final short getAndAddShortAcquire(Object o, long offset, short delta) {
3090         short v;
3091         do {
3092             v = getShortAcquire(o, offset);
3093         } while (!weakCompareAndSetShortAcquire(o, offset, v, (short) (v + delta)));
3094         return v;
3095     }
3096 
3097     @ForceInline
3098     public final char getAndAddChar(Object o, long offset, char delta) {
3099         return (char) getAndAddShort(o, offset, (short) delta);
3100     }
3101 
3102     @ForceInline
3103     public final char getAndAddCharRelease(Object o, long offset, char delta) {
3104         return (char) getAndAddShortRelease(o, offset, (short) delta);
3105     }
3106 
3107     @ForceInline
3108     public final char getAndAddCharAcquire(Object o, long offset, char delta) {
3109         return (char) getAndAddShortAcquire(o, offset, (short) delta);
3110     }
3111 
3112     @ForceInline
3113     public final float getAndAddFloat(Object o, long offset, float delta) {
3114         int expectedBits;
3115         float v;
3116         do {
3117             // Load and CAS with the raw bits to avoid issues with NaNs and
3118             // possible bit conversion from signaling NaNs to quiet NaNs that
3119             // may result in the loop not terminating.
3120             expectedBits = getIntVolatile(o, offset);
3121             v = Float.intBitsToFloat(expectedBits);
3122         } while (!weakCompareAndSetInt(o, offset,
3123                                                 expectedBits, Float.floatToRawIntBits(v + delta)));
3124         return v;
3125     }
3126 
3127     @ForceInline
3128     public final float getAndAddFloatRelease(Object o, long offset, float delta) {
3129         int expectedBits;
3130         float v;
3131         do {
3132             // Load and CAS with the raw bits to avoid issues with NaNs and
3133             // possible bit conversion from signaling NaNs to quiet NaNs that
3134             // may result in the loop not terminating.
3135             expectedBits = getInt(o, offset);
3136             v = Float.intBitsToFloat(expectedBits);
3137         } while (!weakCompareAndSetIntRelease(o, offset,
3138                                                expectedBits, Float.floatToRawIntBits(v + delta)));
3139         return v;
3140     }
3141 
3142     @ForceInline
3143     public final float getAndAddFloatAcquire(Object o, long offset, float delta) {
3144         int expectedBits;
3145         float v;
3146         do {
3147             // Load and CAS with the raw bits to avoid issues with NaNs and
3148             // possible bit conversion from signaling NaNs to quiet NaNs that
3149             // may result in the loop not terminating.
3150             expectedBits = getIntAcquire(o, offset);
3151             v = Float.intBitsToFloat(expectedBits);
3152         } while (!weakCompareAndSetIntAcquire(o, offset,
3153                                                expectedBits, Float.floatToRawIntBits(v + delta)));
3154         return v;
3155     }
3156 
3157     @ForceInline
3158     public final double getAndAddDouble(Object o, long offset, double delta) {
3159         long expectedBits;
3160         double v;
3161         do {
3162             // Load and CAS with the raw bits to avoid issues with NaNs and
3163             // possible bit conversion from signaling NaNs to quiet NaNs that
3164             // may result in the loop not terminating.
3165             expectedBits = getLongVolatile(o, offset);
3166             v = Double.longBitsToDouble(expectedBits);
3167         } while (!weakCompareAndSetLong(o, offset,
3168                                                  expectedBits, Double.doubleToRawLongBits(v + delta)));
3169         return v;
3170     }
3171 
3172     @ForceInline
3173     public final double getAndAddDoubleRelease(Object o, long offset, double delta) {
3174         long expectedBits;
3175         double v;
3176         do {
3177             // Load and CAS with the raw bits to avoid issues with NaNs and
3178             // possible bit conversion from signaling NaNs to quiet NaNs that
3179             // may result in the loop not terminating.
3180             expectedBits = getLong(o, offset);
3181             v = Double.longBitsToDouble(expectedBits);
3182         } while (!weakCompareAndSetLongRelease(o, offset,
3183                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
3184         return v;
3185     }
3186 
3187     @ForceInline
3188     public final double getAndAddDoubleAcquire(Object o, long offset, double delta) {
3189         long expectedBits;
3190         double v;
3191         do {
3192             // Load and CAS with the raw bits to avoid issues with NaNs and
3193             // possible bit conversion from signaling NaNs to quiet NaNs that
3194             // may result in the loop not terminating.
3195             expectedBits = getLongAcquire(o, offset);
3196             v = Double.longBitsToDouble(expectedBits);
3197         } while (!weakCompareAndSetLongAcquire(o, offset,
3198                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
3199         return v;
3200     }
3201 
3202     /**
3203      * Atomically exchanges the given value with the current value of
3204      * a field or array element within the given object {@code o}
3205      * at the given {@code offset}.
3206      *
3207      * @param o object/array to update the field/element in
3208      * @param offset field/element offset
3209      * @param newValue new value
3210      * @return the previous value
3211      * @since 1.8
3212      */
3213     @IntrinsicCandidate
3214     public final int getAndSetInt(Object o, long offset, int newValue) {
3215         int v;
3216         do {
3217             v = getIntVolatile(o, offset);
3218         } while (!weakCompareAndSetInt(o, offset, v, newValue));
3219         return v;
3220     }
3221 
3222     @ForceInline
3223     public final int getAndSetIntRelease(Object o, long offset, int newValue) {
3224         int v;
3225         do {
3226             v = getInt(o, offset);
3227         } while (!weakCompareAndSetIntRelease(o, offset, v, newValue));
3228         return v;
3229     }
3230 
3231     @ForceInline
3232     public final int getAndSetIntAcquire(Object o, long offset, int newValue) {
3233         int v;
3234         do {
3235             v = getIntAcquire(o, offset);
3236         } while (!weakCompareAndSetIntAcquire(o, offset, v, newValue));
3237         return v;
3238     }
3239 
3240     /**
3241      * Atomically exchanges the given value with the current value of
3242      * a field or array element within the given object {@code o}
3243      * at the given {@code offset}.
3244      *
3245      * @param o object/array to update the field/element in
3246      * @param offset field/element offset
3247      * @param newValue new value
3248      * @return the previous value
3249      * @since 1.8
3250      */
3251     @IntrinsicCandidate
3252     public final long getAndSetLong(Object o, long offset, long newValue) {
3253         long v;
3254         do {
3255             v = getLongVolatile(o, offset);
3256         } while (!weakCompareAndSetLong(o, offset, v, newValue));
3257         return v;
3258     }
3259 
3260     @ForceInline
3261     public final long getAndSetLongRelease(Object o, long offset, long newValue) {
3262         long v;
3263         do {
3264             v = getLong(o, offset);
3265         } while (!weakCompareAndSetLongRelease(o, offset, v, newValue));
3266         return v;
3267     }
3268 
3269     @ForceInline
3270     public final long getAndSetLongAcquire(Object o, long offset, long newValue) {
3271         long v;
3272         do {
3273             v = getLongAcquire(o, offset);
3274         } while (!weakCompareAndSetLongAcquire(o, offset, v, newValue));
3275         return v;
3276     }
3277 
3278     /**
3279      * Atomically exchanges the given reference value with the current
3280      * reference value of a field or array element within the given
3281      * object {@code o} at the given {@code offset}.
3282      *
3283      * @param o object/array to update the field/element in
3284      * @param offset field/element offset
3285      * @param newValue new value
3286      * @return the previous value
3287      * @since 1.8
3288      */
3289     @IntrinsicCandidate
3290     public final Object getAndSetReference(Object o, long offset, Object newValue) {
3291         Object v;
3292         do {
3293             v = getReferenceVolatile(o, offset);
3294         } while (!weakCompareAndSetReference(o, offset, v, newValue));
3295         return v;
3296     }
3297 
3298     @ForceInline
3299     public final Object getAndSetReference(Object o, long offset, Class<?> valueType, Object newValue) {
3300         Object v;
3301         do {
3302             v = getReferenceVolatile(o, offset);
3303         } while (!compareAndSetReference(o, offset, valueType, v, newValue));
3304         return v;
3305     }
3306 
3307     @ForceInline
3308     public Object getAndSetFlatValue(Object o, long offset, int layoutKind, Class<?> valueType, Object newValue) {
3309         Object v;
3310         do {
3311             v = getFlatValueVolatile(o, offset, layoutKind, valueType);
3312         } while (!compareAndSetFlatValue(o, offset, layoutKind, valueType, v, newValue));
3313         return v;
3314     }
3315 
3316     @ForceInline
3317     public final Object getAndSetReferenceRelease(Object o, long offset, Object newValue) {
3318         Object v;
3319         do {
3320             v = getReference(o, offset);
3321         } while (!weakCompareAndSetReferenceRelease(o, offset, v, newValue));
3322         return v;
3323     }
3324 
3325     @ForceInline
3326     public final Object getAndSetReferenceRelease(Object o, long offset, Class<?> valueType, Object newValue) {
3327         return getAndSetReference(o, offset, valueType, newValue);
3328     }
3329 
3330     @ForceInline
3331     public Object getAndSetFlatValueRelease(Object o, long offset, int layoutKind, Class<?> valueType, Object x) {
3332         return getAndSetFlatValue(o, offset, layoutKind, valueType, x);
3333     }
3334 
3335     @ForceInline
3336     public final Object getAndSetReferenceAcquire(Object o, long offset, Object newValue) {
3337         Object v;
3338         do {
3339             v = getReferenceAcquire(o, offset);
3340         } while (!weakCompareAndSetReferenceAcquire(o, offset, v, newValue));
3341         return v;
3342     }
3343 
3344     @ForceInline
3345     public final Object getAndSetReferenceAcquire(Object o, long offset, Class<?> valueType, Object newValue) {
3346         return getAndSetReference(o, offset, valueType, newValue);
3347     }
3348 
3349     @ForceInline
3350     public Object getAndSetFlatValueAcquire(Object o, long offset, int layoutKind, Class<?> valueType, Object x) {
3351         return getAndSetFlatValue(o, offset, layoutKind, valueType, x);
3352     }
3353 
3354     @IntrinsicCandidate
3355     public final byte getAndSetByte(Object o, long offset, byte newValue) {
3356         byte v;
3357         do {
3358             v = getByteVolatile(o, offset);
3359         } while (!weakCompareAndSetByte(o, offset, v, newValue));
3360         return v;
3361     }
3362 
3363     @ForceInline
3364     public final byte getAndSetByteRelease(Object o, long offset, byte newValue) {
3365         byte v;
3366         do {
3367             v = getByte(o, offset);
3368         } while (!weakCompareAndSetByteRelease(o, offset, v, newValue));
3369         return v;
3370     }
3371 
3372     @ForceInline
3373     public final byte getAndSetByteAcquire(Object o, long offset, byte newValue) {
3374         byte v;
3375         do {
3376             v = getByteAcquire(o, offset);
3377         } while (!weakCompareAndSetByteAcquire(o, offset, v, newValue));
3378         return v;
3379     }
3380 
3381     @ForceInline
3382     public final boolean getAndSetBoolean(Object o, long offset, boolean newValue) {
3383         return byte2bool(getAndSetByte(o, offset, bool2byte(newValue)));
3384     }
3385 
3386     @ForceInline
3387     public final boolean getAndSetBooleanRelease(Object o, long offset, boolean newValue) {
3388         return byte2bool(getAndSetByteRelease(o, offset, bool2byte(newValue)));
3389     }
3390 
3391     @ForceInline
3392     public final boolean getAndSetBooleanAcquire(Object o, long offset, boolean newValue) {
3393         return byte2bool(getAndSetByteAcquire(o, offset, bool2byte(newValue)));
3394     }
3395 
3396     @IntrinsicCandidate
3397     public final short getAndSetShort(Object o, long offset, short newValue) {
3398         short v;
3399         do {
3400             v = getShortVolatile(o, offset);
3401         } while (!weakCompareAndSetShort(o, offset, v, newValue));
3402         return v;
3403     }
3404 
3405     @ForceInline
3406     public final short getAndSetShortRelease(Object o, long offset, short newValue) {
3407         short v;
3408         do {
3409             v = getShort(o, offset);
3410         } while (!weakCompareAndSetShortRelease(o, offset, v, newValue));
3411         return v;
3412     }
3413 
3414     @ForceInline
3415     public final short getAndSetShortAcquire(Object o, long offset, short newValue) {
3416         short v;
3417         do {
3418             v = getShortAcquire(o, offset);
3419         } while (!weakCompareAndSetShortAcquire(o, offset, v, newValue));
3420         return v;
3421     }
3422 
3423     @ForceInline
3424     public final char getAndSetChar(Object o, long offset, char newValue) {
3425         return s2c(getAndSetShort(o, offset, c2s(newValue)));
3426     }
3427 
3428     @ForceInline
3429     public final char getAndSetCharRelease(Object o, long offset, char newValue) {
3430         return s2c(getAndSetShortRelease(o, offset, c2s(newValue)));
3431     }
3432 
3433     @ForceInline
3434     public final char getAndSetCharAcquire(Object o, long offset, char newValue) {
3435         return s2c(getAndSetShortAcquire(o, offset, c2s(newValue)));
3436     }
3437 
3438     @ForceInline
3439     public final float getAndSetFloat(Object o, long offset, float newValue) {
3440         int v = getAndSetInt(o, offset, Float.floatToRawIntBits(newValue));
3441         return Float.intBitsToFloat(v);
3442     }
3443 
3444     @ForceInline
3445     public final float getAndSetFloatRelease(Object o, long offset, float newValue) {
3446         int v = getAndSetIntRelease(o, offset, Float.floatToRawIntBits(newValue));
3447         return Float.intBitsToFloat(v);
3448     }
3449 
3450     @ForceInline
3451     public final float getAndSetFloatAcquire(Object o, long offset, float newValue) {
3452         int v = getAndSetIntAcquire(o, offset, Float.floatToRawIntBits(newValue));
3453         return Float.intBitsToFloat(v);
3454     }
3455 
3456     @ForceInline
3457     public final double getAndSetDouble(Object o, long offset, double newValue) {
3458         long v = getAndSetLong(o, offset, Double.doubleToRawLongBits(newValue));
3459         return Double.longBitsToDouble(v);
3460     }
3461 
3462     @ForceInline
3463     public final double getAndSetDoubleRelease(Object o, long offset, double newValue) {
3464         long v = getAndSetLongRelease(o, offset, Double.doubleToRawLongBits(newValue));
3465         return Double.longBitsToDouble(v);
3466     }
3467 
3468     @ForceInline
3469     public final double getAndSetDoubleAcquire(Object o, long offset, double newValue) {
3470         long v = getAndSetLongAcquire(o, offset, Double.doubleToRawLongBits(newValue));
3471         return Double.longBitsToDouble(v);
3472     }
3473 
3474 
3475     // The following contain CAS-based Java implementations used on
3476     // platforms not supporting native instructions
3477 
3478     @ForceInline
3479     public final boolean getAndBitwiseOrBoolean(Object o, long offset, boolean mask) {
3480         return byte2bool(getAndBitwiseOrByte(o, offset, bool2byte(mask)));
3481     }
3482 
3483     @ForceInline
3484     public final boolean getAndBitwiseOrBooleanRelease(Object o, long offset, boolean mask) {
3485         return byte2bool(getAndBitwiseOrByteRelease(o, offset, bool2byte(mask)));
3486     }
3487 
3488     @ForceInline
3489     public final boolean getAndBitwiseOrBooleanAcquire(Object o, long offset, boolean mask) {
3490         return byte2bool(getAndBitwiseOrByteAcquire(o, offset, bool2byte(mask)));
3491     }
3492 
3493     @ForceInline
3494     public final boolean getAndBitwiseAndBoolean(Object o, long offset, boolean mask) {
3495         return byte2bool(getAndBitwiseAndByte(o, offset, bool2byte(mask)));
3496     }
3497 
3498     @ForceInline
3499     public final boolean getAndBitwiseAndBooleanRelease(Object o, long offset, boolean mask) {
3500         return byte2bool(getAndBitwiseAndByteRelease(o, offset, bool2byte(mask)));
3501     }
3502 
3503     @ForceInline
3504     public final boolean getAndBitwiseAndBooleanAcquire(Object o, long offset, boolean mask) {
3505         return byte2bool(getAndBitwiseAndByteAcquire(o, offset, bool2byte(mask)));
3506     }
3507 
3508     @ForceInline
3509     public final boolean getAndBitwiseXorBoolean(Object o, long offset, boolean mask) {
3510         return byte2bool(getAndBitwiseXorByte(o, offset, bool2byte(mask)));
3511     }
3512 
3513     @ForceInline
3514     public final boolean getAndBitwiseXorBooleanRelease(Object o, long offset, boolean mask) {
3515         return byte2bool(getAndBitwiseXorByteRelease(o, offset, bool2byte(mask)));
3516     }
3517 
3518     @ForceInline
3519     public final boolean getAndBitwiseXorBooleanAcquire(Object o, long offset, boolean mask) {
3520         return byte2bool(getAndBitwiseXorByteAcquire(o, offset, bool2byte(mask)));
3521     }
3522 
3523 
3524     @ForceInline
3525     public final byte getAndBitwiseOrByte(Object o, long offset, byte mask) {
3526         byte current;
3527         do {
3528             current = getByteVolatile(o, offset);
3529         } while (!weakCompareAndSetByte(o, offset,
3530                                                   current, (byte) (current | mask)));
3531         return current;
3532     }
3533 
3534     @ForceInline
3535     public final byte getAndBitwiseOrByteRelease(Object o, long offset, byte mask) {
3536         byte current;
3537         do {
3538             current = getByte(o, offset);
3539         } while (!weakCompareAndSetByteRelease(o, offset,
3540                                                  current, (byte) (current | mask)));
3541         return current;
3542     }
3543 
3544     @ForceInline
3545     public final byte getAndBitwiseOrByteAcquire(Object o, long offset, byte mask) {
3546         byte current;
3547         do {
3548             // Plain read, the value is a hint, the acquire CAS does the work
3549             current = getByte(o, offset);
3550         } while (!weakCompareAndSetByteAcquire(o, offset,
3551                                                  current, (byte) (current | mask)));
3552         return current;
3553     }
3554 
3555     @ForceInline
3556     public final byte getAndBitwiseAndByte(Object o, long offset, byte mask) {
3557         byte current;
3558         do {
3559             current = getByteVolatile(o, offset);
3560         } while (!weakCompareAndSetByte(o, offset,
3561                                                   current, (byte) (current & mask)));
3562         return current;
3563     }
3564 
3565     @ForceInline
3566     public final byte getAndBitwiseAndByteRelease(Object o, long offset, byte mask) {
3567         byte current;
3568         do {
3569             current = getByte(o, offset);
3570         } while (!weakCompareAndSetByteRelease(o, offset,
3571                                                  current, (byte) (current & mask)));
3572         return current;
3573     }
3574 
3575     @ForceInline
3576     public final byte getAndBitwiseAndByteAcquire(Object o, long offset, byte mask) {
3577         byte current;
3578         do {
3579             // Plain read, the value is a hint, the acquire CAS does the work
3580             current = getByte(o, offset);
3581         } while (!weakCompareAndSetByteAcquire(o, offset,
3582                                                  current, (byte) (current & mask)));
3583         return current;
3584     }
3585 
3586     @ForceInline
3587     public final byte getAndBitwiseXorByte(Object o, long offset, byte mask) {
3588         byte current;
3589         do {
3590             current = getByteVolatile(o, offset);
3591         } while (!weakCompareAndSetByte(o, offset,
3592                                                   current, (byte) (current ^ mask)));
3593         return current;
3594     }
3595 
3596     @ForceInline
3597     public final byte getAndBitwiseXorByteRelease(Object o, long offset, byte mask) {
3598         byte current;
3599         do {
3600             current = getByte(o, offset);
3601         } while (!weakCompareAndSetByteRelease(o, offset,
3602                                                  current, (byte) (current ^ mask)));
3603         return current;
3604     }
3605 
3606     @ForceInline
3607     public final byte getAndBitwiseXorByteAcquire(Object o, long offset, byte mask) {
3608         byte current;
3609         do {
3610             // Plain read, the value is a hint, the acquire CAS does the work
3611             current = getByte(o, offset);
3612         } while (!weakCompareAndSetByteAcquire(o, offset,
3613                                                  current, (byte) (current ^ mask)));
3614         return current;
3615     }
3616 
3617 
3618     @ForceInline
3619     public final char getAndBitwiseOrChar(Object o, long offset, char mask) {
3620         return s2c(getAndBitwiseOrShort(o, offset, c2s(mask)));
3621     }
3622 
3623     @ForceInline
3624     public final char getAndBitwiseOrCharRelease(Object o, long offset, char mask) {
3625         return s2c(getAndBitwiseOrShortRelease(o, offset, c2s(mask)));
3626     }
3627 
3628     @ForceInline
3629     public final char getAndBitwiseOrCharAcquire(Object o, long offset, char mask) {
3630         return s2c(getAndBitwiseOrShortAcquire(o, offset, c2s(mask)));
3631     }
3632 
3633     @ForceInline
3634     public final char getAndBitwiseAndChar(Object o, long offset, char mask) {
3635         return s2c(getAndBitwiseAndShort(o, offset, c2s(mask)));
3636     }
3637 
3638     @ForceInline
3639     public final char getAndBitwiseAndCharRelease(Object o, long offset, char mask) {
3640         return s2c(getAndBitwiseAndShortRelease(o, offset, c2s(mask)));
3641     }
3642 
3643     @ForceInline
3644     public final char getAndBitwiseAndCharAcquire(Object o, long offset, char mask) {
3645         return s2c(getAndBitwiseAndShortAcquire(o, offset, c2s(mask)));
3646     }
3647 
3648     @ForceInline
3649     public final char getAndBitwiseXorChar(Object o, long offset, char mask) {
3650         return s2c(getAndBitwiseXorShort(o, offset, c2s(mask)));
3651     }
3652 
3653     @ForceInline
3654     public final char getAndBitwiseXorCharRelease(Object o, long offset, char mask) {
3655         return s2c(getAndBitwiseXorShortRelease(o, offset, c2s(mask)));
3656     }
3657 
3658     @ForceInline
3659     public final char getAndBitwiseXorCharAcquire(Object o, long offset, char mask) {
3660         return s2c(getAndBitwiseXorShortAcquire(o, offset, c2s(mask)));
3661     }
3662 
3663 
3664     @ForceInline
3665     public final short getAndBitwiseOrShort(Object o, long offset, short mask) {
3666         short current;
3667         do {
3668             current = getShortVolatile(o, offset);
3669         } while (!weakCompareAndSetShort(o, offset,
3670                                                 current, (short) (current | mask)));
3671         return current;
3672     }
3673 
3674     @ForceInline
3675     public final short getAndBitwiseOrShortRelease(Object o, long offset, short mask) {
3676         short current;
3677         do {
3678             current = getShort(o, offset);
3679         } while (!weakCompareAndSetShortRelease(o, offset,
3680                                                current, (short) (current | mask)));
3681         return current;
3682     }
3683 
3684     @ForceInline
3685     public final short getAndBitwiseOrShortAcquire(Object o, long offset, short mask) {
3686         short current;
3687         do {
3688             // Plain read, the value is a hint, the acquire CAS does the work
3689             current = getShort(o, offset);
3690         } while (!weakCompareAndSetShortAcquire(o, offset,
3691                                                current, (short) (current | mask)));
3692         return current;
3693     }
3694 
3695     @ForceInline
3696     public final short getAndBitwiseAndShort(Object o, long offset, short mask) {
3697         short current;
3698         do {
3699             current = getShortVolatile(o, offset);
3700         } while (!weakCompareAndSetShort(o, offset,
3701                                                 current, (short) (current & mask)));
3702         return current;
3703     }
3704 
3705     @ForceInline
3706     public final short getAndBitwiseAndShortRelease(Object o, long offset, short mask) {
3707         short current;
3708         do {
3709             current = getShort(o, offset);
3710         } while (!weakCompareAndSetShortRelease(o, offset,
3711                                                current, (short) (current & mask)));
3712         return current;
3713     }
3714 
3715     @ForceInline
3716     public final short getAndBitwiseAndShortAcquire(Object o, long offset, short mask) {
3717         short current;
3718         do {
3719             // Plain read, the value is a hint, the acquire CAS does the work
3720             current = getShort(o, offset);
3721         } while (!weakCompareAndSetShortAcquire(o, offset,
3722                                                current, (short) (current & mask)));
3723         return current;
3724     }
3725 
3726     @ForceInline
3727     public final short getAndBitwiseXorShort(Object o, long offset, short mask) {
3728         short current;
3729         do {
3730             current = getShortVolatile(o, offset);
3731         } while (!weakCompareAndSetShort(o, offset,
3732                                                 current, (short) (current ^ mask)));
3733         return current;
3734     }
3735 
3736     @ForceInline
3737     public final short getAndBitwiseXorShortRelease(Object o, long offset, short mask) {
3738         short current;
3739         do {
3740             current = getShort(o, offset);
3741         } while (!weakCompareAndSetShortRelease(o, offset,
3742                                                current, (short) (current ^ mask)));
3743         return current;
3744     }
3745 
3746     @ForceInline
3747     public final short getAndBitwiseXorShortAcquire(Object o, long offset, short mask) {
3748         short current;
3749         do {
3750             // Plain read, the value is a hint, the acquire CAS does the work
3751             current = getShort(o, offset);
3752         } while (!weakCompareAndSetShortAcquire(o, offset,
3753                                                current, (short) (current ^ mask)));
3754         return current;
3755     }
3756 
3757 
3758     @ForceInline
3759     public final int getAndBitwiseOrInt(Object o, long offset, int mask) {
3760         int current;
3761         do {
3762             current = getIntVolatile(o, offset);
3763         } while (!weakCompareAndSetInt(o, offset,
3764                                                 current, current | mask));
3765         return current;
3766     }
3767 
3768     @ForceInline
3769     public final int getAndBitwiseOrIntRelease(Object o, long offset, int mask) {
3770         int current;
3771         do {
3772             current = getInt(o, offset);
3773         } while (!weakCompareAndSetIntRelease(o, offset,
3774                                                current, current | mask));
3775         return current;
3776     }
3777 
3778     @ForceInline
3779     public final int getAndBitwiseOrIntAcquire(Object o, long offset, int mask) {
3780         int current;
3781         do {
3782             // Plain read, the value is a hint, the acquire CAS does the work
3783             current = getInt(o, offset);
3784         } while (!weakCompareAndSetIntAcquire(o, offset,
3785                                                current, current | mask));
3786         return current;
3787     }
3788 
3789     /**
3790      * Atomically replaces the current value of a field or array element within
3791      * the given object with the result of bitwise AND between the current value
3792      * and mask.
3793      *
3794      * @param o object/array to update the field/element in
3795      * @param offset field/element offset
3796      * @param mask the mask value
3797      * @return the previous value
3798      * @since 9
3799      */
3800     @ForceInline
3801     public final int getAndBitwiseAndInt(Object o, long offset, int mask) {
3802         int current;
3803         do {
3804             current = getIntVolatile(o, offset);
3805         } while (!weakCompareAndSetInt(o, offset,
3806                                                 current, current & mask));
3807         return current;
3808     }
3809 
3810     @ForceInline
3811     public final int getAndBitwiseAndIntRelease(Object o, long offset, int mask) {
3812         int current;
3813         do {
3814             current = getInt(o, offset);
3815         } while (!weakCompareAndSetIntRelease(o, offset,
3816                                                current, current & mask));
3817         return current;
3818     }
3819 
3820     @ForceInline
3821     public final int getAndBitwiseAndIntAcquire(Object o, long offset, int mask) {
3822         int current;
3823         do {
3824             // Plain read, the value is a hint, the acquire CAS does the work
3825             current = getInt(o, offset);
3826         } while (!weakCompareAndSetIntAcquire(o, offset,
3827                                                current, current & mask));
3828         return current;
3829     }
3830 
3831     @ForceInline
3832     public final int getAndBitwiseXorInt(Object o, long offset, int mask) {
3833         int current;
3834         do {
3835             current = getIntVolatile(o, offset);
3836         } while (!weakCompareAndSetInt(o, offset,
3837                                                 current, current ^ mask));
3838         return current;
3839     }
3840 
3841     @ForceInline
3842     public final int getAndBitwiseXorIntRelease(Object o, long offset, int mask) {
3843         int current;
3844         do {
3845             current = getInt(o, offset);
3846         } while (!weakCompareAndSetIntRelease(o, offset,
3847                                                current, current ^ mask));
3848         return current;
3849     }
3850 
3851     @ForceInline
3852     public final int getAndBitwiseXorIntAcquire(Object o, long offset, int mask) {
3853         int current;
3854         do {
3855             // Plain read, the value is a hint, the acquire CAS does the work
3856             current = getInt(o, offset);
3857         } while (!weakCompareAndSetIntAcquire(o, offset,
3858                                                current, current ^ mask));
3859         return current;
3860     }
3861 
3862 
3863     @ForceInline
3864     public final long getAndBitwiseOrLong(Object o, long offset, long mask) {
3865         long current;
3866         do {
3867             current = getLongVolatile(o, offset);
3868         } while (!weakCompareAndSetLong(o, offset,
3869                                                 current, current | mask));
3870         return current;
3871     }
3872 
3873     @ForceInline
3874     public final long getAndBitwiseOrLongRelease(Object o, long offset, long mask) {
3875         long current;
3876         do {
3877             current = getLong(o, offset);
3878         } while (!weakCompareAndSetLongRelease(o, offset,
3879                                                current, current | mask));
3880         return current;
3881     }
3882 
3883     @ForceInline
3884     public final long getAndBitwiseOrLongAcquire(Object o, long offset, long mask) {
3885         long current;
3886         do {
3887             // Plain read, the value is a hint, the acquire CAS does the work
3888             current = getLong(o, offset);
3889         } while (!weakCompareAndSetLongAcquire(o, offset,
3890                                                current, current | mask));
3891         return current;
3892     }
3893 
3894     @ForceInline
3895     public final long getAndBitwiseAndLong(Object o, long offset, long mask) {
3896         long current;
3897         do {
3898             current = getLongVolatile(o, offset);
3899         } while (!weakCompareAndSetLong(o, offset,
3900                                                 current, current & mask));
3901         return current;
3902     }
3903 
3904     @ForceInline
3905     public final long getAndBitwiseAndLongRelease(Object o, long offset, long mask) {
3906         long current;
3907         do {
3908             current = getLong(o, offset);
3909         } while (!weakCompareAndSetLongRelease(o, offset,
3910                                                current, current & mask));
3911         return current;
3912     }
3913 
3914     @ForceInline
3915     public final long getAndBitwiseAndLongAcquire(Object o, long offset, long mask) {
3916         long current;
3917         do {
3918             // Plain read, the value is a hint, the acquire CAS does the work
3919             current = getLong(o, offset);
3920         } while (!weakCompareAndSetLongAcquire(o, offset,
3921                                                current, current & mask));
3922         return current;
3923     }
3924 
3925     @ForceInline
3926     public final long getAndBitwiseXorLong(Object o, long offset, long mask) {
3927         long current;
3928         do {
3929             current = getLongVolatile(o, offset);
3930         } while (!weakCompareAndSetLong(o, offset,
3931                                                 current, current ^ mask));
3932         return current;
3933     }
3934 
3935     @ForceInline
3936     public final long getAndBitwiseXorLongRelease(Object o, long offset, long mask) {
3937         long current;
3938         do {
3939             current = getLong(o, offset);
3940         } while (!weakCompareAndSetLongRelease(o, offset,
3941                                                current, current ^ mask));
3942         return current;
3943     }
3944 
3945     @ForceInline
3946     public final long getAndBitwiseXorLongAcquire(Object o, long offset, long mask) {
3947         long current;
3948         do {
3949             // Plain read, the value is a hint, the acquire CAS does the work
3950             current = getLong(o, offset);
3951         } while (!weakCompareAndSetLongAcquire(o, offset,
3952                                                current, current ^ mask));
3953         return current;
3954     }
3955 
3956 
3957 
3958     /**
3959      * Ensures that loads before the fence will not be reordered with loads and
3960      * stores after the fence; a "LoadLoad plus LoadStore barrier".
3961      *
3962      * Corresponds to C11 atomic_thread_fence(memory_order_acquire)
3963      * (an "acquire fence").
3964      *
3965      * Provides a LoadLoad barrier followed by a LoadStore barrier.
3966      *
3967      * @since 1.8
3968      */
3969     @IntrinsicCandidate
3970     public final void loadFence() {
3971         // If loadFence intrinsic is not available, fall back to full fence.
3972         fullFence();
3973     }
3974 
3975     /**
3976      * Ensures that loads and stores before the fence will not be reordered with
3977      * stores after the fence; a "StoreStore plus LoadStore barrier".
3978      *
3979      * Corresponds to C11 atomic_thread_fence(memory_order_release)
3980      * (a "release fence").
3981      *
3982      * Provides a StoreStore barrier followed by a LoadStore barrier.
3983      *
3984      * @since 1.8
3985      */
3986     @IntrinsicCandidate
3987     public final void storeFence() {
3988         // If storeFence intrinsic is not available, fall back to full fence.
3989         fullFence();
3990     }
3991 
3992     /**
3993      * Ensures that loads and stores before the fence will not be reordered
3994      * with loads and stores after the fence.  Implies the effects of both
3995      * loadFence() and storeFence(), and in addition, the effect of a StoreLoad
3996      * barrier.
3997      *
3998      * Corresponds to C11 atomic_thread_fence(memory_order_seq_cst).
3999      * @since 1.8
4000      */
4001     @IntrinsicCandidate
4002     public native void fullFence();
4003 
4004     /**
4005      * Ensures that loads before the fence will not be reordered with
4006      * loads after the fence.
4007      *
4008      * @implNote
4009      * This method is operationally equivalent to {@link #loadFence()}.
4010      *
4011      * @since 9
4012      */
4013     public final void loadLoadFence() {
4014         loadFence();
4015     }
4016 
4017     /**
4018      * Ensures that stores before the fence will not be reordered with
4019      * stores after the fence.
4020      *
4021      * @since 9
4022      */
4023     @IntrinsicCandidate
4024     public final void storeStoreFence() {
4025         // If storeStoreFence intrinsic is not available, fall back to storeFence.
4026         storeFence();
4027     }
4028 
4029     /**
4030      * Throws IllegalAccessError; for use by the VM for access control
4031      * error support.
4032      * @since 1.8
4033      */
4034     private static void throwIllegalAccessError() {
4035         throw new IllegalAccessError();
4036     }
4037 
4038     /**
4039      * Throws NoSuchMethodError; for use by the VM for redefinition support.
4040      * @since 13
4041      */
4042     private static void throwNoSuchMethodError() {
4043         throw new NoSuchMethodError();
4044     }
4045 
4046     /**
4047      * @return Returns true if the native byte ordering of this
4048      * platform is big-endian, false if it is little-endian.
4049      */
4050     public final boolean isBigEndian() { return BIG_ENDIAN; }
4051 
4052     /**
4053      * @return Returns true if this platform is capable of performing
4054      * accesses at addresses which are not aligned for the type of the
4055      * primitive type being accessed, false otherwise.
4056      */
4057     public final boolean unalignedAccess() { return UNALIGNED_ACCESS; }
4058 
4059     /**
4060      * Fetches a value at some byte offset into a given Java object.
4061      * More specifically, fetches a value within the given object
4062      * <code>o</code> at the given offset, or (if <code>o</code> is
4063      * null) from the memory address whose numerical value is the
4064      * given offset.  <p>
4065      *
4066      * The specification of this method is the same as {@link
4067      * #getLong(Object, long)} except that the offset does not need to
4068      * have been obtained from {@link #objectFieldOffset} on the
4069      * {@link java.lang.reflect.Field} of some Java field.  The value
4070      * in memory is raw data, and need not correspond to any Java
4071      * variable.  Unless <code>o</code> is null, the value accessed
4072      * must be entirely within the allocated object.  The endianness
4073      * of the value in memory is the endianness of the native platform.
4074      *
4075      * <p> The read will be atomic with respect to the largest power
4076      * of two that divides the GCD of the offset and the storage size.
4077      * For example, getLongUnaligned will make atomic reads of 2-, 4-,
4078      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
4079      * respectively.  There are no other guarantees of atomicity.
4080      * <p>
4081      * 8-byte atomicity is only guaranteed on platforms on which
4082      * support atomic accesses to longs.
4083      *
4084      * @param o Java heap object in which the value resides, if any, else
4085      *        null
4086      * @param offset The offset in bytes from the start of the object
4087      * @return the value fetched from the indicated object
4088      * @throws RuntimeException No defined exceptions are thrown, not even
4089      *         {@link NullPointerException}
4090      * @since 9
4091      */
4092     @IntrinsicCandidate
4093     public final long getLongUnaligned(Object o, long offset) {
4094         if ((offset & 7) == 0) {
4095             return getLong(o, offset);
4096         } else if ((offset & 3) == 0) {
4097             return makeLong(getInt(o, offset),
4098                             getInt(o, offset + 4));
4099         } else if ((offset & 1) == 0) {
4100             return makeLong(getShort(o, offset),
4101                             getShort(o, offset + 2),
4102                             getShort(o, offset + 4),
4103                             getShort(o, offset + 6));
4104         } else {
4105             return makeLong(getByte(o, offset),
4106                             getByte(o, offset + 1),
4107                             getByte(o, offset + 2),
4108                             getByte(o, offset + 3),
4109                             getByte(o, offset + 4),
4110                             getByte(o, offset + 5),
4111                             getByte(o, offset + 6),
4112                             getByte(o, offset + 7));
4113         }
4114     }
4115     /**
4116      * As {@link #getLongUnaligned(Object, long)} but with an
4117      * additional argument which specifies the endianness of the value
4118      * as stored in memory.
4119      *
4120      * @param o Java heap object in which the variable resides
4121      * @param offset The offset in bytes from the start of the object
4122      * @param bigEndian The endianness of the value
4123      * @return the value fetched from the indicated object
4124      * @since 9
4125      */
4126     public final long getLongUnaligned(Object o, long offset, boolean bigEndian) {
4127         return convEndian(bigEndian, getLongUnaligned(o, offset));
4128     }
4129 
4130     /** @see #getLongUnaligned(Object, long) */
4131     @IntrinsicCandidate
4132     public final int getIntUnaligned(Object o, long offset) {
4133         if ((offset & 3) == 0) {
4134             return getInt(o, offset);
4135         } else if ((offset & 1) == 0) {
4136             return makeInt(getShort(o, offset),
4137                            getShort(o, offset + 2));
4138         } else {
4139             return makeInt(getByte(o, offset),
4140                            getByte(o, offset + 1),
4141                            getByte(o, offset + 2),
4142                            getByte(o, offset + 3));
4143         }
4144     }
4145     /** @see #getLongUnaligned(Object, long, boolean) */
4146     public final int getIntUnaligned(Object o, long offset, boolean bigEndian) {
4147         return convEndian(bigEndian, getIntUnaligned(o, offset));
4148     }
4149 
4150     /** @see #getLongUnaligned(Object, long) */
4151     @IntrinsicCandidate
4152     public final short getShortUnaligned(Object o, long offset) {
4153         if ((offset & 1) == 0) {
4154             return getShort(o, offset);
4155         } else {
4156             return makeShort(getByte(o, offset),
4157                              getByte(o, offset + 1));
4158         }
4159     }
4160     /** @see #getLongUnaligned(Object, long, boolean) */
4161     public final short getShortUnaligned(Object o, long offset, boolean bigEndian) {
4162         return convEndian(bigEndian, getShortUnaligned(o, offset));
4163     }
4164 
4165     /** @see #getLongUnaligned(Object, long) */
4166     @IntrinsicCandidate
4167     public final char getCharUnaligned(Object o, long offset) {
4168         if ((offset & 1) == 0) {
4169             return getChar(o, offset);
4170         } else {
4171             return (char)makeShort(getByte(o, offset),
4172                                    getByte(o, offset + 1));
4173         }
4174     }
4175 
4176     /** @see #getLongUnaligned(Object, long, boolean) */
4177     public final char getCharUnaligned(Object o, long offset, boolean bigEndian) {
4178         return convEndian(bigEndian, getCharUnaligned(o, offset));
4179     }
4180 
4181     /**
4182      * Stores a value at some byte offset into a given Java object.
4183      * <p>
4184      * The specification of this method is the same as {@link
4185      * #getLong(Object, long)} except that the offset does not need to
4186      * have been obtained from {@link #objectFieldOffset} on the
4187      * {@link java.lang.reflect.Field} of some Java field.  The value
4188      * in memory is raw data, and need not correspond to any Java
4189      * variable.  The endianness of the value in memory is the
4190      * endianness of the native platform.
4191      * <p>
4192      * The write will be atomic with respect to the largest power of
4193      * two that divides the GCD of the offset and the storage size.
4194      * For example, putLongUnaligned will make atomic writes of 2-, 4-,
4195      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
4196      * respectively.  There are no other guarantees of atomicity.
4197      * <p>
4198      * 8-byte atomicity is only guaranteed on platforms on which
4199      * support atomic accesses to longs.
4200      *
4201      * @param o Java heap object in which the value resides, if any, else
4202      *        null
4203      * @param offset The offset in bytes from the start of the object
4204      * @param x the value to store
4205      * @throws RuntimeException No defined exceptions are thrown, not even
4206      *         {@link NullPointerException}
4207      * @since 9
4208      */
4209     @IntrinsicCandidate
4210     public final void putLongUnaligned(Object o, long offset, long x) {
4211         if ((offset & 7) == 0) {
4212             putLong(o, offset, x);
4213         } else if ((offset & 3) == 0) {
4214             putLongParts(o, offset,
4215                          (int)(x >> 0),
4216                          (int)(x >>> 32));
4217         } else if ((offset & 1) == 0) {
4218             putLongParts(o, offset,
4219                          (short)(x >>> 0),
4220                          (short)(x >>> 16),
4221                          (short)(x >>> 32),
4222                          (short)(x >>> 48));
4223         } else {
4224             putLongParts(o, offset,
4225                          (byte)(x >>> 0),
4226                          (byte)(x >>> 8),
4227                          (byte)(x >>> 16),
4228                          (byte)(x >>> 24),
4229                          (byte)(x >>> 32),
4230                          (byte)(x >>> 40),
4231                          (byte)(x >>> 48),
4232                          (byte)(x >>> 56));
4233         }
4234     }
4235 
4236     /**
4237      * As {@link #putLongUnaligned(Object, long, long)} but with an additional
4238      * argument which specifies the endianness of the value as stored in memory.
4239      * @param o Java heap object in which the value resides
4240      * @param offset The offset in bytes from the start of the object
4241      * @param x the value to store
4242      * @param bigEndian The endianness of the value
4243      * @throws RuntimeException No defined exceptions are thrown, not even
4244      *         {@link NullPointerException}
4245      * @since 9
4246      */
4247     public final void putLongUnaligned(Object o, long offset, long x, boolean bigEndian) {
4248         putLongUnaligned(o, offset, convEndian(bigEndian, x));
4249     }
4250 
4251     /** @see #putLongUnaligned(Object, long, long) */
4252     @IntrinsicCandidate
4253     public final void putIntUnaligned(Object o, long offset, int x) {
4254         if ((offset & 3) == 0) {
4255             putInt(o, offset, x);
4256         } else if ((offset & 1) == 0) {
4257             putIntParts(o, offset,
4258                         (short)(x >> 0),
4259                         (short)(x >>> 16));
4260         } else {
4261             putIntParts(o, offset,
4262                         (byte)(x >>> 0),
4263                         (byte)(x >>> 8),
4264                         (byte)(x >>> 16),
4265                         (byte)(x >>> 24));
4266         }
4267     }
4268     /** @see #putLongUnaligned(Object, long, long, boolean) */
4269     public final void putIntUnaligned(Object o, long offset, int x, boolean bigEndian) {
4270         putIntUnaligned(o, offset, convEndian(bigEndian, x));
4271     }
4272 
4273     /** @see #putLongUnaligned(Object, long, long) */
4274     @IntrinsicCandidate
4275     public final void putShortUnaligned(Object o, long offset, short x) {
4276         if ((offset & 1) == 0) {
4277             putShort(o, offset, x);
4278         } else {
4279             putShortParts(o, offset,
4280                           (byte)(x >>> 0),
4281                           (byte)(x >>> 8));
4282         }
4283     }
4284     /** @see #putLongUnaligned(Object, long, long, boolean) */
4285     public final void putShortUnaligned(Object o, long offset, short x, boolean bigEndian) {
4286         putShortUnaligned(o, offset, convEndian(bigEndian, x));
4287     }
4288 
4289     /** @see #putLongUnaligned(Object, long, long) */
4290     @IntrinsicCandidate
4291     public final void putCharUnaligned(Object o, long offset, char x) {
4292         putShortUnaligned(o, offset, (short)x);
4293     }
4294     /** @see #putLongUnaligned(Object, long, long, boolean) */
4295     public final void putCharUnaligned(Object o, long offset, char x, boolean bigEndian) {
4296         putCharUnaligned(o, offset, convEndian(bigEndian, x));
4297     }
4298 
4299     private static int pickPos(int top, int pos) { return BIG_ENDIAN ? top - pos : pos; }
4300 
4301     // These methods construct integers from bytes.  The byte ordering
4302     // is the native endianness of this platform.
4303     private static long makeLong(byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
4304         return ((toUnsignedLong(i0) << pickPos(56, 0))
4305               | (toUnsignedLong(i1) << pickPos(56, 8))
4306               | (toUnsignedLong(i2) << pickPos(56, 16))
4307               | (toUnsignedLong(i3) << pickPos(56, 24))
4308               | (toUnsignedLong(i4) << pickPos(56, 32))
4309               | (toUnsignedLong(i5) << pickPos(56, 40))
4310               | (toUnsignedLong(i6) << pickPos(56, 48))
4311               | (toUnsignedLong(i7) << pickPos(56, 56)));
4312     }
4313     private static long makeLong(short i0, short i1, short i2, short i3) {
4314         return ((toUnsignedLong(i0) << pickPos(48, 0))
4315               | (toUnsignedLong(i1) << pickPos(48, 16))
4316               | (toUnsignedLong(i2) << pickPos(48, 32))
4317               | (toUnsignedLong(i3) << pickPos(48, 48)));
4318     }
4319     private static long makeLong(int i0, int i1) {
4320         return (toUnsignedLong(i0) << pickPos(32, 0))
4321              | (toUnsignedLong(i1) << pickPos(32, 32));
4322     }
4323     private static int makeInt(short i0, short i1) {
4324         return (toUnsignedInt(i0) << pickPos(16, 0))
4325              | (toUnsignedInt(i1) << pickPos(16, 16));
4326     }
4327     private static int makeInt(byte i0, byte i1, byte i2, byte i3) {
4328         return ((toUnsignedInt(i0) << pickPos(24, 0))
4329               | (toUnsignedInt(i1) << pickPos(24, 8))
4330               | (toUnsignedInt(i2) << pickPos(24, 16))
4331               | (toUnsignedInt(i3) << pickPos(24, 24)));
4332     }
4333     private static short makeShort(byte i0, byte i1) {
4334         return (short)((toUnsignedInt(i0) << pickPos(8, 0))
4335                      | (toUnsignedInt(i1) << pickPos(8, 8)));
4336     }
4337 
4338     private static byte  pick(byte  le, byte  be) { return BIG_ENDIAN ? be : le; }
4339     private static short pick(short le, short be) { return BIG_ENDIAN ? be : le; }
4340     private static int   pick(int   le, int   be) { return BIG_ENDIAN ? be : le; }
4341 
4342     // These methods write integers to memory from smaller parts
4343     // provided by their caller.  The ordering in which these parts
4344     // are written is the native endianness of this platform.
4345     private void putLongParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
4346         putByte(o, offset + 0, pick(i0, i7));
4347         putByte(o, offset + 1, pick(i1, i6));
4348         putByte(o, offset + 2, pick(i2, i5));
4349         putByte(o, offset + 3, pick(i3, i4));
4350         putByte(o, offset + 4, pick(i4, i3));
4351         putByte(o, offset + 5, pick(i5, i2));
4352         putByte(o, offset + 6, pick(i6, i1));
4353         putByte(o, offset + 7, pick(i7, i0));
4354     }
4355     private void putLongParts(Object o, long offset, short i0, short i1, short i2, short i3) {
4356         putShort(o, offset + 0, pick(i0, i3));
4357         putShort(o, offset + 2, pick(i1, i2));
4358         putShort(o, offset + 4, pick(i2, i1));
4359         putShort(o, offset + 6, pick(i3, i0));
4360     }
4361     private void putLongParts(Object o, long offset, int i0, int i1) {
4362         putInt(o, offset + 0, pick(i0, i1));
4363         putInt(o, offset + 4, pick(i1, i0));
4364     }
4365     private void putIntParts(Object o, long offset, short i0, short i1) {
4366         putShort(o, offset + 0, pick(i0, i1));
4367         putShort(o, offset + 2, pick(i1, i0));
4368     }
4369     private void putIntParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3) {
4370         putByte(o, offset + 0, pick(i0, i3));
4371         putByte(o, offset + 1, pick(i1, i2));
4372         putByte(o, offset + 2, pick(i2, i1));
4373         putByte(o, offset + 3, pick(i3, i0));
4374     }
4375     private void putShortParts(Object o, long offset, byte i0, byte i1) {
4376         putByte(o, offset + 0, pick(i0, i1));
4377         putByte(o, offset + 1, pick(i1, i0));
4378     }
4379 
4380     // Zero-extend an integer
4381     private static int toUnsignedInt(byte n)    { return n & 0xff; }
4382     private static int toUnsignedInt(short n)   { return n & 0xffff; }
4383     private static long toUnsignedLong(byte n)  { return n & 0xffl; }
4384     private static long toUnsignedLong(short n) { return n & 0xffffl; }
4385     private static long toUnsignedLong(int n)   { return n & 0xffffffffl; }
4386 
4387     // Maybe byte-reverse an integer
4388     private static char convEndian(boolean big, char n)   { return big == BIG_ENDIAN ? n : Character.reverseBytes(n); }
4389     private static short convEndian(boolean big, short n) { return big == BIG_ENDIAN ? n : Short.reverseBytes(n)    ; }
4390     private static int convEndian(boolean big, int n)     { return big == BIG_ENDIAN ? n : Integer.reverseBytes(n)  ; }
4391     private static long convEndian(boolean big, long n)   { return big == BIG_ENDIAN ? n : Long.reverseBytes(n)     ; }
4392 
4393 
4394 
4395     private native long allocateMemory0(long bytes);
4396     private native long reallocateMemory0(long address, long bytes);
4397     private native void freeMemory0(long address);
4398     @IntrinsicCandidate
4399     private native void setMemory0(Object o, long offset, long bytes, byte value);
4400     @IntrinsicCandidate
4401     private native void copyMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
4402     private native void copySwapMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes, long elemSize);
4403     private native long objectFieldOffset0(Field f);
4404     private native long objectFieldOffset1(Class<?> c, String name);
4405     private native long staticFieldOffset0(Field f);
4406     private native Object staticFieldBase0(Field f);
4407     private native boolean shouldBeInitialized0(Class<?> c);
4408     private native void ensureClassInitialized0(Class<?> c);
4409     private native void notifyStrictStaticAccess0(Class<?> c, long staticFieldOffset, boolean writing);
4410     private native int arrayBaseOffset0(Class<?> arrayClass); // public version returns long to promote correct arithmetic
4411     private native int arrayBaseOffset1(Object[] array);
4412     private native int arrayIndexScale0(Class<?> arrayClass);
4413     private native int arrayIndexScale1(Object[] array);
4414     private native long getObjectSize0(Object o);
4415     private native int getLoadAverage0(double[] loadavg, int nelems);
4416 
4417 
4418     /**
4419      * Invokes the given direct byte buffer's cleaner, if any.
4420      *
4421      * @param directBuffer a direct byte buffer
4422      * @throws NullPointerException     if {@code directBuffer} is null
4423      * @throws IllegalArgumentException if {@code directBuffer} is non-direct,
4424      *                                  or is a {@link java.nio.Buffer#slice slice}, or is a
4425      *                                  {@link java.nio.Buffer#duplicate duplicate}
4426      */
4427     public void invokeCleaner(java.nio.ByteBuffer directBuffer) {
4428         if (!directBuffer.isDirect())
4429             throw new IllegalArgumentException("buffer is non-direct");
4430 
4431         DirectBuffer db = (DirectBuffer) directBuffer;
4432         if (db.attachment() != null)
4433             throw new IllegalArgumentException("duplicate or slice");
4434 
4435         Cleaner cleaner = db.cleaner();
4436         if (cleaner != null) {
4437             cleaner.clean();
4438         }
4439     }
4440 }