1 /*
   2  * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.internal.misc;
  27 
  28 import jdk.internal.ref.Cleaner;
  29 import jdk.internal.vm.annotation.ForceInline;
  30 import jdk.internal.vm.annotation.IntrinsicCandidate;
  31 import sun.nio.ch.DirectBuffer;
  32 
  33 import java.lang.reflect.Field;
  34 import java.security.ProtectionDomain;
  35 
  36 import static jdk.internal.misc.UnsafeConstants.*;
  37 
  38 /**
  39  * A collection of methods for performing low-level, unsafe operations.
  40  * Although the class and all methods are public, use of this class is
  41  * limited because only trusted code can obtain instances of it.
  42  *
  43  * <em>Note:</em> It is the responsibility of the caller to make sure
  44  * arguments are checked before methods of this class are
  45  * called. While some rudimentary checks are performed on the input,
  46  * the checks are best effort and when performance is an overriding
  47  * priority, as when methods of this class are optimized by the
  48  * runtime compiler, some or all checks (if any) may be elided. Hence,
  49  * the caller must not rely on the checks and corresponding
  50  * exceptions!
  51  *
  52  * @author John R. Rose
  53  * @see #getUnsafe
  54  */
  55 
  56 public final class Unsafe {
  57 
  58     private static native void registerNatives();
  59     static {
  60         runtimeSetup();
  61     }
  62 
  63     // Called from JVM when loading an AOT cache
  64     private static void runtimeSetup() {
  65         registerNatives();
  66     }
  67 
  68     private Unsafe() {}
  69 
  70     private static final Unsafe theUnsafe = new Unsafe();
  71 
  72     /**
  73      * Provides the caller with the capability of performing unsafe
  74      * operations.
  75      *
  76      * <p>The returned {@code Unsafe} object should be carefully guarded
  77      * by the caller, since it can be used to read and write data at arbitrary
  78      * memory addresses.  It must never be passed to untrusted code.
  79      *
  80      * <p>Most methods in this class are very low-level, and correspond to a
  81      * small number of hardware instructions (on typical machines).  Compilers
  82      * are encouraged to optimize these methods accordingly.
  83      *
  84      * <p>Here is a suggested idiom for using unsafe operations:
  85      *
  86      * <pre> {@code
  87      * class MyTrustedClass {
  88      *   private static final Unsafe unsafe = Unsafe.getUnsafe();
  89      *   ...
  90      *   private long myCountAddress = ...;
  91      *   public int getCount() { return unsafe.getByte(myCountAddress); }
  92      * }}</pre>
  93      *
  94      * (It may assist compilers to make the local variable {@code final}.)
  95      */
  96     public static Unsafe getUnsafe() {
  97         return theUnsafe;
  98     }
  99 
 100     //--- peek and poke operations
 101     // (compilers should optimize these to memory ops)
 102 
 103     // These work on object fields in the Java heap.
 104     // They will not work on elements of packed arrays.
 105 
 106     /**
 107      * Fetches a value from a given Java variable.
 108      * More specifically, fetches a field or array element within the given
 109      * object {@code o} at the given offset, or (if {@code o} is null)
 110      * from the memory address whose numerical value is the given offset.
 111      * <p>
 112      * The results are undefined unless one of the following cases is true:
 113      * <ul>
 114      * <li>The offset was obtained from {@link #objectFieldOffset} on
 115      * the {@link java.lang.reflect.Field} of some Java field and the object
 116      * referred to by {@code o} is of a class compatible with that
 117      * field's class.
 118      *
 119      * <li>The offset and object reference {@code o} (either null or
 120      * non-null) were both obtained via {@link #staticFieldOffset}
 121      * and {@link #staticFieldBase} (respectively) from the
 122      * reflective {@link Field} representation of some Java field.
 123      *
 124      * <li>The object referred to by {@code o} is an array, and the offset
 125      * is an integer of the form {@code B+N*S}, where {@code N} is
 126      * a valid index into the array, and {@code B} and {@code S} are
 127      * the values obtained by {@link #arrayBaseOffset} and {@link
 128      * #arrayIndexScale} (respectively) from the array's class.  The value
 129      * referred to is the {@code N}<em>th</em> element of the array.
 130      *
 131      * </ul>
 132      * <p>
 133      * If one of the above cases is true, the call references a specific Java
 134      * variable (field or array element).  However, the results are undefined
 135      * if that variable is not in fact of the type returned by this method.
 136      * <p>
 137      * This method refers to a variable by means of two parameters, and so
 138      * it provides (in effect) a <em>double-register</em> addressing mode
 139      * for Java variables.  When the object reference is null, this method
 140      * uses its offset as an absolute address.  This is similar in operation
 141      * to methods such as {@link #getInt(long)}, which provide (in effect) a
 142      * <em>single-register</em> addressing mode for non-Java variables.
 143      * However, because Java variables may have a different layout in memory
 144      * from non-Java variables, programmers should not assume that these
 145      * two addressing modes are ever equivalent.  Also, programmers should
 146      * remember that offsets from the double-register addressing mode cannot
 147      * be portably confused with longs used in the single-register addressing
 148      * mode.
 149      *
 150      * @param o Java heap object in which the variable resides, if any, else
 151      *        null
 152      * @param offset indication of where the variable resides in a Java heap
 153      *        object, if any, else a memory address locating the variable
 154      *        statically
 155      * @return the value fetched from the indicated Java variable
 156      * @throws RuntimeException No defined exceptions are thrown, not even
 157      *         {@link NullPointerException}
 158      */
 159     @IntrinsicCandidate
 160     public native int getInt(Object o, long offset);
 161 
 162     /**
 163      * Stores a value into a given Java variable.
 164      * <p>
 165      * The first two parameters are interpreted exactly as with
 166      * {@link #getInt(Object, long)} to refer to a specific
 167      * Java variable (field or array element).  The given value
 168      * is stored into that variable.
 169      * <p>
 170      * The variable must be of the same type as the method
 171      * parameter {@code x}.
 172      *
 173      * @param o Java heap object in which the variable resides, if any, else
 174      *        null
 175      * @param offset indication of where the variable resides in a Java heap
 176      *        object, if any, else a memory address locating the variable
 177      *        statically
 178      * @param x the value to store into the indicated Java variable
 179      * @throws RuntimeException No defined exceptions are thrown, not even
 180      *         {@link NullPointerException}
 181      */
 182     @IntrinsicCandidate
 183     public native void putInt(Object o, long offset, int x);
 184 
 185 
 186     /**
 187      * Returns true if the given field is flattened.
 188      */
 189     public boolean isFlatField(Field f) {
 190         if (f == null) {
 191             throw new NullPointerException();
 192         }
 193         return isFlatField0(f);
 194     }
 195 
 196     private native boolean isFlatField0(Object o);
 197 
 198     /* Returns true if the given field has a null marker
 199      * <p>
 200      * Nullable flat fields are stored in a flattened representation
 201      * and have an associated null marker to indicate if the the field value is
 202      * null or the one stored with the flat representation
 203      */
 204 
 205     public boolean hasNullMarker(Field f) {
 206         if (f == null) {
 207             throw new NullPointerException();
 208         }
 209         return hasNullMarker0(f);
 210     }
 211 
 212     private native boolean hasNullMarker0(Object o);
 213 
 214     /* Returns the offset of the null marker of the field,
 215     * or -1 if the field doesn't have a null marker
 216     */
 217 
 218     public int nullMarkerOffset(Field f) {
 219         if (f == null) {
 220             throw new NullPointerException();
 221         }
 222         return nullMarkerOffset0(f);
 223     }
 224 
 225     private native int nullMarkerOffset0(Object o);
 226 
 227     public static final int NON_FLAT_LAYOUT = 0;
 228 
 229     /* Reports the kind of layout used for an element in the storage
 230      * allocation of the given array. Do not expect to perform any logic
 231      * or layout control with this value, it is just an opaque token
 232      * used for performance reasons.
 233      *
 234      * A layout of 0 indicates this array is not flat.
 235      */
 236     public int arrayLayout(Class<?> arrayClass) {
 237         if (arrayClass == null) {
 238             throw new NullPointerException();
 239         }
 240         return arrayLayout0(arrayClass);
 241     }
 242 
 243     private native int arrayLayout0(Object o);
 244 
 245 
 246     /* Reports the kind of layout used for a given field in the storage
 247      * allocation of its class.  Do not expect to perform any logic
 248      * or layout control with this value, it is just an opaque token
 249      * used for performance reasons.
 250      *
 251      * A layout of 0 indicates this field is not flat.
 252      */
 253     public int fieldLayout(Field f) {
 254         if (f == null) {
 255             throw new NullPointerException();
 256         }
 257         return fieldLayout0(f);
 258     }
 259 
 260     private native int fieldLayout0(Object o);
 261 
 262     /**
 263      * Returns true if the given class is a flattened array.
 264      */
 265     @IntrinsicCandidate
 266     public native boolean isFlatArray(Class<?> arrayClass);
 267 
 268     /**
 269      * Fetches a reference value from a given Java variable.
 270      * This method can return a reference to either an object or value
 271      * or a null reference.
 272      *
 273      * @see #getInt(Object, long)
 274      */
 275     @IntrinsicCandidate
 276     public native Object getReference(Object o, long offset);
 277 
 278     /**
 279      * Stores a reference value into a given Java variable.
 280      * This method can store a reference to either an object or value
 281      * or a null reference.
 282      * <p>
 283      * Unless the reference {@code x} being stored is either null
 284      * or matches the field type, the results are undefined.
 285      * If the reference {@code o} is non-null, card marks or
 286      * other store barriers for that object (if the VM requires them)
 287      * are updated.
 288      * @see #putInt(Object, long, int)
 289      */
 290     @IntrinsicCandidate
 291     public native void putReference(Object o, long offset, Object x);
 292 
 293     /**
 294      * Fetches a value of type {@code <V>} from a given Java variable.
 295      * More specifically, fetches a field or array element within the given
 296      * {@code o} object at the given offset, or (if {@code o} is null)
 297      * from the memory address whose numerical value is the given offset.
 298      *
 299      * @param o Java heap object in which the variable resides, if any, else
 300      *        null
 301      * @param offset indication of where the variable resides in a Java heap
 302      *        object, if any, else a memory address locating the variable
 303      *        statically
 304      * @param valueType value type
 305      * @param <V> the type of a value
 306      * @return the value fetched from the indicated Java variable
 307      * @throws RuntimeException No defined exceptions are thrown, not even
 308      *         {@link NullPointerException}
 309      */
 310     @IntrinsicCandidate
 311     public native <V> V getValue(Object o, long offset, Class<?> valueType);
 312 
 313     /**
 314      * Fetches a value of type {@code <V>} from a given Java variable.
 315      * More specifically, fetches a field or array element within the given
 316      * {@code o} object at the given offset, or (if {@code o} is null)
 317      * from the memory address whose numerical value is the given offset.
 318      *
 319      * @param o Java heap object in which the variable resides, if any, else
 320      *        null
 321      * @param offset indication of where the variable resides in a Java heap
 322      *        object, if any, else a memory address locating the variable
 323      *        statically
 324      * @param layoutKind opaque value used by the VM to know the layout
 325      *        the field or array element. This value must be retrieved with
 326      *        {@link #fieldLayout} or {@link #arrayLayout}.
 327      * @param valueType value type
 328      * @param <V> the type of a value
 329      * @return the value fetched from the indicated Java variable
 330      * @throws RuntimeException No defined exceptions are thrown, not even
 331      *         {@link NullPointerException}
 332      */
 333     public native <V> V getFlatValue(Object o, long offset, int layoutKind, Class<?> valueType);
 334 
 335 
 336     /**
 337      * Stores the given value into a given Java variable.
 338      *
 339      * Unless the reference {@code o} being stored is either null
 340      * or matches the field type, the results are undefined.
 341      *
 342      * @param o Java heap object in which the variable resides, if any, else
 343      *        null
 344      * @param offset indication of where the variable resides in a Java heap
 345      *        object, if any, else a memory address locating the variable
 346      *        statically
 347      * @param valueType value type
 348      * @param v the value to store into the indicated Java variable
 349      * @param <V> the type of a value
 350      * @throws RuntimeException No defined exceptions are thrown, not even
 351      *         {@link NullPointerException}
 352      */
 353     @IntrinsicCandidate
 354     public native <V> void putValue(Object o, long offset, Class<?> valueType, V v);
 355 
 356     /**
 357      * Stores the given value into a given Java variable.
 358      *
 359      * Unless the reference {@code o} being stored is either null
 360      * or matches the field type, the results are undefined.
 361      *
 362      * @param o Java heap object in which the variable resides, if any, else
 363      *        null
 364      * @param offset indication of where the variable resides in a Java heap
 365      *        object, if any, else a memory address locating the variable
 366      *        statically
 367      * @param layoutKind opaque value used by the VM to know the layout
 368      *        the field or array element. This value must be retrieved with
 369      *        {@link #fieldLayout} or {@link #arrayLayout}.
 370      * @param valueType value type
 371      * @param v the value to store into the indicated Java variable
 372      * @param <V> the type of a value
 373      * @throws RuntimeException No defined exceptions are thrown, not even
 374      *         {@link NullPointerException}
 375      */
 376     public native <V> void putFlatValue(Object o, long offset, int layoutKind, Class<?> valueType, V v);
 377 
 378     /**
 379      * Returns an object instance with a private buffered value whose layout
 380      * and contents is exactly the given value instance.  The return object
 381      * is in the larval state that can be updated using the unsafe put operation.
 382      *
 383      * @param value a value instance
 384      * @param <V> the type of the given value instance
 385      */
 386     @IntrinsicCandidate
 387     public native <V> V makePrivateBuffer(V value);
 388 
 389     /**
 390      * Exits the larval state and returns a value instance.
 391      *
 392      * @param value a value instance
 393      * @param <V> the type of the given value instance
 394      */
 395     @IntrinsicCandidate
 396     public native <V> V finishPrivateBuffer(V value);
 397 
 398     /**
 399      * Returns the header size of the given value type.
 400      *
 401      * @param valueType value type
 402      * @return the header size of the value type
 403      */
 404     public native <V> long valueHeaderSize(Class<V> valueType);
 405 
 406     /** @see #getInt(Object, long) */
 407     @IntrinsicCandidate
 408     public native boolean getBoolean(Object o, long offset);
 409 
 410     /** @see #putInt(Object, long, int) */
 411     @IntrinsicCandidate
 412     public native void    putBoolean(Object o, long offset, boolean x);
 413 
 414     /** @see #getInt(Object, long) */
 415     @IntrinsicCandidate
 416     public native byte    getByte(Object o, long offset);
 417 
 418     /** @see #putInt(Object, long, int) */
 419     @IntrinsicCandidate
 420     public native void    putByte(Object o, long offset, byte x);
 421 
 422     /** @see #getInt(Object, long) */
 423     @IntrinsicCandidate
 424     public native short   getShort(Object o, long offset);
 425 
 426     /** @see #putInt(Object, long, int) */
 427     @IntrinsicCandidate
 428     public native void    putShort(Object o, long offset, short x);
 429 
 430     /** @see #getInt(Object, long) */
 431     @IntrinsicCandidate
 432     public native char    getChar(Object o, long offset);
 433 
 434     /** @see #putInt(Object, long, int) */
 435     @IntrinsicCandidate
 436     public native void    putChar(Object o, long offset, char x);
 437 
 438     /** @see #getInt(Object, long) */
 439     @IntrinsicCandidate
 440     public native long    getLong(Object o, long offset);
 441 
 442     /** @see #putInt(Object, long, int) */
 443     @IntrinsicCandidate
 444     public native void    putLong(Object o, long offset, long x);
 445 
 446     /** @see #getInt(Object, long) */
 447     @IntrinsicCandidate
 448     public native float   getFloat(Object o, long offset);
 449 
 450     /** @see #putInt(Object, long, int) */
 451     @IntrinsicCandidate
 452     public native void    putFloat(Object o, long offset, float x);
 453 
 454     /** @see #getInt(Object, long) */
 455     @IntrinsicCandidate
 456     public native double  getDouble(Object o, long offset);
 457 
 458     /** @see #putInt(Object, long, int) */
 459     @IntrinsicCandidate
 460     public native void    putDouble(Object o, long offset, double x);
 461 
 462     /**
 463      * Fetches a native pointer from a given memory address.  If the address is
 464      * zero, or does not point into a block obtained from {@link
 465      * #allocateMemory}, the results are undefined.
 466      *
 467      * <p>If the native pointer is less than 64 bits wide, it is extended as
 468      * an unsigned number to a Java long.  The pointer may be indexed by any
 469      * given byte offset, simply by adding that offset (as a simple integer) to
 470      * the long representing the pointer.  The number of bytes actually read
 471      * from the target address may be determined by consulting {@link
 472      * #addressSize}.
 473      *
 474      * @see #allocateMemory
 475      * @see #getInt(Object, long)
 476      */
 477     @ForceInline
 478     public long getAddress(Object o, long offset) {
 479         if (ADDRESS_SIZE == 4) {
 480             return Integer.toUnsignedLong(getInt(o, offset));
 481         } else {
 482             return getLong(o, offset);
 483         }
 484     }
 485 
 486     /**
 487      * Stores a native pointer into a given memory address.  If the address is
 488      * zero, or does not point into a block obtained from {@link
 489      * #allocateMemory}, the results are undefined.
 490      *
 491      * <p>The number of bytes actually written at the target address may be
 492      * determined by consulting {@link #addressSize}.
 493      *
 494      * @see #allocateMemory
 495      * @see #putInt(Object, long, int)
 496      */
 497     @ForceInline
 498     public void putAddress(Object o, long offset, long x) {
 499         if (ADDRESS_SIZE == 4) {
 500             putInt(o, offset, (int)x);
 501         } else {
 502             putLong(o, offset, x);
 503         }
 504     }
 505 
 506     // These read VM internal data.
 507 
 508     /**
 509      * Fetches an uncompressed reference value from a given native variable
 510      * ignoring the VM's compressed references mode.
 511      *
 512      * @param address a memory address locating the variable
 513      * @return the value fetched from the indicated native variable
 514      */
 515     public native Object getUncompressedObject(long address);
 516 
 517     // These work on values in the C heap.
 518 
 519     /**
 520      * Fetches a value from a given memory address.  If the address is zero, or
 521      * does not point into a block obtained from {@link #allocateMemory}, the
 522      * results are undefined.
 523      *
 524      * @see #allocateMemory
 525      */
 526     @ForceInline
 527     public byte getByte(long address) {
 528         return getByte(null, address);
 529     }
 530 
 531     /**
 532      * Stores a value into a given memory address.  If the address is zero, or
 533      * does not point into a block obtained from {@link #allocateMemory}, the
 534      * results are undefined.
 535      *
 536      * @see #getByte(long)
 537      */
 538     @ForceInline
 539     public void putByte(long address, byte x) {
 540         putByte(null, address, x);
 541     }
 542 
 543     /** @see #getByte(long) */
 544     @ForceInline
 545     public short getShort(long address) {
 546         return getShort(null, address);
 547     }
 548 
 549     /** @see #putByte(long, byte) */
 550     @ForceInline
 551     public void putShort(long address, short x) {
 552         putShort(null, address, x);
 553     }
 554 
 555     /** @see #getByte(long) */
 556     @ForceInline
 557     public char getChar(long address) {
 558         return getChar(null, address);
 559     }
 560 
 561     /** @see #putByte(long, byte) */
 562     @ForceInline
 563     public void putChar(long address, char x) {
 564         putChar(null, address, x);
 565     }
 566 
 567     /** @see #getByte(long) */
 568     @ForceInline
 569     public int getInt(long address) {
 570         return getInt(null, address);
 571     }
 572 
 573     /** @see #putByte(long, byte) */
 574     @ForceInline
 575     public void putInt(long address, int x) {
 576         putInt(null, address, x);
 577     }
 578 
 579     /** @see #getByte(long) */
 580     @ForceInline
 581     public long getLong(long address) {
 582         return getLong(null, address);
 583     }
 584 
 585     /** @see #putByte(long, byte) */
 586     @ForceInline
 587     public void putLong(long address, long x) {
 588         putLong(null, address, x);
 589     }
 590 
 591     /** @see #getByte(long) */
 592     @ForceInline
 593     public float getFloat(long address) {
 594         return getFloat(null, address);
 595     }
 596 
 597     /** @see #putByte(long, byte) */
 598     @ForceInline
 599     public void putFloat(long address, float x) {
 600         putFloat(null, address, x);
 601     }
 602 
 603     /** @see #getByte(long) */
 604     @ForceInline
 605     public double getDouble(long address) {
 606         return getDouble(null, address);
 607     }
 608 
 609     /** @see #putByte(long, byte) */
 610     @ForceInline
 611     public void putDouble(long address, double x) {
 612         putDouble(null, address, x);
 613     }
 614 
 615     /** @see #getAddress(Object, long) */
 616     @ForceInline
 617     public long getAddress(long address) {
 618         return getAddress(null, address);
 619     }
 620 
 621     /** @see #putAddress(Object, long, long) */
 622     @ForceInline
 623     public void putAddress(long address, long x) {
 624         putAddress(null, address, x);
 625     }
 626 
 627 
 628 
 629     //--- helper methods for validating various types of objects/values
 630 
 631     /**
 632      * Create an exception reflecting that some of the input was invalid
 633      *
 634      * <em>Note:</em> It is the responsibility of the caller to make
 635      * sure arguments are checked before the methods are called. While
 636      * some rudimentary checks are performed on the input, the checks
 637      * are best effort and when performance is an overriding priority,
 638      * as when methods of this class are optimized by the runtime
 639      * compiler, some or all checks (if any) may be elided. Hence, the
 640      * caller must not rely on the checks and corresponding
 641      * exceptions!
 642      *
 643      * @return an exception object
 644      */
 645     private RuntimeException invalidInput() {
 646         return new IllegalArgumentException();
 647     }
 648 
 649     /**
 650      * Check if a value is 32-bit clean (32 MSB are all zero)
 651      *
 652      * @param value the 64-bit value to check
 653      *
 654      * @return true if the value is 32-bit clean
 655      */
 656     private boolean is32BitClean(long value) {
 657         return value >>> 32 == 0;
 658     }
 659 
 660     /**
 661      * Check the validity of a size (the equivalent of a size_t)
 662      *
 663      * @throws RuntimeException if the size is invalid
 664      *         (<em>Note:</em> after optimization, invalid inputs may
 665      *         go undetected, which will lead to unpredictable
 666      *         behavior)
 667      */
 668     private void checkSize(long size) {
 669         if (ADDRESS_SIZE == 4) {
 670             // Note: this will also check for negative sizes
 671             if (!is32BitClean(size)) {
 672                 throw invalidInput();
 673             }
 674         } else if (size < 0) {
 675             throw invalidInput();
 676         }
 677     }
 678 
 679     /**
 680      * Check the validity of a native address (the equivalent of void*)
 681      *
 682      * @throws RuntimeException if the address is invalid
 683      *         (<em>Note:</em> after optimization, invalid inputs may
 684      *         go undetected, which will lead to unpredictable
 685      *         behavior)
 686      */
 687     private void checkNativeAddress(long address) {
 688         if (ADDRESS_SIZE == 4) {
 689             // Accept both zero and sign extended pointers. A valid
 690             // pointer will, after the +1 below, either have produced
 691             // the value 0x0 or 0x1. Masking off the low bit allows
 692             // for testing against 0.
 693             if ((((address >> 32) + 1) & ~1) != 0) {
 694                 throw invalidInput();
 695             }
 696         }
 697     }
 698 
 699     /**
 700      * Check the validity of an offset, relative to a base object
 701      *
 702      * @param o the base object
 703      * @param offset the offset to check
 704      *
 705      * @throws RuntimeException if the size is invalid
 706      *         (<em>Note:</em> after optimization, invalid inputs may
 707      *         go undetected, which will lead to unpredictable
 708      *         behavior)
 709      */
 710     private void checkOffset(Object o, long offset) {
 711         if (ADDRESS_SIZE == 4) {
 712             // Note: this will also check for negative offsets
 713             if (!is32BitClean(offset)) {
 714                 throw invalidInput();
 715             }
 716         } else if (offset < 0) {
 717             throw invalidInput();
 718         }
 719     }
 720 
 721     /**
 722      * Check the validity of a double-register pointer
 723      *
 724      * Note: This code deliberately does *not* check for NPE for (at
 725      * least) three reasons:
 726      *
 727      * 1) NPE is not just NULL/0 - there is a range of values all
 728      * resulting in an NPE, which is not trivial to check for
 729      *
 730      * 2) It is the responsibility of the callers of Unsafe methods
 731      * to verify the input, so throwing an exception here is not really
 732      * useful - passing in a NULL pointer is a critical error and the
 733      * must not expect an exception to be thrown anyway.
 734      *
 735      * 3) the actual operations will detect NULL pointers anyway by
 736      * means of traps and signals (like SIGSEGV).
 737      *
 738      * @param o Java heap object, or null
 739      * @param offset indication of where the variable resides in a Java heap
 740      *        object, if any, else a memory address locating the variable
 741      *        statically
 742      *
 743      * @throws RuntimeException if the pointer is invalid
 744      *         (<em>Note:</em> after optimization, invalid inputs may
 745      *         go undetected, which will lead to unpredictable
 746      *         behavior)
 747      */
 748     private void checkPointer(Object o, long offset) {
 749         if (o == null) {
 750             checkNativeAddress(offset);
 751         } else {
 752             checkOffset(o, offset);
 753         }
 754     }
 755 
 756     /**
 757      * Check if a type is a primitive array type
 758      *
 759      * @param c the type to check
 760      *
 761      * @return true if the type is a primitive array type
 762      */
 763     private void checkPrimitiveArray(Class<?> c) {
 764         Class<?> componentType = c.getComponentType();
 765         if (componentType == null || !componentType.isPrimitive()) {
 766             throw invalidInput();
 767         }
 768     }
 769 
 770     /**
 771      * Check that a pointer is a valid primitive array type pointer
 772      *
 773      * Note: pointers off-heap are considered to be primitive arrays
 774      *
 775      * @throws RuntimeException if the pointer is invalid
 776      *         (<em>Note:</em> after optimization, invalid inputs may
 777      *         go undetected, which will lead to unpredictable
 778      *         behavior)
 779      */
 780     private void checkPrimitivePointer(Object o, long offset) {
 781         checkPointer(o, offset);
 782 
 783         if (o != null) {
 784             // If on heap, it must be a primitive array
 785             checkPrimitiveArray(o.getClass());
 786         }
 787     }
 788 
 789 
 790     //--- wrappers for malloc, realloc, free:
 791 
 792     /**
 793      * Round up allocation size to a multiple of HeapWordSize.
 794      */
 795     private long alignToHeapWordSize(long bytes) {
 796         if (bytes >= 0) {
 797             return (bytes + ADDRESS_SIZE - 1) & ~(ADDRESS_SIZE - 1);
 798         } else {
 799             throw invalidInput();
 800         }
 801     }
 802 
 803     /**
 804      * Allocates a new block of native memory, of the given size in bytes.  The
 805      * contents of the memory are uninitialized; they will generally be
 806      * garbage.  The resulting native pointer will be zero if and only if the
 807      * requested size is zero.  The resulting native pointer will be aligned for
 808      * all value types.   Dispose of this memory by calling {@link #freeMemory}
 809      * or resize it with {@link #reallocateMemory}.
 810      *
 811      * <em>Note:</em> It is the responsibility of the caller to make
 812      * sure arguments are checked before the methods are called. While
 813      * some rudimentary checks are performed on the input, the checks
 814      * are best effort and when performance is an overriding priority,
 815      * as when methods of this class are optimized by the runtime
 816      * compiler, some or all checks (if any) may be elided. Hence, the
 817      * caller must not rely on the checks and corresponding
 818      * exceptions!
 819      *
 820      * @throws RuntimeException if the size is negative or too large
 821      *         for the native size_t type
 822      *
 823      * @throws OutOfMemoryError if the allocation is refused by the system
 824      *
 825      * @see #getByte(long)
 826      * @see #putByte(long, byte)
 827      */
 828     public long allocateMemory(long bytes) {
 829         bytes = alignToHeapWordSize(bytes);
 830 
 831         allocateMemoryChecks(bytes);
 832 
 833         if (bytes == 0) {
 834             return 0;
 835         }
 836 
 837         long p = allocateMemory0(bytes);
 838         if (p == 0) {
 839             throw new OutOfMemoryError("Unable to allocate " + bytes + " bytes");
 840         }
 841 
 842         return p;
 843     }
 844 
 845     /**
 846      * Validate the arguments to allocateMemory
 847      *
 848      * @throws RuntimeException if the arguments are invalid
 849      *         (<em>Note:</em> after optimization, invalid inputs may
 850      *         go undetected, which will lead to unpredictable
 851      *         behavior)
 852      */
 853     private void allocateMemoryChecks(long bytes) {
 854         checkSize(bytes);
 855     }
 856 
 857     /**
 858      * Resizes a new block of native memory, to the given size in bytes.  The
 859      * contents of the new block past the size of the old block are
 860      * uninitialized; they will generally be garbage.  The resulting native
 861      * pointer will be zero if and only if the requested size is zero.  The
 862      * resulting native pointer will be aligned for all value types.  Dispose
 863      * of this memory by calling {@link #freeMemory}, or resize it with {@link
 864      * #reallocateMemory}.  The address passed to this method may be null, in
 865      * which case an allocation will be performed.
 866      *
 867      * <em>Note:</em> It is the responsibility of the caller to make
 868      * sure arguments are checked before the methods are called. While
 869      * some rudimentary checks are performed on the input, the checks
 870      * are best effort and when performance is an overriding priority,
 871      * as when methods of this class are optimized by the runtime
 872      * compiler, some or all checks (if any) may be elided. Hence, the
 873      * caller must not rely on the checks and corresponding
 874      * exceptions!
 875      *
 876      * @throws RuntimeException if the size is negative or too large
 877      *         for the native size_t type
 878      *
 879      * @throws OutOfMemoryError if the allocation is refused by the system
 880      *
 881      * @see #allocateMemory
 882      */
 883     public long reallocateMemory(long address, long bytes) {
 884         bytes = alignToHeapWordSize(bytes);
 885 
 886         reallocateMemoryChecks(address, bytes);
 887 
 888         if (bytes == 0) {
 889             freeMemory(address);
 890             return 0;
 891         }
 892 
 893         long p = (address == 0) ? allocateMemory0(bytes) : reallocateMemory0(address, bytes);
 894         if (p == 0) {
 895             throw new OutOfMemoryError("Unable to allocate " + bytes + " bytes");
 896         }
 897 
 898         return p;
 899     }
 900 
 901     /**
 902      * Validate the arguments to reallocateMemory
 903      *
 904      * @throws RuntimeException if the arguments are invalid
 905      *         (<em>Note:</em> after optimization, invalid inputs may
 906      *         go undetected, which will lead to unpredictable
 907      *         behavior)
 908      */
 909     private void reallocateMemoryChecks(long address, long bytes) {
 910         checkPointer(null, address);
 911         checkSize(bytes);
 912     }
 913 
 914     /**
 915      * Sets all bytes in a given block of memory to a fixed value
 916      * (usually zero).
 917      *
 918      * <p>This method determines a block's base address by means of two parameters,
 919      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 920      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 921      * the offset supplies an absolute base address.
 922      *
 923      * <p>The stores are in coherent (atomic) units of a size determined
 924      * by the address and length parameters.  If the effective address and
 925      * length are all even modulo 8, the stores take place in 'long' units.
 926      * If the effective address and length are (resp.) even modulo 4 or 2,
 927      * the stores take place in units of 'int' or 'short'.
 928      *
 929      * <em>Note:</em> It is the responsibility of the caller to make
 930      * sure arguments are checked before the methods are called. While
 931      * some rudimentary checks are performed on the input, the checks
 932      * are best effort and when performance is an overriding priority,
 933      * as when methods of this class are optimized by the runtime
 934      * compiler, some or all checks (if any) may be elided. Hence, the
 935      * caller must not rely on the checks and corresponding
 936      * exceptions!
 937      *
 938      * @throws RuntimeException if any of the arguments is invalid
 939      *
 940      * @since 1.7
 941      */
 942     public void setMemory(Object o, long offset, long bytes, byte value) {
 943         setMemoryChecks(o, offset, bytes, value);
 944 
 945         if (bytes == 0) {
 946             return;
 947         }
 948 
 949         setMemory0(o, offset, bytes, value);
 950     }
 951 
 952     /**
 953      * Sets all bytes in a given block of memory to a fixed value
 954      * (usually zero).  This provides a <em>single-register</em> addressing mode,
 955      * as discussed in {@link #getInt(Object,long)}.
 956      *
 957      * <p>Equivalent to {@code setMemory(null, address, bytes, value)}.
 958      */
 959     public void setMemory(long address, long bytes, byte value) {
 960         setMemory(null, address, bytes, value);
 961     }
 962 
 963     /**
 964      * Validate the arguments to setMemory
 965      *
 966      * @throws RuntimeException if the arguments are invalid
 967      *         (<em>Note:</em> after optimization, invalid inputs may
 968      *         go undetected, which will lead to unpredictable
 969      *         behavior)
 970      */
 971     private void setMemoryChecks(Object o, long offset, long bytes, byte value) {
 972         checkPrimitivePointer(o, offset);
 973         checkSize(bytes);
 974     }
 975 
 976     /**
 977      * Sets all bytes in a given block of memory to a copy of another
 978      * block.
 979      *
 980      * <p>This method determines each block's base address by means of two parameters,
 981      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 982      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 983      * the offset supplies an absolute base address.
 984      *
 985      * <p>The transfers are in coherent (atomic) units of a size determined
 986      * by the address and length parameters.  If the effective addresses and
 987      * length are all even modulo 8, the transfer takes place in 'long' units.
 988      * If the effective addresses and length are (resp.) even modulo 4 or 2,
 989      * the transfer takes place in units of 'int' or 'short'.
 990      *
 991      * <em>Note:</em> It is the responsibility of the caller to make
 992      * sure arguments are checked before the methods are called. While
 993      * some rudimentary checks are performed on the input, the checks
 994      * are best effort and when performance is an overriding priority,
 995      * as when methods of this class are optimized by the runtime
 996      * compiler, some or all checks (if any) may be elided. Hence, the
 997      * caller must not rely on the checks and corresponding
 998      * exceptions!
 999      *
1000      * @throws RuntimeException if any of the arguments is invalid
1001      *
1002      * @since 1.7
1003      */
1004     public void copyMemory(Object srcBase, long srcOffset,
1005                            Object destBase, long destOffset,
1006                            long bytes) {
1007         copyMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes);
1008 
1009         if (bytes == 0) {
1010             return;
1011         }
1012 
1013         copyMemory0(srcBase, srcOffset, destBase, destOffset, bytes);
1014     }
1015 
1016     /**
1017      * Sets all bytes in a given block of memory to a copy of another
1018      * block.  This provides a <em>single-register</em> addressing mode,
1019      * as discussed in {@link #getInt(Object,long)}.
1020      *
1021      * Equivalent to {@code copyMemory(null, srcAddress, null, destAddress, bytes)}.
1022      */
1023     public void copyMemory(long srcAddress, long destAddress, long bytes) {
1024         copyMemory(null, srcAddress, null, destAddress, bytes);
1025     }
1026 
1027     /**
1028      * Validate the arguments to copyMemory
1029      *
1030      * @throws RuntimeException if any of the arguments is invalid
1031      *         (<em>Note:</em> after optimization, invalid inputs may
1032      *         go undetected, which will lead to unpredictable
1033      *         behavior)
1034      */
1035     private void copyMemoryChecks(Object srcBase, long srcOffset,
1036                                   Object destBase, long destOffset,
1037                                   long bytes) {
1038         checkSize(bytes);
1039         checkPrimitivePointer(srcBase, srcOffset);
1040         checkPrimitivePointer(destBase, destOffset);
1041     }
1042 
1043     /**
1044      * Copies all elements from one block of memory to another block,
1045      * *unconditionally* byte swapping the elements on the fly.
1046      *
1047      * <p>This method determines each block's base address by means of two parameters,
1048      * and so it provides (in effect) a <em>double-register</em> addressing mode,
1049      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
1050      * the offset supplies an absolute base address.
1051      *
1052      * <em>Note:</em> It is the responsibility of the caller to make
1053      * sure arguments are checked before the methods are called. While
1054      * some rudimentary checks are performed on the input, the checks
1055      * are best effort and when performance is an overriding priority,
1056      * as when methods of this class are optimized by the runtime
1057      * compiler, some or all checks (if any) may be elided. Hence, the
1058      * caller must not rely on the checks and corresponding
1059      * exceptions!
1060      *
1061      * @throws RuntimeException if any of the arguments is invalid
1062      *
1063      * @since 9
1064      */
1065     public void copySwapMemory(Object srcBase, long srcOffset,
1066                                Object destBase, long destOffset,
1067                                long bytes, long elemSize) {
1068         copySwapMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
1069 
1070         if (bytes == 0) {
1071             return;
1072         }
1073 
1074         copySwapMemory0(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
1075     }
1076 
1077     private void copySwapMemoryChecks(Object srcBase, long srcOffset,
1078                                       Object destBase, long destOffset,
1079                                       long bytes, long elemSize) {
1080         checkSize(bytes);
1081 
1082         if (elemSize != 2 && elemSize != 4 && elemSize != 8) {
1083             throw invalidInput();
1084         }
1085         if (bytes % elemSize != 0) {
1086             throw invalidInput();
1087         }
1088 
1089         checkPrimitivePointer(srcBase, srcOffset);
1090         checkPrimitivePointer(destBase, destOffset);
1091     }
1092 
1093     /**
1094      * Copies all elements from one block of memory to another block, byte swapping the
1095      * elements on the fly.
1096      *
1097      * This provides a <em>single-register</em> addressing mode, as
1098      * discussed in {@link #getInt(Object,long)}.
1099      *
1100      * Equivalent to {@code copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize)}.
1101      */
1102     public void copySwapMemory(long srcAddress, long destAddress, long bytes, long elemSize) {
1103         copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize);
1104     }
1105 
1106     /**
1107      * Disposes of a block of native memory, as obtained from {@link
1108      * #allocateMemory} or {@link #reallocateMemory}.  The address passed to
1109      * this method may be null, in which case no action is taken.
1110      *
1111      * <em>Note:</em> It is the responsibility of the caller to make
1112      * sure arguments are checked before the methods are called. While
1113      * some rudimentary checks are performed on the input, the checks
1114      * are best effort and when performance is an overriding priority,
1115      * as when methods of this class are optimized by the runtime
1116      * compiler, some or all checks (if any) may be elided. Hence, the
1117      * caller must not rely on the checks and corresponding
1118      * exceptions!
1119      *
1120      * @throws RuntimeException if any of the arguments is invalid
1121      *
1122      * @see #allocateMemory
1123      */
1124     public void freeMemory(long address) {
1125         freeMemoryChecks(address);
1126 
1127         if (address == 0) {
1128             return;
1129         }
1130 
1131         freeMemory0(address);
1132     }
1133 
1134     /**
1135      * Validate the arguments to freeMemory
1136      *
1137      * @throws RuntimeException if the arguments are invalid
1138      *         (<em>Note:</em> after optimization, invalid inputs may
1139      *         go undetected, which will lead to unpredictable
1140      *         behavior)
1141      */
1142     private void freeMemoryChecks(long address) {
1143         checkPointer(null, address);
1144     }
1145 
1146     /**
1147      * Ensure writeback of a specified virtual memory address range
1148      * from cache to physical memory. All bytes in the address range
1149      * are guaranteed to have been written back to physical memory on
1150      * return from this call i.e. subsequently executed store
1151      * instructions are guaranteed not to be visible before the
1152      * writeback is completed.
1153      *
1154      * @param address
1155      *        the lowest byte address that must be guaranteed written
1156      *        back to memory. bytes at lower addresses may also be
1157      *        written back.
1158      *
1159      * @param length
1160      *        the length in bytes of the region starting at address
1161      *        that must be guaranteed written back to memory.
1162      *
1163      * @throws RuntimeException if memory writeback is not supported
1164      *         on the current hardware of if the arguments are invalid.
1165      *         (<em>Note:</em> after optimization, invalid inputs may
1166      *         go undetected, which will lead to unpredictable
1167      *         behavior)
1168      *
1169      * @since 14
1170      */
1171 
1172     public void writebackMemory(long address, long length) {
1173         checkWritebackEnabled();
1174         checkWritebackMemory(address, length);
1175 
1176         // perform any required pre-writeback barrier
1177         writebackPreSync0();
1178 
1179         // write back one cache line at a time
1180         long line = dataCacheLineAlignDown(address);
1181         long end = address + length;
1182         while (line < end) {
1183             writeback0(line);
1184             line += dataCacheLineFlushSize();
1185         }
1186 
1187         // perform any required post-writeback barrier
1188         writebackPostSync0();
1189     }
1190 
1191     /**
1192      * Validate the arguments to writebackMemory
1193      *
1194      * @throws RuntimeException if the arguments are invalid
1195      *         (<em>Note:</em> after optimization, invalid inputs may
1196      *         go undetected, which will lead to unpredictable
1197      *         behavior)
1198      */
1199     private void checkWritebackMemory(long address, long length) {
1200         checkNativeAddress(address);
1201         checkSize(length);
1202     }
1203 
1204     /**
1205      * Validate that the current hardware supports memory writeback.
1206      * (<em>Note:</em> this is a belt and braces check.  Clients are
1207      * expected to test whether writeback is enabled by calling
1208      * ({@link isWritebackEnabled #isWritebackEnabled} and avoid
1209      * calling method {@link writeback #writeback} if it is disabled).
1210      *
1211      *
1212      * @throws RuntimeException if memory writeback is not supported
1213      */
1214     private void checkWritebackEnabled() {
1215         if (!isWritebackEnabled()) {
1216             throw new RuntimeException("writebackMemory not enabled!");
1217         }
1218     }
1219 
1220     /**
1221      * force writeback of an individual cache line.
1222      *
1223      * @param address
1224      *        the start address of the cache line to be written back
1225      */
1226     @IntrinsicCandidate
1227     private native void writeback0(long address);
1228 
1229      /**
1230       * Serialize writeback operations relative to preceding memory writes.
1231       */
1232     @IntrinsicCandidate
1233     private native void writebackPreSync0();
1234 
1235      /**
1236       * Serialize writeback operations relative to following memory writes.
1237       */
1238     @IntrinsicCandidate
1239     private native void writebackPostSync0();
1240 
1241     //--- random queries
1242 
1243     /**
1244      * This constant differs from all results that will ever be returned from
1245      * {@link #staticFieldOffset}, {@link #objectFieldOffset},
1246      * or {@link #arrayBaseOffset}.
1247      */
1248     public static final int INVALID_FIELD_OFFSET = -1;
1249 
1250     /**
1251      * Reports the location of a given field in the storage allocation of its
1252      * class.  Do not expect to perform any sort of arithmetic on this offset;
1253      * it is just a cookie which is passed to the unsafe heap memory accessors.
1254      *
1255      * <p>Any given field will always have the same offset and base, and no
1256      * two distinct fields of the same class will ever have the same offset
1257      * and base.
1258      *
1259      * <p>As of 1.4.1, offsets for fields are represented as long values,
1260      * although the Sun JVM does not use the most significant 32 bits.
1261      * However, JVM implementations which store static fields at absolute
1262      * addresses can use long offsets and null base pointers to express
1263      * the field locations in a form usable by {@link #getInt(Object,long)}.
1264      * Therefore, code which will be ported to such JVMs on 64-bit platforms
1265      * must preserve all bits of static field offsets.
1266      * @see #getInt(Object, long)
1267      */
1268     public long objectFieldOffset(Field f) {
1269         if (f == null) {
1270             throw new NullPointerException();
1271         }
1272 
1273         return objectFieldOffset0(f);
1274     }
1275 
1276     /**
1277      * Reports the location of the field with a given name in the storage
1278      * allocation of its class.
1279      *
1280      * @throws NullPointerException if any parameter is {@code null}.
1281      * @throws InternalError if there is no field named {@code name} declared
1282      *         in class {@code c}, i.e., if {@code c.getDeclaredField(name)}
1283      *         would throw {@code java.lang.NoSuchFieldException}.
1284      *
1285      * @see #objectFieldOffset(Field)
1286      */
1287     public long objectFieldOffset(Class<?> c, String name) {
1288         if (c == null || name == null) {
1289             throw new NullPointerException();
1290         }
1291 
1292         return objectFieldOffset1(c, name);
1293     }
1294 
1295     /**
1296      * Reports the location of a given static field, in conjunction with {@link
1297      * #staticFieldBase}.
1298      * <p>Do not expect to perform any sort of arithmetic on this offset;
1299      * it is just a cookie which is passed to the unsafe heap memory accessors.
1300      *
1301      * <p>Any given field will always have the same offset, and no two distinct
1302      * fields of the same class will ever have the same offset.
1303      *
1304      * <p>As of 1.4.1, offsets for fields are represented as long values,
1305      * although the Sun JVM does not use the most significant 32 bits.
1306      * It is hard to imagine a JVM technology which needs more than
1307      * a few bits to encode an offset within a non-array object,
1308      * However, for consistency with other methods in this class,
1309      * this method reports its result as a long value.
1310      * @see #getInt(Object, long)
1311      */
1312     public long staticFieldOffset(Field f) {
1313         if (f == null) {
1314             throw new NullPointerException();
1315         }
1316 
1317         return staticFieldOffset0(f);
1318     }
1319 
1320     /**
1321      * Reports the location of a given static field, in conjunction with {@link
1322      * #staticFieldOffset}.
1323      * <p>Fetch the base "Object", if any, with which static fields of the
1324      * given class can be accessed via methods like {@link #getInt(Object,
1325      * long)}.  This value may be null.  This value may refer to an object
1326      * which is a "cookie", not guaranteed to be a real Object, and it should
1327      * not be used in any way except as argument to the get and put routines in
1328      * this class.
1329      */
1330     public Object staticFieldBase(Field f) {
1331         if (f == null) {
1332             throw new NullPointerException();
1333         }
1334 
1335         return staticFieldBase0(f);
1336     }
1337 
1338     /**
1339      * Detects if the given class may need to be initialized. This is often
1340      * needed in conjunction with obtaining the static field base of a
1341      * class.
1342      * @return false only if a call to {@code ensureClassInitialized} would have no effect
1343      */
1344     public boolean shouldBeInitialized(Class<?> c) {
1345         if (c == null) {
1346             throw new NullPointerException();
1347         }
1348 
1349         return shouldBeInitialized0(c);
1350     }
1351 
1352     /**
1353      * Ensures the given class has been initialized (see JVMS-5.5 for details).
1354      * This is often needed in conjunction with obtaining the static field base
1355      * of a class.
1356      *
1357      * The call returns when either class {@code c} is fully initialized or
1358      * class {@code c} is being initialized and the call is performed from
1359      * the initializing thread. In the latter case a subsequent call to
1360      * {@link #shouldBeInitialized} will return {@code true}.
1361      */
1362     public void ensureClassInitialized(Class<?> c) {
1363         if (c == null) {
1364             throw new NullPointerException();
1365         }
1366 
1367         ensureClassInitialized0(c);
1368     }
1369 
1370     /**
1371      * Reports the offset of the first element in the storage allocation of a
1372      * given array class.  If {@link #arrayIndexScale} returns a non-zero value
1373      * for the same class, you may use that scale factor, together with this
1374      * base offset, to form new offsets to access elements of arrays of the
1375      * given class.
1376      *
1377      * @see #getInt(Object, long)
1378      * @see #putInt(Object, long, int)
1379      */
1380     public int arrayBaseOffset(Class<?> arrayClass) {
1381         if (arrayClass == null) {
1382             throw new NullPointerException();
1383         }
1384 
1385         return arrayBaseOffset0(arrayClass);
1386     }
1387 
1388 
1389     /** The value of {@code arrayBaseOffset(boolean[].class)} */
1390     public static final int ARRAY_BOOLEAN_BASE_OFFSET
1391             = theUnsafe.arrayBaseOffset(boolean[].class);
1392 
1393     /** The value of {@code arrayBaseOffset(byte[].class)} */
1394     public static final int ARRAY_BYTE_BASE_OFFSET
1395             = theUnsafe.arrayBaseOffset(byte[].class);
1396 
1397     /** The value of {@code arrayBaseOffset(short[].class)} */
1398     public static final int ARRAY_SHORT_BASE_OFFSET
1399             = theUnsafe.arrayBaseOffset(short[].class);
1400 
1401     /** The value of {@code arrayBaseOffset(char[].class)} */
1402     public static final int ARRAY_CHAR_BASE_OFFSET
1403             = theUnsafe.arrayBaseOffset(char[].class);
1404 
1405     /** The value of {@code arrayBaseOffset(int[].class)} */
1406     public static final int ARRAY_INT_BASE_OFFSET
1407             = theUnsafe.arrayBaseOffset(int[].class);
1408 
1409     /** The value of {@code arrayBaseOffset(long[].class)} */
1410     public static final int ARRAY_LONG_BASE_OFFSET
1411             = theUnsafe.arrayBaseOffset(long[].class);
1412 
1413     /** The value of {@code arrayBaseOffset(float[].class)} */
1414     public static final int ARRAY_FLOAT_BASE_OFFSET
1415             = theUnsafe.arrayBaseOffset(float[].class);
1416 
1417     /** The value of {@code arrayBaseOffset(double[].class)} */
1418     public static final int ARRAY_DOUBLE_BASE_OFFSET
1419             = theUnsafe.arrayBaseOffset(double[].class);
1420 
1421     /** The value of {@code arrayBaseOffset(Object[].class)} */
1422     public static final int ARRAY_OBJECT_BASE_OFFSET
1423             = theUnsafe.arrayBaseOffset(Object[].class);
1424 
1425     /**
1426      * Reports the scale factor for addressing elements in the storage
1427      * allocation of a given array class.  However, arrays of "narrow" types
1428      * will generally not work properly with accessors like {@link
1429      * #getByte(Object, long)}, so the scale factor for such classes is reported
1430      * as zero.
1431      *
1432      * @see #arrayBaseOffset
1433      * @see #getInt(Object, long)
1434      * @see #putInt(Object, long, int)
1435      */
1436     public int arrayIndexScale(Class<?> arrayClass) {
1437         if (arrayClass == null) {
1438             throw new NullPointerException();
1439         }
1440 
1441         return arrayIndexScale0(arrayClass);
1442     }
1443 
1444     /**
1445      * Return the size of the object in the heap.
1446      * @param o an object
1447      * @return the objects's size
1448      * @since Valhalla
1449      */
1450     public long getObjectSize(Object o) {
1451         if (o == null)
1452             throw new NullPointerException();
1453         return getObjectSize0(o);
1454     }
1455 
1456     /** The value of {@code arrayIndexScale(boolean[].class)} */
1457     public static final int ARRAY_BOOLEAN_INDEX_SCALE
1458             = theUnsafe.arrayIndexScale(boolean[].class);
1459 
1460     /** The value of {@code arrayIndexScale(byte[].class)} */
1461     public static final int ARRAY_BYTE_INDEX_SCALE
1462             = theUnsafe.arrayIndexScale(byte[].class);
1463 
1464     /** The value of {@code arrayIndexScale(short[].class)} */
1465     public static final int ARRAY_SHORT_INDEX_SCALE
1466             = theUnsafe.arrayIndexScale(short[].class);
1467 
1468     /** The value of {@code arrayIndexScale(char[].class)} */
1469     public static final int ARRAY_CHAR_INDEX_SCALE
1470             = theUnsafe.arrayIndexScale(char[].class);
1471 
1472     /** The value of {@code arrayIndexScale(int[].class)} */
1473     public static final int ARRAY_INT_INDEX_SCALE
1474             = theUnsafe.arrayIndexScale(int[].class);
1475 
1476     /** The value of {@code arrayIndexScale(long[].class)} */
1477     public static final int ARRAY_LONG_INDEX_SCALE
1478             = theUnsafe.arrayIndexScale(long[].class);
1479 
1480     /** The value of {@code arrayIndexScale(float[].class)} */
1481     public static final int ARRAY_FLOAT_INDEX_SCALE
1482             = theUnsafe.arrayIndexScale(float[].class);
1483 
1484     /** The value of {@code arrayIndexScale(double[].class)} */
1485     public static final int ARRAY_DOUBLE_INDEX_SCALE
1486             = theUnsafe.arrayIndexScale(double[].class);
1487 
1488     /** The value of {@code arrayIndexScale(Object[].class)} */
1489     public static final int ARRAY_OBJECT_INDEX_SCALE
1490             = theUnsafe.arrayIndexScale(Object[].class);
1491 
1492     /**
1493      * Reports the size in bytes of a native pointer, as stored via {@link
1494      * #putAddress}.  This value will be either 4 or 8.  Note that the sizes of
1495      * other primitive types (as stored in native memory blocks) is determined
1496      * fully by their information content.
1497      */
1498     public int addressSize() {
1499         return ADDRESS_SIZE;
1500     }
1501 
1502     /** The value of {@code addressSize()} */
1503     public static final int ADDRESS_SIZE = ADDRESS_SIZE0;
1504 
1505     /**
1506      * Reports the size in bytes of a native memory page (whatever that is).
1507      * This value will always be a power of two.
1508      */
1509     public int pageSize() { return PAGE_SIZE; }
1510 
1511     /**
1512      * Reports the size in bytes of a data cache line written back by
1513      * the hardware cache line flush operation available to the JVM or
1514      * 0 if data cache line flushing is not enabled.
1515      */
1516     public int dataCacheLineFlushSize() { return DATA_CACHE_LINE_FLUSH_SIZE; }
1517 
1518     /**
1519      * Rounds down address to a data cache line boundary as
1520      * determined by {@link #dataCacheLineFlushSize}
1521      * @return the rounded down address
1522      */
1523     public long dataCacheLineAlignDown(long address) {
1524         return (address & ~(DATA_CACHE_LINE_FLUSH_SIZE - 1));
1525     }
1526 
1527     /**
1528      * Returns true if data cache line writeback
1529      */
1530     public static boolean isWritebackEnabled() { return DATA_CACHE_LINE_FLUSH_SIZE != 0; }
1531 
1532     //--- random trusted operations from JNI:
1533 
1534     /**
1535      * Tells the VM to define a class, without security checks.  By default, the
1536      * class loader and protection domain come from the caller's class.
1537      */
1538     public Class<?> defineClass(String name, byte[] b, int off, int len,
1539                                 ClassLoader loader,
1540                                 ProtectionDomain protectionDomain) {
1541         if (b == null) {
1542             throw new NullPointerException();
1543         }
1544         if (len < 0) {
1545             throw new ArrayIndexOutOfBoundsException();
1546         }
1547 
1548         return defineClass0(name, b, off, len, loader, protectionDomain);
1549     }
1550 
1551     public native Class<?> defineClass0(String name, byte[] b, int off, int len,
1552                                         ClassLoader loader,
1553                                         ProtectionDomain protectionDomain);
1554 
1555     /**
1556      * Allocates an instance but does not run any constructor.
1557      * Initializes the class if it has not yet been.
1558      */
1559     @IntrinsicCandidate
1560     public native Object allocateInstance(Class<?> cls)
1561         throws InstantiationException;
1562 
1563     /**
1564      * Allocates an array of a given type, but does not do zeroing.
1565      * <p>
1566      * This method should only be used in the very rare cases where a high-performance code
1567      * overwrites the destination array completely, and compilers cannot assist in zeroing elimination.
1568      * In an overwhelming majority of cases, a normal Java allocation should be used instead.
1569      * <p>
1570      * Users of this method are <b>required</b> to overwrite the initial (garbage) array contents
1571      * before allowing untrusted code, or code in other threads, to observe the reference
1572      * to the newly allocated array. In addition, the publication of the array reference must be
1573      * safe according to the Java Memory Model requirements.
1574      * <p>
1575      * The safest approach to deal with an uninitialized array is to keep the reference to it in local
1576      * variable at least until the initialization is complete, and then publish it <b>once</b>, either
1577      * by writing it to a <em>volatile</em> field, or storing it into a <em>final</em> field in constructor,
1578      * or issuing a {@link #storeFence} before publishing the reference.
1579      * <p>
1580      * @implnote This method can only allocate primitive arrays, to avoid garbage reference
1581      * elements that could break heap integrity.
1582      *
1583      * @param componentType array component type to allocate
1584      * @param length array size to allocate
1585      * @throws IllegalArgumentException if component type is null, or not a primitive class;
1586      *                                  or the length is negative
1587      */
1588     public Object allocateUninitializedArray(Class<?> componentType, int length) {
1589        if (componentType == null) {
1590            throw new IllegalArgumentException("Component type is null");
1591        }
1592        if (!componentType.isPrimitive()) {
1593            throw new IllegalArgumentException("Component type is not primitive");
1594        }
1595        if (length < 0) {
1596            throw new IllegalArgumentException("Negative length");
1597        }
1598        return allocateUninitializedArray0(componentType, length);
1599     }
1600 
1601     @IntrinsicCandidate
1602     private Object allocateUninitializedArray0(Class<?> componentType, int length) {
1603        // These fallbacks provide zeroed arrays, but intrinsic is not required to
1604        // return the zeroed arrays.
1605        if (componentType == byte.class)    return new byte[length];
1606        if (componentType == boolean.class) return new boolean[length];
1607        if (componentType == short.class)   return new short[length];
1608        if (componentType == char.class)    return new char[length];
1609        if (componentType == int.class)     return new int[length];
1610        if (componentType == float.class)   return new float[length];
1611        if (componentType == long.class)    return new long[length];
1612        if (componentType == double.class)  return new double[length];
1613        return null;
1614     }
1615 
1616     /** Throws the exception without telling the verifier. */
1617     public native void throwException(Throwable ee);
1618 
1619     /**
1620      * Atomically updates Java variable to {@code x} if it is currently
1621      * holding {@code expected}.
1622      *
1623      * <p>This operation has memory semantics of a {@code volatile} read
1624      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1625      *
1626      * @return {@code true} if successful
1627      */
1628     @IntrinsicCandidate
1629     public final native boolean compareAndSetReference(Object o, long offset,
1630                                                        Object expected,
1631                                                        Object x);
1632 
1633     private final boolean isValueObject(Object o) {
1634         return o != null && o.getClass().isValue();
1635     }
1636 
1637     /*
1638      * For value type, CAS should do substitutability test as opposed
1639      * to two pointers comparison.
1640      *
1641      * TODO: replace global lock workaround with the proper support for
1642      * atomic access to value objects and loosely consistent values.
1643      */
1644     public final <V> boolean compareAndSetReference(Object o, long offset,
1645                                                     Class<?> type,
1646                                                     V expected,
1647                                                     V x) {
1648         if (type.isValue() || isValueObject(expected)) {
1649             synchronized (valueLock) {
1650                 Object witness = getReference(o, offset);
1651                 if (witness == expected) {
1652                     putReference(o, offset, x);
1653                     return true;
1654                 } else {
1655                     return false;
1656                 }
1657             }
1658         } else {
1659             return compareAndSetReference(o, offset, expected, x);
1660         }
1661     }
1662 
1663     @ForceInline
1664     public final <V> boolean compareAndSetFlatValue(Object o, long offset,
1665                                                 int layout,
1666                                                 Class<?> valueType,
1667                                                 V expected,
1668                                                 V x) {
1669         synchronized (valueLock) {
1670             Object witness = getFlatValue(o, offset, layout, valueType);
1671             if (witness == expected) {
1672                 putFlatValue(o, offset, layout, valueType, x);
1673                 return true;
1674             }
1675             else {
1676                 return false;
1677             }
1678         }
1679     }
1680 
1681     @IntrinsicCandidate
1682     public final native Object compareAndExchangeReference(Object o, long offset,
1683                                                            Object expected,
1684                                                            Object x);
1685 
1686     public final <V> Object compareAndExchangeReference(Object o, long offset,
1687                                                         Class<?> valueType,
1688                                                         V expected,
1689                                                         V x) {
1690         if (valueType.isValue() || isValueObject(expected)) {
1691             synchronized (valueLock) {
1692                 Object witness = getReference(o, offset);
1693                 if (witness == expected) {
1694                     putReference(o, offset, x);
1695                 }
1696                 return witness;
1697             }
1698         } else {
1699             return compareAndExchangeReference(o, offset, expected, x);
1700         }
1701     }
1702 
1703     @ForceInline
1704     public final <V> Object compareAndExchangeFlatValue(Object o, long offset,
1705                                                     int layout,
1706                                                     Class<?> valueType,
1707                                                     V expected,
1708                                                     V x) {
1709         synchronized (valueLock) {
1710             Object witness = getFlatValue(o, offset, layout, valueType);
1711             if (witness == expected) {
1712                 putFlatValue(o, offset, layout, valueType, x);
1713             }
1714             return witness;
1715         }
1716     }
1717 
1718     @IntrinsicCandidate
1719     public final Object compareAndExchangeReferenceAcquire(Object o, long offset,
1720                                                            Object expected,
1721                                                            Object x) {
1722         return compareAndExchangeReference(o, offset, expected, x);
1723     }
1724 
1725     public final <V> Object compareAndExchangeReferenceAcquire(Object o, long offset,
1726                                                                Class<?> valueType,
1727                                                                V expected,
1728                                                                V x) {
1729         return compareAndExchangeReference(o, offset, valueType, expected, x);
1730     }
1731 
1732     @ForceInline
1733     public final <V> Object compareAndExchangeFlatValueAcquire(Object o, long offset,
1734                                                            int layout,
1735                                                            Class<?> valueType,
1736                                                            V expected,
1737                                                            V x) {
1738         return compareAndExchangeFlatValue(o, offset, layout, valueType, expected, x);
1739     }
1740 
1741     @IntrinsicCandidate
1742     public final Object compareAndExchangeReferenceRelease(Object o, long offset,
1743                                                            Object expected,
1744                                                            Object x) {
1745         return compareAndExchangeReference(o, offset, expected, x);
1746     }
1747 
1748     public final <V> Object compareAndExchangeReferenceRelease(Object o, long offset,
1749                                                                Class<?> valueType,
1750                                                                V expected,
1751                                                                V x) {
1752         return compareAndExchangeReference(o, offset, valueType, expected, x);
1753     }
1754 
1755     @ForceInline
1756     public final <V> Object compareAndExchangeFlatValueRelease(Object o, long offset,
1757                                                            int layout,
1758                                                            Class<?> valueType,
1759                                                            V expected,
1760                                                            V x) {
1761         return compareAndExchangeFlatValue(o, offset, layout, valueType, expected, x);
1762     }
1763 
1764     @IntrinsicCandidate
1765     public final boolean weakCompareAndSetReferencePlain(Object o, long offset,
1766                                                          Object expected,
1767                                                          Object x) {
1768         return compareAndSetReference(o, offset, expected, x);
1769     }
1770 
1771     public final <V> boolean weakCompareAndSetReferencePlain(Object o, long offset,
1772                                                              Class<?> valueType,
1773                                                              V expected,
1774                                                              V x) {
1775         if (valueType.isValue() || isValueObject(expected)) {
1776             return compareAndSetReference(o, offset, valueType, expected, x);
1777         } else {
1778             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1779         }
1780     }
1781 
1782     @ForceInline
1783     public final <V> boolean weakCompareAndSetFlatValuePlain(Object o, long offset,
1784                                                          int layout,
1785                                                          Class<?> valueType,
1786                                                          V expected,
1787                                                          V x) {
1788         return compareAndSetFlatValue(o, offset, layout, valueType, expected, x);
1789     }
1790 
1791     @IntrinsicCandidate
1792     public final boolean weakCompareAndSetReferenceAcquire(Object o, long offset,
1793                                                            Object expected,
1794                                                            Object x) {
1795         return compareAndSetReference(o, offset, expected, x);
1796     }
1797 
1798     public final <V> boolean weakCompareAndSetReferenceAcquire(Object o, long offset,
1799                                                                Class<?> valueType,
1800                                                                V expected,
1801                                                                V x) {
1802         if (valueType.isValue() || isValueObject(expected)) {
1803             return compareAndSetReference(o, offset, valueType, expected, x);
1804         } else {
1805             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1806         }
1807     }
1808 
1809     @ForceInline
1810     public final <V> boolean weakCompareAndSetFlatValueAcquire(Object o, long offset,
1811                                                            int layout,
1812                                                            Class<?> valueType,
1813                                                            V expected,
1814                                                            V x) {
1815         return compareAndSetFlatValue(o, offset, layout, valueType, expected, x);
1816     }
1817 
1818     @IntrinsicCandidate
1819     public final boolean weakCompareAndSetReferenceRelease(Object o, long offset,
1820                                                            Object expected,
1821                                                            Object x) {
1822         return compareAndSetReference(o, offset, expected, x);
1823     }
1824 
1825     public final <V> boolean weakCompareAndSetReferenceRelease(Object o, long offset,
1826                                                                Class<?> valueType,
1827                                                                V expected,
1828                                                                V x) {
1829         if (valueType.isValue() || isValueObject(expected)) {
1830             return compareAndSetReference(o, offset, valueType, expected, x);
1831         } else {
1832             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1833         }
1834     }
1835 
1836     @ForceInline
1837     public final <V> boolean weakCompareAndSetFlatValueRelease(Object o, long offset,
1838                                                            int layout,
1839                                                            Class<?> valueType,
1840                                                            V expected,
1841                                                            V x) {
1842         return compareAndSetFlatValue(o, offset, layout, valueType, expected, x);
1843     }
1844 
1845     @IntrinsicCandidate
1846     public final boolean weakCompareAndSetReference(Object o, long offset,
1847                                                     Object expected,
1848                                                     Object x) {
1849         return compareAndSetReference(o, offset, expected, x);
1850     }
1851 
1852     public final <V> boolean weakCompareAndSetReference(Object o, long offset,
1853                                                         Class<?> valueType,
1854                                                         V expected,
1855                                                         V x) {
1856         if (valueType.isValue() || isValueObject(expected)) {
1857             return compareAndSetReference(o, offset, valueType, expected, x);
1858         } else {
1859             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1860         }
1861     }
1862 
1863     @ForceInline
1864     public final <V> boolean weakCompareAndSetFlatValue(Object o, long offset,
1865                                                     int layout,
1866                                                     Class<?> valueType,
1867                                                     V expected,
1868                                                     V x) {
1869         return compareAndSetFlatValue(o, offset, layout, valueType, expected, x);
1870     }
1871 
1872     /**
1873      * Atomically updates Java variable to {@code x} if it is currently
1874      * holding {@code expected}.
1875      *
1876      * <p>This operation has memory semantics of a {@code volatile} read
1877      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1878      *
1879      * @return {@code true} if successful
1880      */
1881     @IntrinsicCandidate
1882     public final native boolean compareAndSetInt(Object o, long offset,
1883                                                  int expected,
1884                                                  int x);
1885 
1886     @IntrinsicCandidate
1887     public final native int compareAndExchangeInt(Object o, long offset,
1888                                                   int expected,
1889                                                   int x);
1890 
1891     @IntrinsicCandidate
1892     public final int compareAndExchangeIntAcquire(Object o, long offset,
1893                                                          int expected,
1894                                                          int x) {
1895         return compareAndExchangeInt(o, offset, expected, x);
1896     }
1897 
1898     @IntrinsicCandidate
1899     public final int compareAndExchangeIntRelease(Object o, long offset,
1900                                                          int expected,
1901                                                          int x) {
1902         return compareAndExchangeInt(o, offset, expected, x);
1903     }
1904 
1905     @IntrinsicCandidate
1906     public final boolean weakCompareAndSetIntPlain(Object o, long offset,
1907                                                    int expected,
1908                                                    int x) {
1909         return compareAndSetInt(o, offset, expected, x);
1910     }
1911 
1912     @IntrinsicCandidate
1913     public final boolean weakCompareAndSetIntAcquire(Object o, long offset,
1914                                                      int expected,
1915                                                      int x) {
1916         return compareAndSetInt(o, offset, expected, x);
1917     }
1918 
1919     @IntrinsicCandidate
1920     public final boolean weakCompareAndSetIntRelease(Object o, long offset,
1921                                                      int expected,
1922                                                      int x) {
1923         return compareAndSetInt(o, offset, expected, x);
1924     }
1925 
1926     @IntrinsicCandidate
1927     public final boolean weakCompareAndSetInt(Object o, long offset,
1928                                               int expected,
1929                                               int x) {
1930         return compareAndSetInt(o, offset, expected, x);
1931     }
1932 
1933     @IntrinsicCandidate
1934     public final byte compareAndExchangeByte(Object o, long offset,
1935                                              byte expected,
1936                                              byte x) {
1937         long wordOffset = offset & ~3;
1938         int shift = (int) (offset & 3) << 3;
1939         if (BIG_ENDIAN) {
1940             shift = 24 - shift;
1941         }
1942         int mask           = 0xFF << shift;
1943         int maskedExpected = (expected & 0xFF) << shift;
1944         int maskedX        = (x & 0xFF) << shift;
1945         int fullWord;
1946         do {
1947             fullWord = getIntVolatile(o, wordOffset);
1948             if ((fullWord & mask) != maskedExpected)
1949                 return (byte) ((fullWord & mask) >> shift);
1950         } while (!weakCompareAndSetInt(o, wordOffset,
1951                                                 fullWord, (fullWord & ~mask) | maskedX));
1952         return expected;
1953     }
1954 
1955     @IntrinsicCandidate
1956     public final boolean compareAndSetByte(Object o, long offset,
1957                                            byte expected,
1958                                            byte x) {
1959         return compareAndExchangeByte(o, offset, expected, x) == expected;
1960     }
1961 
1962     @IntrinsicCandidate
1963     public final boolean weakCompareAndSetByte(Object o, long offset,
1964                                                byte expected,
1965                                                byte x) {
1966         return compareAndSetByte(o, offset, expected, x);
1967     }
1968 
1969     @IntrinsicCandidate
1970     public final boolean weakCompareAndSetByteAcquire(Object o, long offset,
1971                                                       byte expected,
1972                                                       byte x) {
1973         return weakCompareAndSetByte(o, offset, expected, x);
1974     }
1975 
1976     @IntrinsicCandidate
1977     public final boolean weakCompareAndSetByteRelease(Object o, long offset,
1978                                                       byte expected,
1979                                                       byte x) {
1980         return weakCompareAndSetByte(o, offset, expected, x);
1981     }
1982 
1983     @IntrinsicCandidate
1984     public final boolean weakCompareAndSetBytePlain(Object o, long offset,
1985                                                     byte expected,
1986                                                     byte x) {
1987         return weakCompareAndSetByte(o, offset, expected, x);
1988     }
1989 
1990     @IntrinsicCandidate
1991     public final byte compareAndExchangeByteAcquire(Object o, long offset,
1992                                                     byte expected,
1993                                                     byte x) {
1994         return compareAndExchangeByte(o, offset, expected, x);
1995     }
1996 
1997     @IntrinsicCandidate
1998     public final byte compareAndExchangeByteRelease(Object o, long offset,
1999                                                     byte expected,
2000                                                     byte x) {
2001         return compareAndExchangeByte(o, offset, expected, x);
2002     }
2003 
2004     @IntrinsicCandidate
2005     public final short compareAndExchangeShort(Object o, long offset,
2006                                                short expected,
2007                                                short x) {
2008         if ((offset & 3) == 3) {
2009             throw new IllegalArgumentException("Update spans the word, not supported");
2010         }
2011         long wordOffset = offset & ~3;
2012         int shift = (int) (offset & 3) << 3;
2013         if (BIG_ENDIAN) {
2014             shift = 16 - shift;
2015         }
2016         int mask           = 0xFFFF << shift;
2017         int maskedExpected = (expected & 0xFFFF) << shift;
2018         int maskedX        = (x & 0xFFFF) << shift;
2019         int fullWord;
2020         do {
2021             fullWord = getIntVolatile(o, wordOffset);
2022             if ((fullWord & mask) != maskedExpected) {
2023                 return (short) ((fullWord & mask) >> shift);
2024             }
2025         } while (!weakCompareAndSetInt(o, wordOffset,
2026                                                 fullWord, (fullWord & ~mask) | maskedX));
2027         return expected;
2028     }
2029 
2030     @IntrinsicCandidate
2031     public final boolean compareAndSetShort(Object o, long offset,
2032                                             short expected,
2033                                             short x) {
2034         return compareAndExchangeShort(o, offset, expected, x) == expected;
2035     }
2036 
2037     @IntrinsicCandidate
2038     public final boolean weakCompareAndSetShort(Object o, long offset,
2039                                                 short expected,
2040                                                 short x) {
2041         return compareAndSetShort(o, offset, expected, x);
2042     }
2043 
2044     @IntrinsicCandidate
2045     public final boolean weakCompareAndSetShortAcquire(Object o, long offset,
2046                                                        short expected,
2047                                                        short x) {
2048         return weakCompareAndSetShort(o, offset, expected, x);
2049     }
2050 
2051     @IntrinsicCandidate
2052     public final boolean weakCompareAndSetShortRelease(Object o, long offset,
2053                                                        short expected,
2054                                                        short x) {
2055         return weakCompareAndSetShort(o, offset, expected, x);
2056     }
2057 
2058     @IntrinsicCandidate
2059     public final boolean weakCompareAndSetShortPlain(Object o, long offset,
2060                                                      short expected,
2061                                                      short x) {
2062         return weakCompareAndSetShort(o, offset, expected, x);
2063     }
2064 
2065 
2066     @IntrinsicCandidate
2067     public final short compareAndExchangeShortAcquire(Object o, long offset,
2068                                                      short expected,
2069                                                      short x) {
2070         return compareAndExchangeShort(o, offset, expected, x);
2071     }
2072 
2073     @IntrinsicCandidate
2074     public final short compareAndExchangeShortRelease(Object o, long offset,
2075                                                     short expected,
2076                                                     short x) {
2077         return compareAndExchangeShort(o, offset, expected, x);
2078     }
2079 
2080     @ForceInline
2081     private char s2c(short s) {
2082         return (char) s;
2083     }
2084 
2085     @ForceInline
2086     private short c2s(char s) {
2087         return (short) s;
2088     }
2089 
2090     @ForceInline
2091     public final boolean compareAndSetChar(Object o, long offset,
2092                                            char expected,
2093                                            char x) {
2094         return compareAndSetShort(o, offset, c2s(expected), c2s(x));
2095     }
2096 
2097     @ForceInline
2098     public final char compareAndExchangeChar(Object o, long offset,
2099                                              char expected,
2100                                              char x) {
2101         return s2c(compareAndExchangeShort(o, offset, c2s(expected), c2s(x)));
2102     }
2103 
2104     @ForceInline
2105     public final char compareAndExchangeCharAcquire(Object o, long offset,
2106                                             char expected,
2107                                             char x) {
2108         return s2c(compareAndExchangeShortAcquire(o, offset, c2s(expected), c2s(x)));
2109     }
2110 
2111     @ForceInline
2112     public final char compareAndExchangeCharRelease(Object o, long offset,
2113                                             char expected,
2114                                             char x) {
2115         return s2c(compareAndExchangeShortRelease(o, offset, c2s(expected), c2s(x)));
2116     }
2117 
2118     @ForceInline
2119     public final boolean weakCompareAndSetChar(Object o, long offset,
2120                                                char expected,
2121                                                char x) {
2122         return weakCompareAndSetShort(o, offset, c2s(expected), c2s(x));
2123     }
2124 
2125     @ForceInline
2126     public final boolean weakCompareAndSetCharAcquire(Object o, long offset,
2127                                                       char expected,
2128                                                       char x) {
2129         return weakCompareAndSetShortAcquire(o, offset, c2s(expected), c2s(x));
2130     }
2131 
2132     @ForceInline
2133     public final boolean weakCompareAndSetCharRelease(Object o, long offset,
2134                                                       char expected,
2135                                                       char x) {
2136         return weakCompareAndSetShortRelease(o, offset, c2s(expected), c2s(x));
2137     }
2138 
2139     @ForceInline
2140     public final boolean weakCompareAndSetCharPlain(Object o, long offset,
2141                                                     char expected,
2142                                                     char x) {
2143         return weakCompareAndSetShortPlain(o, offset, c2s(expected), c2s(x));
2144     }
2145 
2146     /**
2147      * The JVM converts integral values to boolean values using two
2148      * different conventions, byte testing against zero and truncation
2149      * to least-significant bit.
2150      *
2151      * <p>The JNI documents specify that, at least for returning
2152      * values from native methods, a Java boolean value is converted
2153      * to the value-set 0..1 by first truncating to a byte (0..255 or
2154      * maybe -128..127) and then testing against zero. Thus, Java
2155      * booleans in non-Java data structures are by convention
2156      * represented as 8-bit containers containing either zero (for
2157      * false) or any non-zero value (for true).
2158      *
2159      * <p>Java booleans in the heap are also stored in bytes, but are
2160      * strongly normalized to the value-set 0..1 (i.e., they are
2161      * truncated to the least-significant bit).
2162      *
2163      * <p>The main reason for having different conventions for
2164      * conversion is performance: Truncation to the least-significant
2165      * bit can be usually implemented with fewer (machine)
2166      * instructions than byte testing against zero.
2167      *
2168      * <p>A number of Unsafe methods load boolean values from the heap
2169      * as bytes. Unsafe converts those values according to the JNI
2170      * rules (i.e, using the "testing against zero" convention). The
2171      * method {@code byte2bool} implements that conversion.
2172      *
2173      * @param b the byte to be converted to boolean
2174      * @return the result of the conversion
2175      */
2176     @ForceInline
2177     private boolean byte2bool(byte b) {
2178         return b != 0;
2179     }
2180 
2181     /**
2182      * Convert a boolean value to a byte. The return value is strongly
2183      * normalized to the value-set 0..1 (i.e., the value is truncated
2184      * to the least-significant bit). See {@link #byte2bool(byte)} for
2185      * more details on conversion conventions.
2186      *
2187      * @param b the boolean to be converted to byte (and then normalized)
2188      * @return the result of the conversion
2189      */
2190     @ForceInline
2191     private byte bool2byte(boolean b) {
2192         return b ? (byte)1 : (byte)0;
2193     }
2194 
2195     @ForceInline
2196     public final boolean compareAndSetBoolean(Object o, long offset,
2197                                               boolean expected,
2198                                               boolean x) {
2199         return compareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
2200     }
2201 
2202     @ForceInline
2203     public final boolean compareAndExchangeBoolean(Object o, long offset,
2204                                                    boolean expected,
2205                                                    boolean x) {
2206         return byte2bool(compareAndExchangeByte(o, offset, bool2byte(expected), bool2byte(x)));
2207     }
2208 
2209     @ForceInline
2210     public final boolean compareAndExchangeBooleanAcquire(Object o, long offset,
2211                                                     boolean expected,
2212                                                     boolean x) {
2213         return byte2bool(compareAndExchangeByteAcquire(o, offset, bool2byte(expected), bool2byte(x)));
2214     }
2215 
2216     @ForceInline
2217     public final boolean compareAndExchangeBooleanRelease(Object o, long offset,
2218                                                        boolean expected,
2219                                                        boolean x) {
2220         return byte2bool(compareAndExchangeByteRelease(o, offset, bool2byte(expected), bool2byte(x)));
2221     }
2222 
2223     @ForceInline
2224     public final boolean weakCompareAndSetBoolean(Object o, long offset,
2225                                                   boolean expected,
2226                                                   boolean x) {
2227         return weakCompareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
2228     }
2229 
2230     @ForceInline
2231     public final boolean weakCompareAndSetBooleanAcquire(Object o, long offset,
2232                                                          boolean expected,
2233                                                          boolean x) {
2234         return weakCompareAndSetByteAcquire(o, offset, bool2byte(expected), bool2byte(x));
2235     }
2236 
2237     @ForceInline
2238     public final boolean weakCompareAndSetBooleanRelease(Object o, long offset,
2239                                                          boolean expected,
2240                                                          boolean x) {
2241         return weakCompareAndSetByteRelease(o, offset, bool2byte(expected), bool2byte(x));
2242     }
2243 
2244     @ForceInline
2245     public final boolean weakCompareAndSetBooleanPlain(Object o, long offset,
2246                                                        boolean expected,
2247                                                        boolean x) {
2248         return weakCompareAndSetBytePlain(o, offset, bool2byte(expected), bool2byte(x));
2249     }
2250 
2251     /**
2252      * Atomically updates Java variable to {@code x} if it is currently
2253      * holding {@code expected}.
2254      *
2255      * <p>This operation has memory semantics of a {@code volatile} read
2256      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2257      *
2258      * @return {@code true} if successful
2259      */
2260     @ForceInline
2261     public final boolean compareAndSetFloat(Object o, long offset,
2262                                             float expected,
2263                                             float x) {
2264         return compareAndSetInt(o, offset,
2265                                  Float.floatToRawIntBits(expected),
2266                                  Float.floatToRawIntBits(x));
2267     }
2268 
2269     @ForceInline
2270     public final float compareAndExchangeFloat(Object o, long offset,
2271                                                float expected,
2272                                                float x) {
2273         int w = compareAndExchangeInt(o, offset,
2274                                       Float.floatToRawIntBits(expected),
2275                                       Float.floatToRawIntBits(x));
2276         return Float.intBitsToFloat(w);
2277     }
2278 
2279     @ForceInline
2280     public final float compareAndExchangeFloatAcquire(Object o, long offset,
2281                                                   float expected,
2282                                                   float x) {
2283         int w = compareAndExchangeIntAcquire(o, offset,
2284                                              Float.floatToRawIntBits(expected),
2285                                              Float.floatToRawIntBits(x));
2286         return Float.intBitsToFloat(w);
2287     }
2288 
2289     @ForceInline
2290     public final float compareAndExchangeFloatRelease(Object o, long offset,
2291                                                   float expected,
2292                                                   float x) {
2293         int w = compareAndExchangeIntRelease(o, offset,
2294                                              Float.floatToRawIntBits(expected),
2295                                              Float.floatToRawIntBits(x));
2296         return Float.intBitsToFloat(w);
2297     }
2298 
2299     @ForceInline
2300     public final boolean weakCompareAndSetFloatPlain(Object o, long offset,
2301                                                      float expected,
2302                                                      float x) {
2303         return weakCompareAndSetIntPlain(o, offset,
2304                                      Float.floatToRawIntBits(expected),
2305                                      Float.floatToRawIntBits(x));
2306     }
2307 
2308     @ForceInline
2309     public final boolean weakCompareAndSetFloatAcquire(Object o, long offset,
2310                                                        float expected,
2311                                                        float x) {
2312         return weakCompareAndSetIntAcquire(o, offset,
2313                                             Float.floatToRawIntBits(expected),
2314                                             Float.floatToRawIntBits(x));
2315     }
2316 
2317     @ForceInline
2318     public final boolean weakCompareAndSetFloatRelease(Object o, long offset,
2319                                                        float expected,
2320                                                        float x) {
2321         return weakCompareAndSetIntRelease(o, offset,
2322                                             Float.floatToRawIntBits(expected),
2323                                             Float.floatToRawIntBits(x));
2324     }
2325 
2326     @ForceInline
2327     public final boolean weakCompareAndSetFloat(Object o, long offset,
2328                                                 float expected,
2329                                                 float x) {
2330         return weakCompareAndSetInt(o, offset,
2331                                              Float.floatToRawIntBits(expected),
2332                                              Float.floatToRawIntBits(x));
2333     }
2334 
2335     /**
2336      * Atomically updates Java variable to {@code x} if it is currently
2337      * holding {@code expected}.
2338      *
2339      * <p>This operation has memory semantics of a {@code volatile} read
2340      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2341      *
2342      * @return {@code true} if successful
2343      */
2344     @ForceInline
2345     public final boolean compareAndSetDouble(Object o, long offset,
2346                                              double expected,
2347                                              double x) {
2348         return compareAndSetLong(o, offset,
2349                                  Double.doubleToRawLongBits(expected),
2350                                  Double.doubleToRawLongBits(x));
2351     }
2352 
2353     @ForceInline
2354     public final double compareAndExchangeDouble(Object o, long offset,
2355                                                  double expected,
2356                                                  double x) {
2357         long w = compareAndExchangeLong(o, offset,
2358                                         Double.doubleToRawLongBits(expected),
2359                                         Double.doubleToRawLongBits(x));
2360         return Double.longBitsToDouble(w);
2361     }
2362 
2363     @ForceInline
2364     public final double compareAndExchangeDoubleAcquire(Object o, long offset,
2365                                                         double expected,
2366                                                         double x) {
2367         long w = compareAndExchangeLongAcquire(o, offset,
2368                                                Double.doubleToRawLongBits(expected),
2369                                                Double.doubleToRawLongBits(x));
2370         return Double.longBitsToDouble(w);
2371     }
2372 
2373     @ForceInline
2374     public final double compareAndExchangeDoubleRelease(Object o, long offset,
2375                                                         double expected,
2376                                                         double x) {
2377         long w = compareAndExchangeLongRelease(o, offset,
2378                                                Double.doubleToRawLongBits(expected),
2379                                                Double.doubleToRawLongBits(x));
2380         return Double.longBitsToDouble(w);
2381     }
2382 
2383     @ForceInline
2384     public final boolean weakCompareAndSetDoublePlain(Object o, long offset,
2385                                                       double expected,
2386                                                       double x) {
2387         return weakCompareAndSetLongPlain(o, offset,
2388                                      Double.doubleToRawLongBits(expected),
2389                                      Double.doubleToRawLongBits(x));
2390     }
2391 
2392     @ForceInline
2393     public final boolean weakCompareAndSetDoubleAcquire(Object o, long offset,
2394                                                         double expected,
2395                                                         double x) {
2396         return weakCompareAndSetLongAcquire(o, offset,
2397                                              Double.doubleToRawLongBits(expected),
2398                                              Double.doubleToRawLongBits(x));
2399     }
2400 
2401     @ForceInline
2402     public final boolean weakCompareAndSetDoubleRelease(Object o, long offset,
2403                                                         double expected,
2404                                                         double x) {
2405         return weakCompareAndSetLongRelease(o, offset,
2406                                              Double.doubleToRawLongBits(expected),
2407                                              Double.doubleToRawLongBits(x));
2408     }
2409 
2410     @ForceInline
2411     public final boolean weakCompareAndSetDouble(Object o, long offset,
2412                                                  double expected,
2413                                                  double x) {
2414         return weakCompareAndSetLong(o, offset,
2415                                               Double.doubleToRawLongBits(expected),
2416                                               Double.doubleToRawLongBits(x));
2417     }
2418 
2419     /**
2420      * Atomically updates Java variable to {@code x} if it is currently
2421      * holding {@code expected}.
2422      *
2423      * <p>This operation has memory semantics of a {@code volatile} read
2424      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2425      *
2426      * @return {@code true} if successful
2427      */
2428     @IntrinsicCandidate
2429     public final native boolean compareAndSetLong(Object o, long offset,
2430                                                   long expected,
2431                                                   long x);
2432 
2433     @IntrinsicCandidate
2434     public final native long compareAndExchangeLong(Object o, long offset,
2435                                                     long expected,
2436                                                     long x);
2437 
2438     @IntrinsicCandidate
2439     public final long compareAndExchangeLongAcquire(Object o, long offset,
2440                                                            long expected,
2441                                                            long x) {
2442         return compareAndExchangeLong(o, offset, expected, x);
2443     }
2444 
2445     @IntrinsicCandidate
2446     public final long compareAndExchangeLongRelease(Object o, long offset,
2447                                                            long expected,
2448                                                            long x) {
2449         return compareAndExchangeLong(o, offset, expected, x);
2450     }
2451 
2452     @IntrinsicCandidate
2453     public final boolean weakCompareAndSetLongPlain(Object o, long offset,
2454                                                     long expected,
2455                                                     long x) {
2456         return compareAndSetLong(o, offset, expected, x);
2457     }
2458 
2459     @IntrinsicCandidate
2460     public final boolean weakCompareAndSetLongAcquire(Object o, long offset,
2461                                                       long expected,
2462                                                       long x) {
2463         return compareAndSetLong(o, offset, expected, x);
2464     }
2465 
2466     @IntrinsicCandidate
2467     public final boolean weakCompareAndSetLongRelease(Object o, long offset,
2468                                                       long expected,
2469                                                       long x) {
2470         return compareAndSetLong(o, offset, expected, x);
2471     }
2472 
2473     @IntrinsicCandidate
2474     public final boolean weakCompareAndSetLong(Object o, long offset,
2475                                                long expected,
2476                                                long x) {
2477         return compareAndSetLong(o, offset, expected, x);
2478     }
2479 
2480     /**
2481      * Fetches a reference value from a given Java variable, with volatile
2482      * load semantics. Otherwise identical to {@link #getReference(Object, long)}
2483      */
2484     @IntrinsicCandidate
2485     public native Object getReferenceVolatile(Object o, long offset);
2486 
2487     /**
2488      * Global lock for atomic and volatile strength access to any value of
2489      * a value type.  This is a temporary workaround until better localized
2490      * atomic access mechanisms are supported for value class and primitive class.
2491      */
2492     private static final Object valueLock = new Object();
2493 
2494     public final <V> Object getFlatValueVolatile(Object base, long offset, int layout, Class<?> valueType) {
2495         synchronized (valueLock) {
2496             return getFlatValue(base, offset, layout, valueType);
2497         }
2498     }
2499 
2500     /**
2501      * Stores a reference value into a given Java variable, with
2502      * volatile store semantics. Otherwise identical to {@link #putReference(Object, long, Object)}
2503      */
2504     @IntrinsicCandidate
2505     public native void putReferenceVolatile(Object o, long offset, Object x);
2506 
2507     public final <V> void putFlatValueVolatile(Object o, long offset, int layout, Class<?> valueType, V x) {
2508         synchronized (valueLock) {
2509             putFlatValue(o, offset, layout, valueType, x);
2510         }
2511     }
2512 
2513     /** Volatile version of {@link #getInt(Object, long)}  */
2514     @IntrinsicCandidate
2515     public native int     getIntVolatile(Object o, long offset);
2516 
2517     /** Volatile version of {@link #putInt(Object, long, int)}  */
2518     @IntrinsicCandidate
2519     public native void    putIntVolatile(Object o, long offset, int x);
2520 
2521     /** Volatile version of {@link #getBoolean(Object, long)}  */
2522     @IntrinsicCandidate
2523     public native boolean getBooleanVolatile(Object o, long offset);
2524 
2525     /** Volatile version of {@link #putBoolean(Object, long, boolean)}  */
2526     @IntrinsicCandidate
2527     public native void    putBooleanVolatile(Object o, long offset, boolean x);
2528 
2529     /** Volatile version of {@link #getByte(Object, long)}  */
2530     @IntrinsicCandidate
2531     public native byte    getByteVolatile(Object o, long offset);
2532 
2533     /** Volatile version of {@link #putByte(Object, long, byte)}  */
2534     @IntrinsicCandidate
2535     public native void    putByteVolatile(Object o, long offset, byte x);
2536 
2537     /** Volatile version of {@link #getShort(Object, long)}  */
2538     @IntrinsicCandidate
2539     public native short   getShortVolatile(Object o, long offset);
2540 
2541     /** Volatile version of {@link #putShort(Object, long, short)}  */
2542     @IntrinsicCandidate
2543     public native void    putShortVolatile(Object o, long offset, short x);
2544 
2545     /** Volatile version of {@link #getChar(Object, long)}  */
2546     @IntrinsicCandidate
2547     public native char    getCharVolatile(Object o, long offset);
2548 
2549     /** Volatile version of {@link #putChar(Object, long, char)}  */
2550     @IntrinsicCandidate
2551     public native void    putCharVolatile(Object o, long offset, char x);
2552 
2553     /** Volatile version of {@link #getLong(Object, long)}  */
2554     @IntrinsicCandidate
2555     public native long    getLongVolatile(Object o, long offset);
2556 
2557     /** Volatile version of {@link #putLong(Object, long, long)}  */
2558     @IntrinsicCandidate
2559     public native void    putLongVolatile(Object o, long offset, long x);
2560 
2561     /** Volatile version of {@link #getFloat(Object, long)}  */
2562     @IntrinsicCandidate
2563     public native float   getFloatVolatile(Object o, long offset);
2564 
2565     /** Volatile version of {@link #putFloat(Object, long, float)}  */
2566     @IntrinsicCandidate
2567     public native void    putFloatVolatile(Object o, long offset, float x);
2568 
2569     /** Volatile version of {@link #getDouble(Object, long)}  */
2570     @IntrinsicCandidate
2571     public native double  getDoubleVolatile(Object o, long offset);
2572 
2573     /** Volatile version of {@link #putDouble(Object, long, double)}  */
2574     @IntrinsicCandidate
2575     public native void    putDoubleVolatile(Object o, long offset, double x);
2576 
2577 
2578 
2579     /** Acquire version of {@link #getReferenceVolatile(Object, long)} */
2580     @IntrinsicCandidate
2581     public final Object getReferenceAcquire(Object o, long offset) {
2582         return getReferenceVolatile(o, offset);
2583     }
2584 
2585     public final <V> Object getFlatValueAcquire(Object base, long offset, int layout, Class<?> valueType) {
2586         return getFlatValueVolatile(base, offset, layout, valueType);
2587     }
2588 
2589     /** Acquire version of {@link #getBooleanVolatile(Object, long)} */
2590     @IntrinsicCandidate
2591     public final boolean getBooleanAcquire(Object o, long offset) {
2592         return getBooleanVolatile(o, offset);
2593     }
2594 
2595     /** Acquire version of {@link #getByteVolatile(Object, long)} */
2596     @IntrinsicCandidate
2597     public final byte getByteAcquire(Object o, long offset) {
2598         return getByteVolatile(o, offset);
2599     }
2600 
2601     /** Acquire version of {@link #getShortVolatile(Object, long)} */
2602     @IntrinsicCandidate
2603     public final short getShortAcquire(Object o, long offset) {
2604         return getShortVolatile(o, offset);
2605     }
2606 
2607     /** Acquire version of {@link #getCharVolatile(Object, long)} */
2608     @IntrinsicCandidate
2609     public final char getCharAcquire(Object o, long offset) {
2610         return getCharVolatile(o, offset);
2611     }
2612 
2613     /** Acquire version of {@link #getIntVolatile(Object, long)} */
2614     @IntrinsicCandidate
2615     public final int getIntAcquire(Object o, long offset) {
2616         return getIntVolatile(o, offset);
2617     }
2618 
2619     /** Acquire version of {@link #getFloatVolatile(Object, long)} */
2620     @IntrinsicCandidate
2621     public final float getFloatAcquire(Object o, long offset) {
2622         return getFloatVolatile(o, offset);
2623     }
2624 
2625     /** Acquire version of {@link #getLongVolatile(Object, long)} */
2626     @IntrinsicCandidate
2627     public final long getLongAcquire(Object o, long offset) {
2628         return getLongVolatile(o, offset);
2629     }
2630 
2631     /** Acquire version of {@link #getDoubleVolatile(Object, long)} */
2632     @IntrinsicCandidate
2633     public final double getDoubleAcquire(Object o, long offset) {
2634         return getDoubleVolatile(o, offset);
2635     }
2636 
2637     /*
2638      * Versions of {@link #putReferenceVolatile(Object, long, Object)}
2639      * that do not guarantee immediate visibility of the store to
2640      * other threads. This method is generally only useful if the
2641      * underlying field is a Java volatile (or if an array cell, one
2642      * that is otherwise only accessed using volatile accesses).
2643      *
2644      * Corresponds to C11 atomic_store_explicit(..., memory_order_release).
2645      */
2646 
2647     /** Release version of {@link #putReferenceVolatile(Object, long, Object)} */
2648     @IntrinsicCandidate
2649     public final void putReferenceRelease(Object o, long offset, Object x) {
2650         putReferenceVolatile(o, offset, x);
2651     }
2652 
2653     public final <V> void putFlatValueRelease(Object o, long offset, int layout, Class<?> valueType, V x) {
2654         putFlatValueVolatile(o, offset, layout, valueType, x);
2655     }
2656 
2657     /** Release version of {@link #putBooleanVolatile(Object, long, boolean)} */
2658     @IntrinsicCandidate
2659     public final void putBooleanRelease(Object o, long offset, boolean x) {
2660         putBooleanVolatile(o, offset, x);
2661     }
2662 
2663     /** Release version of {@link #putByteVolatile(Object, long, byte)} */
2664     @IntrinsicCandidate
2665     public final void putByteRelease(Object o, long offset, byte x) {
2666         putByteVolatile(o, offset, x);
2667     }
2668 
2669     /** Release version of {@link #putShortVolatile(Object, long, short)} */
2670     @IntrinsicCandidate
2671     public final void putShortRelease(Object o, long offset, short x) {
2672         putShortVolatile(o, offset, x);
2673     }
2674 
2675     /** Release version of {@link #putCharVolatile(Object, long, char)} */
2676     @IntrinsicCandidate
2677     public final void putCharRelease(Object o, long offset, char x) {
2678         putCharVolatile(o, offset, x);
2679     }
2680 
2681     /** Release version of {@link #putIntVolatile(Object, long, int)} */
2682     @IntrinsicCandidate
2683     public final void putIntRelease(Object o, long offset, int x) {
2684         putIntVolatile(o, offset, x);
2685     }
2686 
2687     /** Release version of {@link #putFloatVolatile(Object, long, float)} */
2688     @IntrinsicCandidate
2689     public final void putFloatRelease(Object o, long offset, float x) {
2690         putFloatVolatile(o, offset, x);
2691     }
2692 
2693     /** Release version of {@link #putLongVolatile(Object, long, long)} */
2694     @IntrinsicCandidate
2695     public final void putLongRelease(Object o, long offset, long x) {
2696         putLongVolatile(o, offset, x);
2697     }
2698 
2699     /** Release version of {@link #putDoubleVolatile(Object, long, double)} */
2700     @IntrinsicCandidate
2701     public final void putDoubleRelease(Object o, long offset, double x) {
2702         putDoubleVolatile(o, offset, x);
2703     }
2704 
2705     // ------------------------------ Opaque --------------------------------------
2706 
2707     /** Opaque version of {@link #getReferenceVolatile(Object, long)} */
2708     @IntrinsicCandidate
2709     public final Object getReferenceOpaque(Object o, long offset) {
2710         return getReferenceVolatile(o, offset);
2711     }
2712 
2713     public final <V> Object getFlatValueOpaque(Object base, long offset, int layout, Class<?> valueType) {
2714         return getFlatValueVolatile(base, offset, layout, valueType);
2715     }
2716 
2717     /** Opaque version of {@link #getBooleanVolatile(Object, long)} */
2718     @IntrinsicCandidate
2719     public final boolean getBooleanOpaque(Object o, long offset) {
2720         return getBooleanVolatile(o, offset);
2721     }
2722 
2723     /** Opaque version of {@link #getByteVolatile(Object, long)} */
2724     @IntrinsicCandidate
2725     public final byte getByteOpaque(Object o, long offset) {
2726         return getByteVolatile(o, offset);
2727     }
2728 
2729     /** Opaque version of {@link #getShortVolatile(Object, long)} */
2730     @IntrinsicCandidate
2731     public final short getShortOpaque(Object o, long offset) {
2732         return getShortVolatile(o, offset);
2733     }
2734 
2735     /** Opaque version of {@link #getCharVolatile(Object, long)} */
2736     @IntrinsicCandidate
2737     public final char getCharOpaque(Object o, long offset) {
2738         return getCharVolatile(o, offset);
2739     }
2740 
2741     /** Opaque version of {@link #getIntVolatile(Object, long)} */
2742     @IntrinsicCandidate
2743     public final int getIntOpaque(Object o, long offset) {
2744         return getIntVolatile(o, offset);
2745     }
2746 
2747     /** Opaque version of {@link #getFloatVolatile(Object, long)} */
2748     @IntrinsicCandidate
2749     public final float getFloatOpaque(Object o, long offset) {
2750         return getFloatVolatile(o, offset);
2751     }
2752 
2753     /** Opaque version of {@link #getLongVolatile(Object, long)} */
2754     @IntrinsicCandidate
2755     public final long getLongOpaque(Object o, long offset) {
2756         return getLongVolatile(o, offset);
2757     }
2758 
2759     /** Opaque version of {@link #getDoubleVolatile(Object, long)} */
2760     @IntrinsicCandidate
2761     public final double getDoubleOpaque(Object o, long offset) {
2762         return getDoubleVolatile(o, offset);
2763     }
2764 
2765     /** Opaque version of {@link #putReferenceVolatile(Object, long, Object)} */
2766     @IntrinsicCandidate
2767     public final void putReferenceOpaque(Object o, long offset, Object x) {
2768         putReferenceVolatile(o, offset, x);
2769     }
2770 
2771     public final <V> void putFlatValueOpaque(Object o, long offset, int layout, Class<?> valueType, V x) {
2772         putFlatValueVolatile(o, offset, layout, valueType, x);
2773     }
2774 
2775     /** Opaque version of {@link #putBooleanVolatile(Object, long, boolean)} */
2776     @IntrinsicCandidate
2777     public final void putBooleanOpaque(Object o, long offset, boolean x) {
2778         putBooleanVolatile(o, offset, x);
2779     }
2780 
2781     /** Opaque version of {@link #putByteVolatile(Object, long, byte)} */
2782     @IntrinsicCandidate
2783     public final void putByteOpaque(Object o, long offset, byte x) {
2784         putByteVolatile(o, offset, x);
2785     }
2786 
2787     /** Opaque version of {@link #putShortVolatile(Object, long, short)} */
2788     @IntrinsicCandidate
2789     public final void putShortOpaque(Object o, long offset, short x) {
2790         putShortVolatile(o, offset, x);
2791     }
2792 
2793     /** Opaque version of {@link #putCharVolatile(Object, long, char)} */
2794     @IntrinsicCandidate
2795     public final void putCharOpaque(Object o, long offset, char x) {
2796         putCharVolatile(o, offset, x);
2797     }
2798 
2799     /** Opaque version of {@link #putIntVolatile(Object, long, int)} */
2800     @IntrinsicCandidate
2801     public final void putIntOpaque(Object o, long offset, int x) {
2802         putIntVolatile(o, offset, x);
2803     }
2804 
2805     /** Opaque version of {@link #putFloatVolatile(Object, long, float)} */
2806     @IntrinsicCandidate
2807     public final void putFloatOpaque(Object o, long offset, float x) {
2808         putFloatVolatile(o, offset, x);
2809     }
2810 
2811     /** Opaque version of {@link #putLongVolatile(Object, long, long)} */
2812     @IntrinsicCandidate
2813     public final void putLongOpaque(Object o, long offset, long x) {
2814         putLongVolatile(o, offset, x);
2815     }
2816 
2817     /** Opaque version of {@link #putDoubleVolatile(Object, long, double)} */
2818     @IntrinsicCandidate
2819     public final void putDoubleOpaque(Object o, long offset, double x) {
2820         putDoubleVolatile(o, offset, x);
2821     }
2822 
2823     /**
2824      * Unblocks the given thread blocked on {@code park}, or, if it is
2825      * not blocked, causes the subsequent call to {@code park} not to
2826      * block.  Note: this operation is "unsafe" solely because the
2827      * caller must somehow ensure that the thread has not been
2828      * destroyed. Nothing special is usually required to ensure this
2829      * when called from Java (in which there will ordinarily be a live
2830      * reference to the thread) but this is not nearly-automatically
2831      * so when calling from native code.
2832      *
2833      * @param thread the thread to unpark.
2834      */
2835     @IntrinsicCandidate
2836     public native void unpark(Object thread);
2837 
2838     /**
2839      * Blocks current thread, returning when a balancing
2840      * {@code unpark} occurs, or a balancing {@code unpark} has
2841      * already occurred, or the thread is interrupted, or, if not
2842      * absolute and time is not zero, the given time nanoseconds have
2843      * elapsed, or if absolute, the given deadline in milliseconds
2844      * since Epoch has passed, or spuriously (i.e., returning for no
2845      * "reason"). Note: This operation is in the Unsafe class only
2846      * because {@code unpark} is, so it would be strange to place it
2847      * elsewhere.
2848      */
2849     @IntrinsicCandidate
2850     public native void park(boolean isAbsolute, long time);
2851 
2852     /**
2853      * Gets the load average in the system run queue assigned
2854      * to the available processors averaged over various periods of time.
2855      * This method retrieves the given {@code nelem} samples and
2856      * assigns to the elements of the given {@code loadavg} array.
2857      * The system imposes a maximum of 3 samples, representing
2858      * averages over the last 1,  5,  and  15 minutes, respectively.
2859      *
2860      * @param loadavg an array of double of size nelems
2861      * @param nelems the number of samples to be retrieved and
2862      *        must be 1 to 3.
2863      *
2864      * @return the number of samples actually retrieved; or -1
2865      *         if the load average is unobtainable.
2866      */
2867     public int getLoadAverage(double[] loadavg, int nelems) {
2868         if (nelems < 0 || nelems > 3 || nelems > loadavg.length) {
2869             throw new ArrayIndexOutOfBoundsException();
2870         }
2871 
2872         return getLoadAverage0(loadavg, nelems);
2873     }
2874 
2875     // The following contain CAS-based Java implementations used on
2876     // platforms not supporting native instructions
2877 
2878     /**
2879      * Atomically adds the given value to the current value of a field
2880      * or array element within the given object {@code o}
2881      * at the given {@code offset}.
2882      *
2883      * @param o object/array to update the field/element in
2884      * @param offset field/element offset
2885      * @param delta the value to add
2886      * @return the previous value
2887      * @since 1.8
2888      */
2889     @IntrinsicCandidate
2890     public final int getAndAddInt(Object o, long offset, int delta) {
2891         int v;
2892         do {
2893             v = getIntVolatile(o, offset);
2894         } while (!weakCompareAndSetInt(o, offset, v, v + delta));
2895         return v;
2896     }
2897 
2898     @ForceInline
2899     public final int getAndAddIntRelease(Object o, long offset, int delta) {
2900         int v;
2901         do {
2902             v = getInt(o, offset);
2903         } while (!weakCompareAndSetIntRelease(o, offset, v, v + delta));
2904         return v;
2905     }
2906 
2907     @ForceInline
2908     public final int getAndAddIntAcquire(Object o, long offset, int delta) {
2909         int v;
2910         do {
2911             v = getIntAcquire(o, offset);
2912         } while (!weakCompareAndSetIntAcquire(o, offset, v, v + delta));
2913         return v;
2914     }
2915 
2916     /**
2917      * Atomically adds the given value to the current value of a field
2918      * or array element within the given object {@code o}
2919      * at the given {@code offset}.
2920      *
2921      * @param o object/array to update the field/element in
2922      * @param offset field/element offset
2923      * @param delta the value to add
2924      * @return the previous value
2925      * @since 1.8
2926      */
2927     @IntrinsicCandidate
2928     public final long getAndAddLong(Object o, long offset, long delta) {
2929         long v;
2930         do {
2931             v = getLongVolatile(o, offset);
2932         } while (!weakCompareAndSetLong(o, offset, v, v + delta));
2933         return v;
2934     }
2935 
2936     @ForceInline
2937     public final long getAndAddLongRelease(Object o, long offset, long delta) {
2938         long v;
2939         do {
2940             v = getLong(o, offset);
2941         } while (!weakCompareAndSetLongRelease(o, offset, v, v + delta));
2942         return v;
2943     }
2944 
2945     @ForceInline
2946     public final long getAndAddLongAcquire(Object o, long offset, long delta) {
2947         long v;
2948         do {
2949             v = getLongAcquire(o, offset);
2950         } while (!weakCompareAndSetLongAcquire(o, offset, v, v + delta));
2951         return v;
2952     }
2953 
2954     @IntrinsicCandidate
2955     public final byte getAndAddByte(Object o, long offset, byte delta) {
2956         byte v;
2957         do {
2958             v = getByteVolatile(o, offset);
2959         } while (!weakCompareAndSetByte(o, offset, v, (byte) (v + delta)));
2960         return v;
2961     }
2962 
2963     @ForceInline
2964     public final byte getAndAddByteRelease(Object o, long offset, byte delta) {
2965         byte v;
2966         do {
2967             v = getByte(o, offset);
2968         } while (!weakCompareAndSetByteRelease(o, offset, v, (byte) (v + delta)));
2969         return v;
2970     }
2971 
2972     @ForceInline
2973     public final byte getAndAddByteAcquire(Object o, long offset, byte delta) {
2974         byte v;
2975         do {
2976             v = getByteAcquire(o, offset);
2977         } while (!weakCompareAndSetByteAcquire(o, offset, v, (byte) (v + delta)));
2978         return v;
2979     }
2980 
2981     @IntrinsicCandidate
2982     public final short getAndAddShort(Object o, long offset, short delta) {
2983         short v;
2984         do {
2985             v = getShortVolatile(o, offset);
2986         } while (!weakCompareAndSetShort(o, offset, v, (short) (v + delta)));
2987         return v;
2988     }
2989 
2990     @ForceInline
2991     public final short getAndAddShortRelease(Object o, long offset, short delta) {
2992         short v;
2993         do {
2994             v = getShort(o, offset);
2995         } while (!weakCompareAndSetShortRelease(o, offset, v, (short) (v + delta)));
2996         return v;
2997     }
2998 
2999     @ForceInline
3000     public final short getAndAddShortAcquire(Object o, long offset, short delta) {
3001         short v;
3002         do {
3003             v = getShortAcquire(o, offset);
3004         } while (!weakCompareAndSetShortAcquire(o, offset, v, (short) (v + delta)));
3005         return v;
3006     }
3007 
3008     @ForceInline
3009     public final char getAndAddChar(Object o, long offset, char delta) {
3010         return (char) getAndAddShort(o, offset, (short) delta);
3011     }
3012 
3013     @ForceInline
3014     public final char getAndAddCharRelease(Object o, long offset, char delta) {
3015         return (char) getAndAddShortRelease(o, offset, (short) delta);
3016     }
3017 
3018     @ForceInline
3019     public final char getAndAddCharAcquire(Object o, long offset, char delta) {
3020         return (char) getAndAddShortAcquire(o, offset, (short) delta);
3021     }
3022 
3023     @ForceInline
3024     public final float getAndAddFloat(Object o, long offset, float delta) {
3025         int expectedBits;
3026         float v;
3027         do {
3028             // Load and CAS with the raw bits to avoid issues with NaNs and
3029             // possible bit conversion from signaling NaNs to quiet NaNs that
3030             // may result in the loop not terminating.
3031             expectedBits = getIntVolatile(o, offset);
3032             v = Float.intBitsToFloat(expectedBits);
3033         } while (!weakCompareAndSetInt(o, offset,
3034                                                 expectedBits, Float.floatToRawIntBits(v + delta)));
3035         return v;
3036     }
3037 
3038     @ForceInline
3039     public final float getAndAddFloatRelease(Object o, long offset, float delta) {
3040         int expectedBits;
3041         float v;
3042         do {
3043             // Load and CAS with the raw bits to avoid issues with NaNs and
3044             // possible bit conversion from signaling NaNs to quiet NaNs that
3045             // may result in the loop not terminating.
3046             expectedBits = getInt(o, offset);
3047             v = Float.intBitsToFloat(expectedBits);
3048         } while (!weakCompareAndSetIntRelease(o, offset,
3049                                                expectedBits, Float.floatToRawIntBits(v + delta)));
3050         return v;
3051     }
3052 
3053     @ForceInline
3054     public final float getAndAddFloatAcquire(Object o, long offset, float delta) {
3055         int expectedBits;
3056         float v;
3057         do {
3058             // Load and CAS with the raw bits to avoid issues with NaNs and
3059             // possible bit conversion from signaling NaNs to quiet NaNs that
3060             // may result in the loop not terminating.
3061             expectedBits = getIntAcquire(o, offset);
3062             v = Float.intBitsToFloat(expectedBits);
3063         } while (!weakCompareAndSetIntAcquire(o, offset,
3064                                                expectedBits, Float.floatToRawIntBits(v + delta)));
3065         return v;
3066     }
3067 
3068     @ForceInline
3069     public final double getAndAddDouble(Object o, long offset, double delta) {
3070         long expectedBits;
3071         double v;
3072         do {
3073             // Load and CAS with the raw bits to avoid issues with NaNs and
3074             // possible bit conversion from signaling NaNs to quiet NaNs that
3075             // may result in the loop not terminating.
3076             expectedBits = getLongVolatile(o, offset);
3077             v = Double.longBitsToDouble(expectedBits);
3078         } while (!weakCompareAndSetLong(o, offset,
3079                                                  expectedBits, Double.doubleToRawLongBits(v + delta)));
3080         return v;
3081     }
3082 
3083     @ForceInline
3084     public final double getAndAddDoubleRelease(Object o, long offset, double delta) {
3085         long expectedBits;
3086         double v;
3087         do {
3088             // Load and CAS with the raw bits to avoid issues with NaNs and
3089             // possible bit conversion from signaling NaNs to quiet NaNs that
3090             // may result in the loop not terminating.
3091             expectedBits = getLong(o, offset);
3092             v = Double.longBitsToDouble(expectedBits);
3093         } while (!weakCompareAndSetLongRelease(o, offset,
3094                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
3095         return v;
3096     }
3097 
3098     @ForceInline
3099     public final double getAndAddDoubleAcquire(Object o, long offset, double delta) {
3100         long expectedBits;
3101         double v;
3102         do {
3103             // Load and CAS with the raw bits to avoid issues with NaNs and
3104             // possible bit conversion from signaling NaNs to quiet NaNs that
3105             // may result in the loop not terminating.
3106             expectedBits = getLongAcquire(o, offset);
3107             v = Double.longBitsToDouble(expectedBits);
3108         } while (!weakCompareAndSetLongAcquire(o, offset,
3109                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
3110         return v;
3111     }
3112 
3113     /**
3114      * Atomically exchanges the given value with the current value of
3115      * a field or array element within the given object {@code o}
3116      * at the given {@code offset}.
3117      *
3118      * @param o object/array to update the field/element in
3119      * @param offset field/element offset
3120      * @param newValue new value
3121      * @return the previous value
3122      * @since 1.8
3123      */
3124     @IntrinsicCandidate
3125     public final int getAndSetInt(Object o, long offset, int newValue) {
3126         int v;
3127         do {
3128             v = getIntVolatile(o, offset);
3129         } while (!weakCompareAndSetInt(o, offset, v, newValue));
3130         return v;
3131     }
3132 
3133     @ForceInline
3134     public final int getAndSetIntRelease(Object o, long offset, int newValue) {
3135         int v;
3136         do {
3137             v = getInt(o, offset);
3138         } while (!weakCompareAndSetIntRelease(o, offset, v, newValue));
3139         return v;
3140     }
3141 
3142     @ForceInline
3143     public final int getAndSetIntAcquire(Object o, long offset, int newValue) {
3144         int v;
3145         do {
3146             v = getIntAcquire(o, offset);
3147         } while (!weakCompareAndSetIntAcquire(o, offset, v, newValue));
3148         return v;
3149     }
3150 
3151     /**
3152      * Atomically exchanges the given value with the current value of
3153      * a field or array element within the given object {@code o}
3154      * at the given {@code offset}.
3155      *
3156      * @param o object/array to update the field/element in
3157      * @param offset field/element offset
3158      * @param newValue new value
3159      * @return the previous value
3160      * @since 1.8
3161      */
3162     @IntrinsicCandidate
3163     public final long getAndSetLong(Object o, long offset, long newValue) {
3164         long v;
3165         do {
3166             v = getLongVolatile(o, offset);
3167         } while (!weakCompareAndSetLong(o, offset, v, newValue));
3168         return v;
3169     }
3170 
3171     @ForceInline
3172     public final long getAndSetLongRelease(Object o, long offset, long newValue) {
3173         long v;
3174         do {
3175             v = getLong(o, offset);
3176         } while (!weakCompareAndSetLongRelease(o, offset, v, newValue));
3177         return v;
3178     }
3179 
3180     @ForceInline
3181     public final long getAndSetLongAcquire(Object o, long offset, long newValue) {
3182         long v;
3183         do {
3184             v = getLongAcquire(o, offset);
3185         } while (!weakCompareAndSetLongAcquire(o, offset, v, newValue));
3186         return v;
3187     }
3188 
3189     /**
3190      * Atomically exchanges the given reference value with the current
3191      * reference value of a field or array element within the given
3192      * object {@code o} at the given {@code offset}.
3193      *
3194      * @param o object/array to update the field/element in
3195      * @param offset field/element offset
3196      * @param newValue new value
3197      * @return the previous value
3198      * @since 1.8
3199      */
3200     @IntrinsicCandidate
3201     public final Object getAndSetReference(Object o, long offset, Object newValue) {
3202         Object v;
3203         do {
3204             v = getReferenceVolatile(o, offset);
3205         } while (!weakCompareAndSetReference(o, offset, v, newValue));
3206         return v;
3207     }
3208 
3209     @SuppressWarnings("unchecked")
3210     public final <V> Object getAndSetFlatValue(Object o, long offset, int layout, Class<?> valueType, V newValue) {
3211         synchronized (valueLock) {
3212             Object oldValue = getFlatValue(o, offset, layout, valueType);
3213             putFlatValue(o, offset, layout, valueType, newValue);
3214             return oldValue;
3215         }
3216     }
3217 
3218     @ForceInline
3219     public final Object getAndSetReferenceRelease(Object o, long offset, Object newValue) {
3220         Object v;
3221         do {
3222             v = getReference(o, offset);
3223         } while (!weakCompareAndSetReferenceRelease(o, offset, v, newValue));
3224         return v;
3225     }
3226 
3227     @ForceInline
3228     public final <V> Object getAndSetFlatValueRelease(Object o, long offset, int layout, Class<?> valueType, V newValue) {
3229         return getAndSetFlatValue(o, offset, layout, valueType, newValue);
3230     }
3231 
3232     @ForceInline
3233     public final Object getAndSetReferenceAcquire(Object o, long offset, Object newValue) {
3234         Object v;
3235         do {
3236             v = getReferenceAcquire(o, offset);
3237         } while (!weakCompareAndSetReferenceAcquire(o, offset, v, newValue));
3238         return v;
3239     }
3240 
3241     @ForceInline
3242     public final <V> Object getAndSetFlatValueAcquire(Object o, long offset, int layout, Class<?> valueType, V newValue) {
3243         return getAndSetFlatValue(o, offset, layout, valueType, newValue);
3244     }
3245 
3246     @IntrinsicCandidate
3247     public final byte getAndSetByte(Object o, long offset, byte newValue) {
3248         byte v;
3249         do {
3250             v = getByteVolatile(o, offset);
3251         } while (!weakCompareAndSetByte(o, offset, v, newValue));
3252         return v;
3253     }
3254 
3255     @ForceInline
3256     public final byte getAndSetByteRelease(Object o, long offset, byte newValue) {
3257         byte v;
3258         do {
3259             v = getByte(o, offset);
3260         } while (!weakCompareAndSetByteRelease(o, offset, v, newValue));
3261         return v;
3262     }
3263 
3264     @ForceInline
3265     public final byte getAndSetByteAcquire(Object o, long offset, byte newValue) {
3266         byte v;
3267         do {
3268             v = getByteAcquire(o, offset);
3269         } while (!weakCompareAndSetByteAcquire(o, offset, v, newValue));
3270         return v;
3271     }
3272 
3273     @ForceInline
3274     public final boolean getAndSetBoolean(Object o, long offset, boolean newValue) {
3275         return byte2bool(getAndSetByte(o, offset, bool2byte(newValue)));
3276     }
3277 
3278     @ForceInline
3279     public final boolean getAndSetBooleanRelease(Object o, long offset, boolean newValue) {
3280         return byte2bool(getAndSetByteRelease(o, offset, bool2byte(newValue)));
3281     }
3282 
3283     @ForceInline
3284     public final boolean getAndSetBooleanAcquire(Object o, long offset, boolean newValue) {
3285         return byte2bool(getAndSetByteAcquire(o, offset, bool2byte(newValue)));
3286     }
3287 
3288     @IntrinsicCandidate
3289     public final short getAndSetShort(Object o, long offset, short newValue) {
3290         short v;
3291         do {
3292             v = getShortVolatile(o, offset);
3293         } while (!weakCompareAndSetShort(o, offset, v, newValue));
3294         return v;
3295     }
3296 
3297     @ForceInline
3298     public final short getAndSetShortRelease(Object o, long offset, short newValue) {
3299         short v;
3300         do {
3301             v = getShort(o, offset);
3302         } while (!weakCompareAndSetShortRelease(o, offset, v, newValue));
3303         return v;
3304     }
3305 
3306     @ForceInline
3307     public final short getAndSetShortAcquire(Object o, long offset, short newValue) {
3308         short v;
3309         do {
3310             v = getShortAcquire(o, offset);
3311         } while (!weakCompareAndSetShortAcquire(o, offset, v, newValue));
3312         return v;
3313     }
3314 
3315     @ForceInline
3316     public final char getAndSetChar(Object o, long offset, char newValue) {
3317         return s2c(getAndSetShort(o, offset, c2s(newValue)));
3318     }
3319 
3320     @ForceInline
3321     public final char getAndSetCharRelease(Object o, long offset, char newValue) {
3322         return s2c(getAndSetShortRelease(o, offset, c2s(newValue)));
3323     }
3324 
3325     @ForceInline
3326     public final char getAndSetCharAcquire(Object o, long offset, char newValue) {
3327         return s2c(getAndSetShortAcquire(o, offset, c2s(newValue)));
3328     }
3329 
3330     @ForceInline
3331     public final float getAndSetFloat(Object o, long offset, float newValue) {
3332         int v = getAndSetInt(o, offset, Float.floatToRawIntBits(newValue));
3333         return Float.intBitsToFloat(v);
3334     }
3335 
3336     @ForceInline
3337     public final float getAndSetFloatRelease(Object o, long offset, float newValue) {
3338         int v = getAndSetIntRelease(o, offset, Float.floatToRawIntBits(newValue));
3339         return Float.intBitsToFloat(v);
3340     }
3341 
3342     @ForceInline
3343     public final float getAndSetFloatAcquire(Object o, long offset, float newValue) {
3344         int v = getAndSetIntAcquire(o, offset, Float.floatToRawIntBits(newValue));
3345         return Float.intBitsToFloat(v);
3346     }
3347 
3348     @ForceInline
3349     public final double getAndSetDouble(Object o, long offset, double newValue) {
3350         long v = getAndSetLong(o, offset, Double.doubleToRawLongBits(newValue));
3351         return Double.longBitsToDouble(v);
3352     }
3353 
3354     @ForceInline
3355     public final double getAndSetDoubleRelease(Object o, long offset, double newValue) {
3356         long v = getAndSetLongRelease(o, offset, Double.doubleToRawLongBits(newValue));
3357         return Double.longBitsToDouble(v);
3358     }
3359 
3360     @ForceInline
3361     public final double getAndSetDoubleAcquire(Object o, long offset, double newValue) {
3362         long v = getAndSetLongAcquire(o, offset, Double.doubleToRawLongBits(newValue));
3363         return Double.longBitsToDouble(v);
3364     }
3365 
3366 
3367     // The following contain CAS-based Java implementations used on
3368     // platforms not supporting native instructions
3369 
3370     @ForceInline
3371     public final boolean getAndBitwiseOrBoolean(Object o, long offset, boolean mask) {
3372         return byte2bool(getAndBitwiseOrByte(o, offset, bool2byte(mask)));
3373     }
3374 
3375     @ForceInline
3376     public final boolean getAndBitwiseOrBooleanRelease(Object o, long offset, boolean mask) {
3377         return byte2bool(getAndBitwiseOrByteRelease(o, offset, bool2byte(mask)));
3378     }
3379 
3380     @ForceInline
3381     public final boolean getAndBitwiseOrBooleanAcquire(Object o, long offset, boolean mask) {
3382         return byte2bool(getAndBitwiseOrByteAcquire(o, offset, bool2byte(mask)));
3383     }
3384 
3385     @ForceInline
3386     public final boolean getAndBitwiseAndBoolean(Object o, long offset, boolean mask) {
3387         return byte2bool(getAndBitwiseAndByte(o, offset, bool2byte(mask)));
3388     }
3389 
3390     @ForceInline
3391     public final boolean getAndBitwiseAndBooleanRelease(Object o, long offset, boolean mask) {
3392         return byte2bool(getAndBitwiseAndByteRelease(o, offset, bool2byte(mask)));
3393     }
3394 
3395     @ForceInline
3396     public final boolean getAndBitwiseAndBooleanAcquire(Object o, long offset, boolean mask) {
3397         return byte2bool(getAndBitwiseAndByteAcquire(o, offset, bool2byte(mask)));
3398     }
3399 
3400     @ForceInline
3401     public final boolean getAndBitwiseXorBoolean(Object o, long offset, boolean mask) {
3402         return byte2bool(getAndBitwiseXorByte(o, offset, bool2byte(mask)));
3403     }
3404 
3405     @ForceInline
3406     public final boolean getAndBitwiseXorBooleanRelease(Object o, long offset, boolean mask) {
3407         return byte2bool(getAndBitwiseXorByteRelease(o, offset, bool2byte(mask)));
3408     }
3409 
3410     @ForceInline
3411     public final boolean getAndBitwiseXorBooleanAcquire(Object o, long offset, boolean mask) {
3412         return byte2bool(getAndBitwiseXorByteAcquire(o, offset, bool2byte(mask)));
3413     }
3414 
3415 
3416     @ForceInline
3417     public final byte getAndBitwiseOrByte(Object o, long offset, byte mask) {
3418         byte current;
3419         do {
3420             current = getByteVolatile(o, offset);
3421         } while (!weakCompareAndSetByte(o, offset,
3422                                                   current, (byte) (current | mask)));
3423         return current;
3424     }
3425 
3426     @ForceInline
3427     public final byte getAndBitwiseOrByteRelease(Object o, long offset, byte mask) {
3428         byte current;
3429         do {
3430             current = getByte(o, offset);
3431         } while (!weakCompareAndSetByteRelease(o, offset,
3432                                                  current, (byte) (current | mask)));
3433         return current;
3434     }
3435 
3436     @ForceInline
3437     public final byte getAndBitwiseOrByteAcquire(Object o, long offset, byte mask) {
3438         byte current;
3439         do {
3440             // Plain read, the value is a hint, the acquire CAS does the work
3441             current = getByte(o, offset);
3442         } while (!weakCompareAndSetByteAcquire(o, offset,
3443                                                  current, (byte) (current | mask)));
3444         return current;
3445     }
3446 
3447     @ForceInline
3448     public final byte getAndBitwiseAndByte(Object o, long offset, byte mask) {
3449         byte current;
3450         do {
3451             current = getByteVolatile(o, offset);
3452         } while (!weakCompareAndSetByte(o, offset,
3453                                                   current, (byte) (current & mask)));
3454         return current;
3455     }
3456 
3457     @ForceInline
3458     public final byte getAndBitwiseAndByteRelease(Object o, long offset, byte mask) {
3459         byte current;
3460         do {
3461             current = getByte(o, offset);
3462         } while (!weakCompareAndSetByteRelease(o, offset,
3463                                                  current, (byte) (current & mask)));
3464         return current;
3465     }
3466 
3467     @ForceInline
3468     public final byte getAndBitwiseAndByteAcquire(Object o, long offset, byte mask) {
3469         byte current;
3470         do {
3471             // Plain read, the value is a hint, the acquire CAS does the work
3472             current = getByte(o, offset);
3473         } while (!weakCompareAndSetByteAcquire(o, offset,
3474                                                  current, (byte) (current & mask)));
3475         return current;
3476     }
3477 
3478     @ForceInline
3479     public final byte getAndBitwiseXorByte(Object o, long offset, byte mask) {
3480         byte current;
3481         do {
3482             current = getByteVolatile(o, offset);
3483         } while (!weakCompareAndSetByte(o, offset,
3484                                                   current, (byte) (current ^ mask)));
3485         return current;
3486     }
3487 
3488     @ForceInline
3489     public final byte getAndBitwiseXorByteRelease(Object o, long offset, byte mask) {
3490         byte current;
3491         do {
3492             current = getByte(o, offset);
3493         } while (!weakCompareAndSetByteRelease(o, offset,
3494                                                  current, (byte) (current ^ mask)));
3495         return current;
3496     }
3497 
3498     @ForceInline
3499     public final byte getAndBitwiseXorByteAcquire(Object o, long offset, byte mask) {
3500         byte current;
3501         do {
3502             // Plain read, the value is a hint, the acquire CAS does the work
3503             current = getByte(o, offset);
3504         } while (!weakCompareAndSetByteAcquire(o, offset,
3505                                                  current, (byte) (current ^ mask)));
3506         return current;
3507     }
3508 
3509 
3510     @ForceInline
3511     public final char getAndBitwiseOrChar(Object o, long offset, char mask) {
3512         return s2c(getAndBitwiseOrShort(o, offset, c2s(mask)));
3513     }
3514 
3515     @ForceInline
3516     public final char getAndBitwiseOrCharRelease(Object o, long offset, char mask) {
3517         return s2c(getAndBitwiseOrShortRelease(o, offset, c2s(mask)));
3518     }
3519 
3520     @ForceInline
3521     public final char getAndBitwiseOrCharAcquire(Object o, long offset, char mask) {
3522         return s2c(getAndBitwiseOrShortAcquire(o, offset, c2s(mask)));
3523     }
3524 
3525     @ForceInline
3526     public final char getAndBitwiseAndChar(Object o, long offset, char mask) {
3527         return s2c(getAndBitwiseAndShort(o, offset, c2s(mask)));
3528     }
3529 
3530     @ForceInline
3531     public final char getAndBitwiseAndCharRelease(Object o, long offset, char mask) {
3532         return s2c(getAndBitwiseAndShortRelease(o, offset, c2s(mask)));
3533     }
3534 
3535     @ForceInline
3536     public final char getAndBitwiseAndCharAcquire(Object o, long offset, char mask) {
3537         return s2c(getAndBitwiseAndShortAcquire(o, offset, c2s(mask)));
3538     }
3539 
3540     @ForceInline
3541     public final char getAndBitwiseXorChar(Object o, long offset, char mask) {
3542         return s2c(getAndBitwiseXorShort(o, offset, c2s(mask)));
3543     }
3544 
3545     @ForceInline
3546     public final char getAndBitwiseXorCharRelease(Object o, long offset, char mask) {
3547         return s2c(getAndBitwiseXorShortRelease(o, offset, c2s(mask)));
3548     }
3549 
3550     @ForceInline
3551     public final char getAndBitwiseXorCharAcquire(Object o, long offset, char mask) {
3552         return s2c(getAndBitwiseXorShortAcquire(o, offset, c2s(mask)));
3553     }
3554 
3555 
3556     @ForceInline
3557     public final short getAndBitwiseOrShort(Object o, long offset, short mask) {
3558         short current;
3559         do {
3560             current = getShortVolatile(o, offset);
3561         } while (!weakCompareAndSetShort(o, offset,
3562                                                 current, (short) (current | mask)));
3563         return current;
3564     }
3565 
3566     @ForceInline
3567     public final short getAndBitwiseOrShortRelease(Object o, long offset, short mask) {
3568         short current;
3569         do {
3570             current = getShort(o, offset);
3571         } while (!weakCompareAndSetShortRelease(o, offset,
3572                                                current, (short) (current | mask)));
3573         return current;
3574     }
3575 
3576     @ForceInline
3577     public final short getAndBitwiseOrShortAcquire(Object o, long offset, short mask) {
3578         short current;
3579         do {
3580             // Plain read, the value is a hint, the acquire CAS does the work
3581             current = getShort(o, offset);
3582         } while (!weakCompareAndSetShortAcquire(o, offset,
3583                                                current, (short) (current | mask)));
3584         return current;
3585     }
3586 
3587     @ForceInline
3588     public final short getAndBitwiseAndShort(Object o, long offset, short mask) {
3589         short current;
3590         do {
3591             current = getShortVolatile(o, offset);
3592         } while (!weakCompareAndSetShort(o, offset,
3593                                                 current, (short) (current & mask)));
3594         return current;
3595     }
3596 
3597     @ForceInline
3598     public final short getAndBitwiseAndShortRelease(Object o, long offset, short mask) {
3599         short current;
3600         do {
3601             current = getShort(o, offset);
3602         } while (!weakCompareAndSetShortRelease(o, offset,
3603                                                current, (short) (current & mask)));
3604         return current;
3605     }
3606 
3607     @ForceInline
3608     public final short getAndBitwiseAndShortAcquire(Object o, long offset, short mask) {
3609         short current;
3610         do {
3611             // Plain read, the value is a hint, the acquire CAS does the work
3612             current = getShort(o, offset);
3613         } while (!weakCompareAndSetShortAcquire(o, offset,
3614                                                current, (short) (current & mask)));
3615         return current;
3616     }
3617 
3618     @ForceInline
3619     public final short getAndBitwiseXorShort(Object o, long offset, short mask) {
3620         short current;
3621         do {
3622             current = getShortVolatile(o, offset);
3623         } while (!weakCompareAndSetShort(o, offset,
3624                                                 current, (short) (current ^ mask)));
3625         return current;
3626     }
3627 
3628     @ForceInline
3629     public final short getAndBitwiseXorShortRelease(Object o, long offset, short mask) {
3630         short current;
3631         do {
3632             current = getShort(o, offset);
3633         } while (!weakCompareAndSetShortRelease(o, offset,
3634                                                current, (short) (current ^ mask)));
3635         return current;
3636     }
3637 
3638     @ForceInline
3639     public final short getAndBitwiseXorShortAcquire(Object o, long offset, short mask) {
3640         short current;
3641         do {
3642             // Plain read, the value is a hint, the acquire CAS does the work
3643             current = getShort(o, offset);
3644         } while (!weakCompareAndSetShortAcquire(o, offset,
3645                                                current, (short) (current ^ mask)));
3646         return current;
3647     }
3648 
3649 
3650     @ForceInline
3651     public final int getAndBitwiseOrInt(Object o, long offset, int mask) {
3652         int current;
3653         do {
3654             current = getIntVolatile(o, offset);
3655         } while (!weakCompareAndSetInt(o, offset,
3656                                                 current, current | mask));
3657         return current;
3658     }
3659 
3660     @ForceInline
3661     public final int getAndBitwiseOrIntRelease(Object o, long offset, int mask) {
3662         int current;
3663         do {
3664             current = getInt(o, offset);
3665         } while (!weakCompareAndSetIntRelease(o, offset,
3666                                                current, current | mask));
3667         return current;
3668     }
3669 
3670     @ForceInline
3671     public final int getAndBitwiseOrIntAcquire(Object o, long offset, int mask) {
3672         int current;
3673         do {
3674             // Plain read, the value is a hint, the acquire CAS does the work
3675             current = getInt(o, offset);
3676         } while (!weakCompareAndSetIntAcquire(o, offset,
3677                                                current, current | mask));
3678         return current;
3679     }
3680 
3681     /**
3682      * Atomically replaces the current value of a field or array element within
3683      * the given object with the result of bitwise AND between the current value
3684      * and mask.
3685      *
3686      * @param o object/array to update the field/element in
3687      * @param offset field/element offset
3688      * @param mask the mask value
3689      * @return the previous value
3690      * @since 9
3691      */
3692     @ForceInline
3693     public final int getAndBitwiseAndInt(Object o, long offset, int mask) {
3694         int current;
3695         do {
3696             current = getIntVolatile(o, offset);
3697         } while (!weakCompareAndSetInt(o, offset,
3698                                                 current, current & mask));
3699         return current;
3700     }
3701 
3702     @ForceInline
3703     public final int getAndBitwiseAndIntRelease(Object o, long offset, int mask) {
3704         int current;
3705         do {
3706             current = getInt(o, offset);
3707         } while (!weakCompareAndSetIntRelease(o, offset,
3708                                                current, current & mask));
3709         return current;
3710     }
3711 
3712     @ForceInline
3713     public final int getAndBitwiseAndIntAcquire(Object o, long offset, int mask) {
3714         int current;
3715         do {
3716             // Plain read, the value is a hint, the acquire CAS does the work
3717             current = getInt(o, offset);
3718         } while (!weakCompareAndSetIntAcquire(o, offset,
3719                                                current, current & mask));
3720         return current;
3721     }
3722 
3723     @ForceInline
3724     public final int getAndBitwiseXorInt(Object o, long offset, int mask) {
3725         int current;
3726         do {
3727             current = getIntVolatile(o, offset);
3728         } while (!weakCompareAndSetInt(o, offset,
3729                                                 current, current ^ mask));
3730         return current;
3731     }
3732 
3733     @ForceInline
3734     public final int getAndBitwiseXorIntRelease(Object o, long offset, int mask) {
3735         int current;
3736         do {
3737             current = getInt(o, offset);
3738         } while (!weakCompareAndSetIntRelease(o, offset,
3739                                                current, current ^ mask));
3740         return current;
3741     }
3742 
3743     @ForceInline
3744     public final int getAndBitwiseXorIntAcquire(Object o, long offset, int mask) {
3745         int current;
3746         do {
3747             // Plain read, the value is a hint, the acquire CAS does the work
3748             current = getInt(o, offset);
3749         } while (!weakCompareAndSetIntAcquire(o, offset,
3750                                                current, current ^ mask));
3751         return current;
3752     }
3753 
3754 
3755     @ForceInline
3756     public final long getAndBitwiseOrLong(Object o, long offset, long mask) {
3757         long current;
3758         do {
3759             current = getLongVolatile(o, offset);
3760         } while (!weakCompareAndSetLong(o, offset,
3761                                                 current, current | mask));
3762         return current;
3763     }
3764 
3765     @ForceInline
3766     public final long getAndBitwiseOrLongRelease(Object o, long offset, long mask) {
3767         long current;
3768         do {
3769             current = getLong(o, offset);
3770         } while (!weakCompareAndSetLongRelease(o, offset,
3771                                                current, current | mask));
3772         return current;
3773     }
3774 
3775     @ForceInline
3776     public final long getAndBitwiseOrLongAcquire(Object o, long offset, long mask) {
3777         long current;
3778         do {
3779             // Plain read, the value is a hint, the acquire CAS does the work
3780             current = getLong(o, offset);
3781         } while (!weakCompareAndSetLongAcquire(o, offset,
3782                                                current, current | mask));
3783         return current;
3784     }
3785 
3786     @ForceInline
3787     public final long getAndBitwiseAndLong(Object o, long offset, long mask) {
3788         long current;
3789         do {
3790             current = getLongVolatile(o, offset);
3791         } while (!weakCompareAndSetLong(o, offset,
3792                                                 current, current & mask));
3793         return current;
3794     }
3795 
3796     @ForceInline
3797     public final long getAndBitwiseAndLongRelease(Object o, long offset, long mask) {
3798         long current;
3799         do {
3800             current = getLong(o, offset);
3801         } while (!weakCompareAndSetLongRelease(o, offset,
3802                                                current, current & mask));
3803         return current;
3804     }
3805 
3806     @ForceInline
3807     public final long getAndBitwiseAndLongAcquire(Object o, long offset, long mask) {
3808         long current;
3809         do {
3810             // Plain read, the value is a hint, the acquire CAS does the work
3811             current = getLong(o, offset);
3812         } while (!weakCompareAndSetLongAcquire(o, offset,
3813                                                current, current & mask));
3814         return current;
3815     }
3816 
3817     @ForceInline
3818     public final long getAndBitwiseXorLong(Object o, long offset, long mask) {
3819         long current;
3820         do {
3821             current = getLongVolatile(o, offset);
3822         } while (!weakCompareAndSetLong(o, offset,
3823                                                 current, current ^ mask));
3824         return current;
3825     }
3826 
3827     @ForceInline
3828     public final long getAndBitwiseXorLongRelease(Object o, long offset, long mask) {
3829         long current;
3830         do {
3831             current = getLong(o, offset);
3832         } while (!weakCompareAndSetLongRelease(o, offset,
3833                                                current, current ^ mask));
3834         return current;
3835     }
3836 
3837     @ForceInline
3838     public final long getAndBitwiseXorLongAcquire(Object o, long offset, long mask) {
3839         long current;
3840         do {
3841             // Plain read, the value is a hint, the acquire CAS does the work
3842             current = getLong(o, offset);
3843         } while (!weakCompareAndSetLongAcquire(o, offset,
3844                                                current, current ^ mask));
3845         return current;
3846     }
3847 
3848 
3849 
3850     /**
3851      * Ensures that loads before the fence will not be reordered with loads and
3852      * stores after the fence; a "LoadLoad plus LoadStore barrier".
3853      *
3854      * Corresponds to C11 atomic_thread_fence(memory_order_acquire)
3855      * (an "acquire fence").
3856      *
3857      * Provides a LoadLoad barrier followed by a LoadStore barrier.
3858      *
3859      * @since 1.8
3860      */
3861     @IntrinsicCandidate
3862     public final void loadFence() {
3863         // If loadFence intrinsic is not available, fall back to full fence.
3864         fullFence();
3865     }
3866 
3867     /**
3868      * Ensures that loads and stores before the fence will not be reordered with
3869      * stores after the fence; a "StoreStore plus LoadStore barrier".
3870      *
3871      * Corresponds to C11 atomic_thread_fence(memory_order_release)
3872      * (a "release fence").
3873      *
3874      * Provides a StoreStore barrier followed by a LoadStore barrier.
3875      *
3876      * @since 1.8
3877      */
3878     @IntrinsicCandidate
3879     public final void storeFence() {
3880         // If storeFence intrinsic is not available, fall back to full fence.
3881         fullFence();
3882     }
3883 
3884     /**
3885      * Ensures that loads and stores before the fence will not be reordered
3886      * with loads and stores after the fence.  Implies the effects of both
3887      * loadFence() and storeFence(), and in addition, the effect of a StoreLoad
3888      * barrier.
3889      *
3890      * Corresponds to C11 atomic_thread_fence(memory_order_seq_cst).
3891      * @since 1.8
3892      */
3893     @IntrinsicCandidate
3894     public native void fullFence();
3895 
3896     /**
3897      * Ensures that loads before the fence will not be reordered with
3898      * loads after the fence.
3899      *
3900      * @implNote
3901      * This method is operationally equivalent to {@link #loadFence()}.
3902      *
3903      * @since 9
3904      */
3905     public final void loadLoadFence() {
3906         loadFence();
3907     }
3908 
3909     /**
3910      * Ensures that stores before the fence will not be reordered with
3911      * stores after the fence.
3912      *
3913      * @since 9
3914      */
3915     @IntrinsicCandidate
3916     public final void storeStoreFence() {
3917         // If storeStoreFence intrinsic is not available, fall back to storeFence.
3918         storeFence();
3919     }
3920 
3921     /**
3922      * Throws IllegalAccessError; for use by the VM for access control
3923      * error support.
3924      * @since 1.8
3925      */
3926     private static void throwIllegalAccessError() {
3927         throw new IllegalAccessError();
3928     }
3929 
3930     /**
3931      * Throws NoSuchMethodError; for use by the VM for redefinition support.
3932      * @since 13
3933      */
3934     private static void throwNoSuchMethodError() {
3935         throw new NoSuchMethodError();
3936     }
3937 
3938     /**
3939      * @return Returns true if the native byte ordering of this
3940      * platform is big-endian, false if it is little-endian.
3941      */
3942     public final boolean isBigEndian() { return BIG_ENDIAN; }
3943 
3944     /**
3945      * @return Returns true if this platform is capable of performing
3946      * accesses at addresses which are not aligned for the type of the
3947      * primitive type being accessed, false otherwise.
3948      */
3949     public final boolean unalignedAccess() { return UNALIGNED_ACCESS; }
3950 
3951     /**
3952      * Fetches a value at some byte offset into a given Java object.
3953      * More specifically, fetches a value within the given object
3954      * <code>o</code> at the given offset, or (if <code>o</code> is
3955      * null) from the memory address whose numerical value is the
3956      * given offset.  <p>
3957      *
3958      * The specification of this method is the same as {@link
3959      * #getLong(Object, long)} except that the offset does not need to
3960      * have been obtained from {@link #objectFieldOffset} on the
3961      * {@link java.lang.reflect.Field} of some Java field.  The value
3962      * in memory is raw data, and need not correspond to any Java
3963      * variable.  Unless <code>o</code> is null, the value accessed
3964      * must be entirely within the allocated object.  The endianness
3965      * of the value in memory is the endianness of the native platform.
3966      *
3967      * <p> The read will be atomic with respect to the largest power
3968      * of two that divides the GCD of the offset and the storage size.
3969      * For example, getLongUnaligned will make atomic reads of 2-, 4-,
3970      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
3971      * respectively.  There are no other guarantees of atomicity.
3972      * <p>
3973      * 8-byte atomicity is only guaranteed on platforms on which
3974      * support atomic accesses to longs.
3975      *
3976      * @param o Java heap object in which the value resides, if any, else
3977      *        null
3978      * @param offset The offset in bytes from the start of the object
3979      * @return the value fetched from the indicated object
3980      * @throws RuntimeException No defined exceptions are thrown, not even
3981      *         {@link NullPointerException}
3982      * @since 9
3983      */
3984     @IntrinsicCandidate
3985     public final long getLongUnaligned(Object o, long offset) {
3986         if ((offset & 7) == 0) {
3987             return getLong(o, offset);
3988         } else if ((offset & 3) == 0) {
3989             return makeLong(getInt(o, offset),
3990                             getInt(o, offset + 4));
3991         } else if ((offset & 1) == 0) {
3992             return makeLong(getShort(o, offset),
3993                             getShort(o, offset + 2),
3994                             getShort(o, offset + 4),
3995                             getShort(o, offset + 6));
3996         } else {
3997             return makeLong(getByte(o, offset),
3998                             getByte(o, offset + 1),
3999                             getByte(o, offset + 2),
4000                             getByte(o, offset + 3),
4001                             getByte(o, offset + 4),
4002                             getByte(o, offset + 5),
4003                             getByte(o, offset + 6),
4004                             getByte(o, offset + 7));
4005         }
4006     }
4007     /**
4008      * As {@link #getLongUnaligned(Object, long)} but with an
4009      * additional argument which specifies the endianness of the value
4010      * as stored in memory.
4011      *
4012      * @param o Java heap object in which the variable resides
4013      * @param offset The offset in bytes from the start of the object
4014      * @param bigEndian The endianness of the value
4015      * @return the value fetched from the indicated object
4016      * @since 9
4017      */
4018     public final long getLongUnaligned(Object o, long offset, boolean bigEndian) {
4019         return convEndian(bigEndian, getLongUnaligned(o, offset));
4020     }
4021 
4022     /** @see #getLongUnaligned(Object, long) */
4023     @IntrinsicCandidate
4024     public final int getIntUnaligned(Object o, long offset) {
4025         if ((offset & 3) == 0) {
4026             return getInt(o, offset);
4027         } else if ((offset & 1) == 0) {
4028             return makeInt(getShort(o, offset),
4029                            getShort(o, offset + 2));
4030         } else {
4031             return makeInt(getByte(o, offset),
4032                            getByte(o, offset + 1),
4033                            getByte(o, offset + 2),
4034                            getByte(o, offset + 3));
4035         }
4036     }
4037     /** @see #getLongUnaligned(Object, long, boolean) */
4038     public final int getIntUnaligned(Object o, long offset, boolean bigEndian) {
4039         return convEndian(bigEndian, getIntUnaligned(o, offset));
4040     }
4041 
4042     /** @see #getLongUnaligned(Object, long) */
4043     @IntrinsicCandidate
4044     public final short getShortUnaligned(Object o, long offset) {
4045         if ((offset & 1) == 0) {
4046             return getShort(o, offset);
4047         } else {
4048             return makeShort(getByte(o, offset),
4049                              getByte(o, offset + 1));
4050         }
4051     }
4052     /** @see #getLongUnaligned(Object, long, boolean) */
4053     public final short getShortUnaligned(Object o, long offset, boolean bigEndian) {
4054         return convEndian(bigEndian, getShortUnaligned(o, offset));
4055     }
4056 
4057     /** @see #getLongUnaligned(Object, long) */
4058     @IntrinsicCandidate
4059     public final char getCharUnaligned(Object o, long offset) {
4060         if ((offset & 1) == 0) {
4061             return getChar(o, offset);
4062         } else {
4063             return (char)makeShort(getByte(o, offset),
4064                                    getByte(o, offset + 1));
4065         }
4066     }
4067 
4068     /** @see #getLongUnaligned(Object, long, boolean) */
4069     public final char getCharUnaligned(Object o, long offset, boolean bigEndian) {
4070         return convEndian(bigEndian, getCharUnaligned(o, offset));
4071     }
4072 
4073     /**
4074      * Stores a value at some byte offset into a given Java object.
4075      * <p>
4076      * The specification of this method is the same as {@link
4077      * #getLong(Object, long)} except that the offset does not need to
4078      * have been obtained from {@link #objectFieldOffset} on the
4079      * {@link java.lang.reflect.Field} of some Java field.  The value
4080      * in memory is raw data, and need not correspond to any Java
4081      * variable.  The endianness of the value in memory is the
4082      * endianness of the native platform.
4083      * <p>
4084      * The write will be atomic with respect to the largest power of
4085      * two that divides the GCD of the offset and the storage size.
4086      * For example, putLongUnaligned will make atomic writes of 2-, 4-,
4087      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
4088      * respectively.  There are no other guarantees of atomicity.
4089      * <p>
4090      * 8-byte atomicity is only guaranteed on platforms on which
4091      * support atomic accesses to longs.
4092      *
4093      * @param o Java heap object in which the value resides, if any, else
4094      *        null
4095      * @param offset The offset in bytes from the start of the object
4096      * @param x the value to store
4097      * @throws RuntimeException No defined exceptions are thrown, not even
4098      *         {@link NullPointerException}
4099      * @since 9
4100      */
4101     @IntrinsicCandidate
4102     public final void putLongUnaligned(Object o, long offset, long x) {
4103         if ((offset & 7) == 0) {
4104             putLong(o, offset, x);
4105         } else if ((offset & 3) == 0) {
4106             putLongParts(o, offset,
4107                          (int)(x >> 0),
4108                          (int)(x >>> 32));
4109         } else if ((offset & 1) == 0) {
4110             putLongParts(o, offset,
4111                          (short)(x >>> 0),
4112                          (short)(x >>> 16),
4113                          (short)(x >>> 32),
4114                          (short)(x >>> 48));
4115         } else {
4116             putLongParts(o, offset,
4117                          (byte)(x >>> 0),
4118                          (byte)(x >>> 8),
4119                          (byte)(x >>> 16),
4120                          (byte)(x >>> 24),
4121                          (byte)(x >>> 32),
4122                          (byte)(x >>> 40),
4123                          (byte)(x >>> 48),
4124                          (byte)(x >>> 56));
4125         }
4126     }
4127 
4128     /**
4129      * As {@link #putLongUnaligned(Object, long, long)} but with an additional
4130      * argument which specifies the endianness of the value as stored in memory.
4131      * @param o Java heap object in which the value resides
4132      * @param offset The offset in bytes from the start of the object
4133      * @param x the value to store
4134      * @param bigEndian The endianness of the value
4135      * @throws RuntimeException No defined exceptions are thrown, not even
4136      *         {@link NullPointerException}
4137      * @since 9
4138      */
4139     public final void putLongUnaligned(Object o, long offset, long x, boolean bigEndian) {
4140         putLongUnaligned(o, offset, convEndian(bigEndian, x));
4141     }
4142 
4143     /** @see #putLongUnaligned(Object, long, long) */
4144     @IntrinsicCandidate
4145     public final void putIntUnaligned(Object o, long offset, int x) {
4146         if ((offset & 3) == 0) {
4147             putInt(o, offset, x);
4148         } else if ((offset & 1) == 0) {
4149             putIntParts(o, offset,
4150                         (short)(x >> 0),
4151                         (short)(x >>> 16));
4152         } else {
4153             putIntParts(o, offset,
4154                         (byte)(x >>> 0),
4155                         (byte)(x >>> 8),
4156                         (byte)(x >>> 16),
4157                         (byte)(x >>> 24));
4158         }
4159     }
4160     /** @see #putLongUnaligned(Object, long, long, boolean) */
4161     public final void putIntUnaligned(Object o, long offset, int x, boolean bigEndian) {
4162         putIntUnaligned(o, offset, convEndian(bigEndian, x));
4163     }
4164 
4165     /** @see #putLongUnaligned(Object, long, long) */
4166     @IntrinsicCandidate
4167     public final void putShortUnaligned(Object o, long offset, short x) {
4168         if ((offset & 1) == 0) {
4169             putShort(o, offset, x);
4170         } else {
4171             putShortParts(o, offset,
4172                           (byte)(x >>> 0),
4173                           (byte)(x >>> 8));
4174         }
4175     }
4176     /** @see #putLongUnaligned(Object, long, long, boolean) */
4177     public final void putShortUnaligned(Object o, long offset, short x, boolean bigEndian) {
4178         putShortUnaligned(o, offset, convEndian(bigEndian, x));
4179     }
4180 
4181     /** @see #putLongUnaligned(Object, long, long) */
4182     @IntrinsicCandidate
4183     public final void putCharUnaligned(Object o, long offset, char x) {
4184         putShortUnaligned(o, offset, (short)x);
4185     }
4186     /** @see #putLongUnaligned(Object, long, long, boolean) */
4187     public final void putCharUnaligned(Object o, long offset, char x, boolean bigEndian) {
4188         putCharUnaligned(o, offset, convEndian(bigEndian, x));
4189     }
4190 
4191     private static int pickPos(int top, int pos) { return BIG_ENDIAN ? top - pos : pos; }
4192 
4193     // These methods construct integers from bytes.  The byte ordering
4194     // is the native endianness of this platform.
4195     private static long makeLong(byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
4196         return ((toUnsignedLong(i0) << pickPos(56, 0))
4197               | (toUnsignedLong(i1) << pickPos(56, 8))
4198               | (toUnsignedLong(i2) << pickPos(56, 16))
4199               | (toUnsignedLong(i3) << pickPos(56, 24))
4200               | (toUnsignedLong(i4) << pickPos(56, 32))
4201               | (toUnsignedLong(i5) << pickPos(56, 40))
4202               | (toUnsignedLong(i6) << pickPos(56, 48))
4203               | (toUnsignedLong(i7) << pickPos(56, 56)));
4204     }
4205     private static long makeLong(short i0, short i1, short i2, short i3) {
4206         return ((toUnsignedLong(i0) << pickPos(48, 0))
4207               | (toUnsignedLong(i1) << pickPos(48, 16))
4208               | (toUnsignedLong(i2) << pickPos(48, 32))
4209               | (toUnsignedLong(i3) << pickPos(48, 48)));
4210     }
4211     private static long makeLong(int i0, int i1) {
4212         return (toUnsignedLong(i0) << pickPos(32, 0))
4213              | (toUnsignedLong(i1) << pickPos(32, 32));
4214     }
4215     private static int makeInt(short i0, short i1) {
4216         return (toUnsignedInt(i0) << pickPos(16, 0))
4217              | (toUnsignedInt(i1) << pickPos(16, 16));
4218     }
4219     private static int makeInt(byte i0, byte i1, byte i2, byte i3) {
4220         return ((toUnsignedInt(i0) << pickPos(24, 0))
4221               | (toUnsignedInt(i1) << pickPos(24, 8))
4222               | (toUnsignedInt(i2) << pickPos(24, 16))
4223               | (toUnsignedInt(i3) << pickPos(24, 24)));
4224     }
4225     private static short makeShort(byte i0, byte i1) {
4226         return (short)((toUnsignedInt(i0) << pickPos(8, 0))
4227                      | (toUnsignedInt(i1) << pickPos(8, 8)));
4228     }
4229 
4230     private static byte  pick(byte  le, byte  be) { return BIG_ENDIAN ? be : le; }
4231     private static short pick(short le, short be) { return BIG_ENDIAN ? be : le; }
4232     private static int   pick(int   le, int   be) { return BIG_ENDIAN ? be : le; }
4233 
4234     // These methods write integers to memory from smaller parts
4235     // provided by their caller.  The ordering in which these parts
4236     // are written is the native endianness of this platform.
4237     private void putLongParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
4238         putByte(o, offset + 0, pick(i0, i7));
4239         putByte(o, offset + 1, pick(i1, i6));
4240         putByte(o, offset + 2, pick(i2, i5));
4241         putByte(o, offset + 3, pick(i3, i4));
4242         putByte(o, offset + 4, pick(i4, i3));
4243         putByte(o, offset + 5, pick(i5, i2));
4244         putByte(o, offset + 6, pick(i6, i1));
4245         putByte(o, offset + 7, pick(i7, i0));
4246     }
4247     private void putLongParts(Object o, long offset, short i0, short i1, short i2, short i3) {
4248         putShort(o, offset + 0, pick(i0, i3));
4249         putShort(o, offset + 2, pick(i1, i2));
4250         putShort(o, offset + 4, pick(i2, i1));
4251         putShort(o, offset + 6, pick(i3, i0));
4252     }
4253     private void putLongParts(Object o, long offset, int i0, int i1) {
4254         putInt(o, offset + 0, pick(i0, i1));
4255         putInt(o, offset + 4, pick(i1, i0));
4256     }
4257     private void putIntParts(Object o, long offset, short i0, short i1) {
4258         putShort(o, offset + 0, pick(i0, i1));
4259         putShort(o, offset + 2, pick(i1, i0));
4260     }
4261     private void putIntParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3) {
4262         putByte(o, offset + 0, pick(i0, i3));
4263         putByte(o, offset + 1, pick(i1, i2));
4264         putByte(o, offset + 2, pick(i2, i1));
4265         putByte(o, offset + 3, pick(i3, i0));
4266     }
4267     private void putShortParts(Object o, long offset, byte i0, byte i1) {
4268         putByte(o, offset + 0, pick(i0, i1));
4269         putByte(o, offset + 1, pick(i1, i0));
4270     }
4271 
4272     // Zero-extend an integer
4273     private static int toUnsignedInt(byte n)    { return n & 0xff; }
4274     private static int toUnsignedInt(short n)   { return n & 0xffff; }
4275     private static long toUnsignedLong(byte n)  { return n & 0xffl; }
4276     private static long toUnsignedLong(short n) { return n & 0xffffl; }
4277     private static long toUnsignedLong(int n)   { return n & 0xffffffffl; }
4278 
4279     // Maybe byte-reverse an integer
4280     private static char convEndian(boolean big, char n)   { return big == BIG_ENDIAN ? n : Character.reverseBytes(n); }
4281     private static short convEndian(boolean big, short n) { return big == BIG_ENDIAN ? n : Short.reverseBytes(n)    ; }
4282     private static int convEndian(boolean big, int n)     { return big == BIG_ENDIAN ? n : Integer.reverseBytes(n)  ; }
4283     private static long convEndian(boolean big, long n)   { return big == BIG_ENDIAN ? n : Long.reverseBytes(n)     ; }
4284 
4285 
4286 
4287     private native long allocateMemory0(long bytes);
4288     private native long reallocateMemory0(long address, long bytes);
4289     private native void freeMemory0(long address);
4290     @IntrinsicCandidate
4291     private native void setMemory0(Object o, long offset, long bytes, byte value);
4292     @IntrinsicCandidate
4293     private native void copyMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
4294     private native void copySwapMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes, long elemSize);
4295     private native long objectFieldOffset0(Field f);
4296     private native long objectFieldOffset1(Class<?> c, String name);
4297     private native long staticFieldOffset0(Field f);
4298     private native Object staticFieldBase0(Field f);
4299     private native boolean shouldBeInitialized0(Class<?> c);
4300     private native void ensureClassInitialized0(Class<?> c);
4301     private native int arrayBaseOffset0(Class<?> arrayClass);
4302     private native int arrayIndexScale0(Class<?> arrayClass);
4303     private native long getObjectSize0(Object o);
4304     private native int getLoadAverage0(double[] loadavg, int nelems);
4305 
4306 
4307     /**
4308      * Invokes the given direct byte buffer's cleaner, if any.
4309      *
4310      * @param directBuffer a direct byte buffer
4311      * @throws NullPointerException     if {@code directBuffer} is null
4312      * @throws IllegalArgumentException if {@code directBuffer} is non-direct,
4313      *                                  or is a {@link java.nio.Buffer#slice slice}, or is a
4314      *                                  {@link java.nio.Buffer#duplicate duplicate}
4315      */
4316     public void invokeCleaner(java.nio.ByteBuffer directBuffer) {
4317         if (!directBuffer.isDirect())
4318             throw new IllegalArgumentException("buffer is non-direct");
4319 
4320         DirectBuffer db = (DirectBuffer) directBuffer;
4321         if (db.attachment() != null)
4322             throw new IllegalArgumentException("duplicate or slice");
4323 
4324         Cleaner cleaner = db.cleaner();
4325         if (cleaner != null) {
4326             cleaner.clean();
4327         }
4328     }
4329 }