1 /*
   2  * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.internal.misc;
  27 
  28 import jdk.internal.ref.Cleaner;
  29 import jdk.internal.vm.annotation.ForceInline;
  30 import jdk.internal.vm.annotation.IntrinsicCandidate;
  31 import sun.nio.ch.DirectBuffer;
  32 
  33 import java.lang.reflect.Field;
  34 import java.security.ProtectionDomain;
  35 
  36 import static jdk.internal.misc.UnsafeConstants.*;
  37 
  38 /**
  39  * A collection of methods for performing low-level, unsafe operations.
  40  * Although the class and all methods are public, use of this class is
  41  * limited because only trusted code can obtain instances of it.
  42  *
  43  * <em>Note:</em> It is the responsibility of the caller to make sure
  44  * arguments are checked before methods of this class are
  45  * called. While some rudimentary checks are performed on the input,
  46  * the checks are best effort and when performance is an overriding
  47  * priority, as when methods of this class are optimized by the
  48  * runtime compiler, some or all checks (if any) may be elided. Hence,
  49  * the caller must not rely on the checks and corresponding
  50  * exceptions!
  51  *
  52  * @author John R. Rose
  53  * @see #getUnsafe
  54  */
  55 
  56 public final class Unsafe {
  57 
  58     private static native void registerNatives();
  59     static {
  60         registerNatives();
  61     }
  62 
  63     private Unsafe() {}
  64 
  65     private static final Unsafe theUnsafe = new Unsafe();
  66 
  67     /**
  68      * Provides the caller with the capability of performing unsafe
  69      * operations.
  70      *
  71      * <p>The returned {@code Unsafe} object should be carefully guarded
  72      * by the caller, since it can be used to read and write data at arbitrary
  73      * memory addresses.  It must never be passed to untrusted code.
  74      *
  75      * <p>Most methods in this class are very low-level, and correspond to a
  76      * small number of hardware instructions (on typical machines).  Compilers
  77      * are encouraged to optimize these methods accordingly.
  78      *
  79      * <p>Here is a suggested idiom for using unsafe operations:
  80      *
  81      * <pre> {@code
  82      * class MyTrustedClass {
  83      *   private static final Unsafe unsafe = Unsafe.getUnsafe();
  84      *   ...
  85      *   private long myCountAddress = ...;
  86      *   public int getCount() { return unsafe.getByte(myCountAddress); }
  87      * }}</pre>
  88      *
  89      * (It may assist compilers to make the local variable {@code final}.)
  90      */
  91     public static Unsafe getUnsafe() {
  92         return theUnsafe;
  93     }
  94 
  95     /// peek and poke operations
  96     /// (compilers should optimize these to memory ops)
  97 
  98     // These work on object fields in the Java heap.
  99     // They will not work on elements of packed arrays.
 100 
 101     /**
 102      * Fetches a value from a given Java variable.
 103      * More specifically, fetches a field or array element within the given
 104      * object {@code o} at the given offset, or (if {@code o} is null)
 105      * from the memory address whose numerical value is the given offset.
 106      * <p>
 107      * The results are undefined unless one of the following cases is true:
 108      * <ul>
 109      * <li>The offset was obtained from {@link #objectFieldOffset} on
 110      * the {@link java.lang.reflect.Field} of some Java field and the object
 111      * referred to by {@code o} is of a class compatible with that
 112      * field's class.
 113      *
 114      * <li>The offset and object reference {@code o} (either null or
 115      * non-null) were both obtained via {@link #staticFieldOffset}
 116      * and {@link #staticFieldBase} (respectively) from the
 117      * reflective {@link Field} representation of some Java field.
 118      *
 119      * <li>The object referred to by {@code o} is an array, and the offset
 120      * is an integer of the form {@code B+N*S}, where {@code N} is
 121      * a valid index into the array, and {@code B} and {@code S} are
 122      * the values obtained by {@link #arrayBaseOffset} and {@link
 123      * #arrayIndexScale} (respectively) from the array's class.  The value
 124      * referred to is the {@code N}<em>th</em> element of the array.
 125      *
 126      * </ul>
 127      * <p>
 128      * If one of the above cases is true, the call references a specific Java
 129      * variable (field or array element).  However, the results are undefined
 130      * if that variable is not in fact of the type returned by this method.
 131      * <p>
 132      * This method refers to a variable by means of two parameters, and so
 133      * it provides (in effect) a <em>double-register</em> addressing mode
 134      * for Java variables.  When the object reference is null, this method
 135      * uses its offset as an absolute address.  This is similar in operation
 136      * to methods such as {@link #getInt(long)}, which provide (in effect) a
 137      * <em>single-register</em> addressing mode for non-Java variables.
 138      * However, because Java variables may have a different layout in memory
 139      * from non-Java variables, programmers should not assume that these
 140      * two addressing modes are ever equivalent.  Also, programmers should
 141      * remember that offsets from the double-register addressing mode cannot
 142      * be portably confused with longs used in the single-register addressing
 143      * mode.
 144      *
 145      * @param o Java heap object in which the variable resides, if any, else
 146      *        null
 147      * @param offset indication of where the variable resides in a Java heap
 148      *        object, if any, else a memory address locating the variable
 149      *        statically
 150      * @return the value fetched from the indicated Java variable
 151      * @throws RuntimeException No defined exceptions are thrown, not even
 152      *         {@link NullPointerException}
 153      */
 154     @IntrinsicCandidate
 155     public native int getInt(Object o, long offset);
 156 
 157     /**
 158      * Stores a value into a given Java variable.
 159      * <p>
 160      * The first two parameters are interpreted exactly as with
 161      * {@link #getInt(Object, long)} to refer to a specific
 162      * Java variable (field or array element).  The given value
 163      * is stored into that variable.
 164      * <p>
 165      * The variable must be of the same type as the method
 166      * parameter {@code x}.
 167      *
 168      * @param o Java heap object in which the variable resides, if any, else
 169      *        null
 170      * @param offset indication of where the variable resides in a Java heap
 171      *        object, if any, else a memory address locating the variable
 172      *        statically
 173      * @param x the value to store into the indicated Java variable
 174      * @throws RuntimeException No defined exceptions are thrown, not even
 175      *         {@link NullPointerException}
 176      */
 177     @IntrinsicCandidate
 178     public native void putInt(Object o, long offset, int x);
 179 
 180 
 181     /**
 182      * Returns true if the given field is flattened.
 183      */
 184     public boolean isFlatField(Field f) {
 185         if (f == null) {
 186             throw new NullPointerException();
 187         }
 188         return isFlatField0(f);
 189     }
 190 
 191     private native boolean isFlatField0(Object o);
 192 
 193     /* Returns true if the given field has a null marker
 194      * <p>
 195      * Nullable flat fields are stored in a flattened representation
 196      * and have an associated null marker to indicate if the the field value is
 197      * null or the one stored with the flat representation
 198      */
 199 
 200      public boolean hasNullMarker(Field f) {
 201         if (f == null) {
 202             throw new NullPointerException();
 203         }
 204         return hasNullMarker0(f);
 205      }
 206 
 207      private native boolean hasNullMarker0(Object o);
 208 
 209      /* Returns the offset of the null marker of the field,
 210       * or -1 if the field doesn't have a null marker
 211       */
 212 
 213      public int nullMarkerOffset(Field f) {
 214         if (f == null) {
 215             throw new NullPointerException();
 216         }
 217         return nullMarkerOffset0(f);
 218      }
 219 
 220      private native int nullMarkerOffset0(Object o);
 221 
 222     /**
 223      * Returns true if the given class is a flattened array.
 224      */
 225     @IntrinsicCandidate
 226     public native boolean isFlatArray(Class<?> arrayClass);
 227 
 228     /**
 229      * Fetches a reference value from a given Java variable.
 230      * This method can return a reference to either an object or value
 231      * or a null reference.
 232      *
 233      * @see #getInt(Object, long)
 234      */
 235     @IntrinsicCandidate
 236     public native Object getReference(Object o, long offset);
 237 
 238     /**
 239      * Stores a reference value into a given Java variable.
 240      * This method can store a reference to either an object or value
 241      * or a null reference.
 242      * <p>
 243      * Unless the reference {@code x} being stored is either null
 244      * or matches the field type, the results are undefined.
 245      * If the reference {@code o} is non-null, card marks or
 246      * other store barriers for that object (if the VM requires them)
 247      * are updated.
 248      * @see #putInt(Object, long, int)
 249      */
 250     @IntrinsicCandidate
 251     public native void putReference(Object o, long offset, Object x);
 252 
 253     /**
 254      * Fetches a value of type {@code <V>} from a given Java variable.
 255      * More specifically, fetches a field or array element within the given
 256      * {@code o} object at the given offset, or (if {@code o} is null)
 257      * from the memory address whose numerical value is the given offset.
 258      *
 259      * @param o Java heap object in which the variable resides, if any, else
 260      *        null
 261      * @param offset indication of where the variable resides in a Java heap
 262      *        object, if any, else a memory address locating the variable
 263      *        statically
 264      * @param valueType value type
 265      * @param <V> the type of a value
 266      * @return the value fetched from the indicated Java variable
 267      * @throws RuntimeException No defined exceptions are thrown, not even
 268      *         {@link NullPointerException}
 269      */
 270     @IntrinsicCandidate
 271     public native <V> V getValue(Object o, long offset, Class<?> valueType);
 272 
 273     /**
 274      * Stores the given value into a given Java variable.
 275      *
 276      * Unless the reference {@code o} being stored is either null
 277      * or matches the field type, the results are undefined.
 278      *
 279      * @param o Java heap object in which the variable resides, if any, else
 280      *        null
 281      * @param offset indication of where the variable resides in a Java heap
 282      *        object, if any, else a memory address locating the variable
 283      *        statically
 284      * @param valueType value type
 285      * @param v the value to store into the indicated Java variable
 286      * @param <V> the type of a value
 287      * @throws RuntimeException No defined exceptions are thrown, not even
 288      *         {@link NullPointerException}
 289      */
 290     @IntrinsicCandidate
 291     public native <V> void putValue(Object o, long offset, Class<?> valueType, V v);
 292 
 293     /**
 294      * Returns an uninitialized default instance of the given value class.
 295      */
 296     public native <V> V uninitializedDefaultValue(Class<?> type);
 297 
 298     /**
 299      * Returns an object instance with a private buffered value whose layout
 300      * and contents is exactly the given value instance.  The return object
 301      * is in the larval state that can be updated using the unsafe put operation.
 302      *
 303      * @param value a value instance
 304      * @param <V> the type of the given value instance
 305      */
 306     @IntrinsicCandidate
 307     public native <V> V makePrivateBuffer(V value);
 308 
 309     /**
 310      * Exits the larval state and returns a value instance.
 311      *
 312      * @param value a value instance
 313      * @param <V> the type of the given value instance
 314      */
 315     @IntrinsicCandidate
 316     public native <V> V finishPrivateBuffer(V value);
 317 
 318     /**
 319      * Returns the header size of the given value type.
 320      *
 321      * @param valueType value type
 322      * @return the header size of the value type
 323      */
 324     public native <V> long valueHeaderSize(Class<V> valueType);
 325 
 326     /** @see #getInt(Object, long) */
 327     @IntrinsicCandidate
 328     public native boolean getBoolean(Object o, long offset);
 329 
 330     /** @see #putInt(Object, long, int) */
 331     @IntrinsicCandidate
 332     public native void    putBoolean(Object o, long offset, boolean x);
 333 
 334     /** @see #getInt(Object, long) */
 335     @IntrinsicCandidate
 336     public native byte    getByte(Object o, long offset);
 337 
 338     /** @see #putInt(Object, long, int) */
 339     @IntrinsicCandidate
 340     public native void    putByte(Object o, long offset, byte x);
 341 
 342     /** @see #getInt(Object, long) */
 343     @IntrinsicCandidate
 344     public native short   getShort(Object o, long offset);
 345 
 346     /** @see #putInt(Object, long, int) */
 347     @IntrinsicCandidate
 348     public native void    putShort(Object o, long offset, short x);
 349 
 350     /** @see #getInt(Object, long) */
 351     @IntrinsicCandidate
 352     public native char    getChar(Object o, long offset);
 353 
 354     /** @see #putInt(Object, long, int) */
 355     @IntrinsicCandidate
 356     public native void    putChar(Object o, long offset, char x);
 357 
 358     /** @see #getInt(Object, long) */
 359     @IntrinsicCandidate
 360     public native long    getLong(Object o, long offset);
 361 
 362     /** @see #putInt(Object, long, int) */
 363     @IntrinsicCandidate
 364     public native void    putLong(Object o, long offset, long x);
 365 
 366     /** @see #getInt(Object, long) */
 367     @IntrinsicCandidate
 368     public native float   getFloat(Object o, long offset);
 369 
 370     /** @see #putInt(Object, long, int) */
 371     @IntrinsicCandidate
 372     public native void    putFloat(Object o, long offset, float x);
 373 
 374     /** @see #getInt(Object, long) */
 375     @IntrinsicCandidate
 376     public native double  getDouble(Object o, long offset);
 377 
 378     /** @see #putInt(Object, long, int) */
 379     @IntrinsicCandidate
 380     public native void    putDouble(Object o, long offset, double x);
 381 
 382     /**
 383      * Fetches a native pointer from a given memory address.  If the address is
 384      * zero, or does not point into a block obtained from {@link
 385      * #allocateMemory}, the results are undefined.
 386      *
 387      * <p>If the native pointer is less than 64 bits wide, it is extended as
 388      * an unsigned number to a Java long.  The pointer may be indexed by any
 389      * given byte offset, simply by adding that offset (as a simple integer) to
 390      * the long representing the pointer.  The number of bytes actually read
 391      * from the target address may be determined by consulting {@link
 392      * #addressSize}.
 393      *
 394      * @see #allocateMemory
 395      * @see #getInt(Object, long)
 396      */
 397     @ForceInline
 398     public long getAddress(Object o, long offset) {
 399         if (ADDRESS_SIZE == 4) {
 400             return Integer.toUnsignedLong(getInt(o, offset));
 401         } else {
 402             return getLong(o, offset);
 403         }
 404     }
 405 
 406     /**
 407      * Stores a native pointer into a given memory address.  If the address is
 408      * zero, or does not point into a block obtained from {@link
 409      * #allocateMemory}, the results are undefined.
 410      *
 411      * <p>The number of bytes actually written at the target address may be
 412      * determined by consulting {@link #addressSize}.
 413      *
 414      * @see #allocateMemory
 415      * @see #putInt(Object, long, int)
 416      */
 417     @ForceInline
 418     public void putAddress(Object o, long offset, long x) {
 419         if (ADDRESS_SIZE == 4) {
 420             putInt(o, offset, (int)x);
 421         } else {
 422             putLong(o, offset, x);
 423         }
 424     }
 425 
 426     // These read VM internal data.
 427 
 428     /**
 429      * Fetches an uncompressed reference value from a given native variable
 430      * ignoring the VM's compressed references mode.
 431      *
 432      * @param address a memory address locating the variable
 433      * @return the value fetched from the indicated native variable
 434      */
 435     public native Object getUncompressedObject(long address);
 436 
 437     // These work on values in the C heap.
 438 
 439     /**
 440      * Fetches a value from a given memory address.  If the address is zero, or
 441      * does not point into a block obtained from {@link #allocateMemory}, the
 442      * results are undefined.
 443      *
 444      * @see #allocateMemory
 445      */
 446     @ForceInline
 447     public byte getByte(long address) {
 448         return getByte(null, address);
 449     }
 450 
 451     /**
 452      * Stores a value into a given memory address.  If the address is zero, or
 453      * does not point into a block obtained from {@link #allocateMemory}, the
 454      * results are undefined.
 455      *
 456      * @see #getByte(long)
 457      */
 458     @ForceInline
 459     public void putByte(long address, byte x) {
 460         putByte(null, address, x);
 461     }
 462 
 463     /** @see #getByte(long) */
 464     @ForceInline
 465     public short getShort(long address) {
 466         return getShort(null, address);
 467     }
 468 
 469     /** @see #putByte(long, byte) */
 470     @ForceInline
 471     public void putShort(long address, short x) {
 472         putShort(null, address, x);
 473     }
 474 
 475     /** @see #getByte(long) */
 476     @ForceInline
 477     public char getChar(long address) {
 478         return getChar(null, address);
 479     }
 480 
 481     /** @see #putByte(long, byte) */
 482     @ForceInline
 483     public void putChar(long address, char x) {
 484         putChar(null, address, x);
 485     }
 486 
 487     /** @see #getByte(long) */
 488     @ForceInline
 489     public int getInt(long address) {
 490         return getInt(null, address);
 491     }
 492 
 493     /** @see #putByte(long, byte) */
 494     @ForceInline
 495     public void putInt(long address, int x) {
 496         putInt(null, address, x);
 497     }
 498 
 499     /** @see #getByte(long) */
 500     @ForceInline
 501     public long getLong(long address) {
 502         return getLong(null, address);
 503     }
 504 
 505     /** @see #putByte(long, byte) */
 506     @ForceInline
 507     public void putLong(long address, long x) {
 508         putLong(null, address, x);
 509     }
 510 
 511     /** @see #getByte(long) */
 512     @ForceInline
 513     public float getFloat(long address) {
 514         return getFloat(null, address);
 515     }
 516 
 517     /** @see #putByte(long, byte) */
 518     @ForceInline
 519     public void putFloat(long address, float x) {
 520         putFloat(null, address, x);
 521     }
 522 
 523     /** @see #getByte(long) */
 524     @ForceInline
 525     public double getDouble(long address) {
 526         return getDouble(null, address);
 527     }
 528 
 529     /** @see #putByte(long, byte) */
 530     @ForceInline
 531     public void putDouble(long address, double x) {
 532         putDouble(null, address, x);
 533     }
 534 
 535     /** @see #getAddress(Object, long) */
 536     @ForceInline
 537     public long getAddress(long address) {
 538         return getAddress(null, address);
 539     }
 540 
 541     /** @see #putAddress(Object, long, long) */
 542     @ForceInline
 543     public void putAddress(long address, long x) {
 544         putAddress(null, address, x);
 545     }
 546 
 547 
 548 
 549     /// helper methods for validating various types of objects/values
 550 
 551     /**
 552      * Create an exception reflecting that some of the input was invalid
 553      *
 554      * <em>Note:</em> It is the responsibility of the caller to make
 555      * sure arguments are checked before the methods are called. While
 556      * some rudimentary checks are performed on the input, the checks
 557      * are best effort and when performance is an overriding priority,
 558      * as when methods of this class are optimized by the runtime
 559      * compiler, some or all checks (if any) may be elided. Hence, the
 560      * caller must not rely on the checks and corresponding
 561      * exceptions!
 562      *
 563      * @return an exception object
 564      */
 565     private RuntimeException invalidInput() {
 566         return new IllegalArgumentException();
 567     }
 568 
 569     /**
 570      * Check if a value is 32-bit clean (32 MSB are all zero)
 571      *
 572      * @param value the 64-bit value to check
 573      *
 574      * @return true if the value is 32-bit clean
 575      */
 576     private boolean is32BitClean(long value) {
 577         return value >>> 32 == 0;
 578     }
 579 
 580     /**
 581      * Check the validity of a size (the equivalent of a size_t)
 582      *
 583      * @throws RuntimeException if the size is invalid
 584      *         (<em>Note:</em> after optimization, invalid inputs may
 585      *         go undetected, which will lead to unpredictable
 586      *         behavior)
 587      */
 588     private void checkSize(long size) {
 589         if (ADDRESS_SIZE == 4) {
 590             // Note: this will also check for negative sizes
 591             if (!is32BitClean(size)) {
 592                 throw invalidInput();
 593             }
 594         } else if (size < 0) {
 595             throw invalidInput();
 596         }
 597     }
 598 
 599     /**
 600      * Check the validity of a native address (the equivalent of void*)
 601      *
 602      * @throws RuntimeException if the address is invalid
 603      *         (<em>Note:</em> after optimization, invalid inputs may
 604      *         go undetected, which will lead to unpredictable
 605      *         behavior)
 606      */
 607     private void checkNativeAddress(long address) {
 608         if (ADDRESS_SIZE == 4) {
 609             // Accept both zero and sign extended pointers. A valid
 610             // pointer will, after the +1 below, either have produced
 611             // the value 0x0 or 0x1. Masking off the low bit allows
 612             // for testing against 0.
 613             if ((((address >> 32) + 1) & ~1) != 0) {
 614                 throw invalidInput();
 615             }
 616         }
 617     }
 618 
 619     /**
 620      * Check the validity of an offset, relative to a base object
 621      *
 622      * @param o the base object
 623      * @param offset the offset to check
 624      *
 625      * @throws RuntimeException if the size is invalid
 626      *         (<em>Note:</em> after optimization, invalid inputs may
 627      *         go undetected, which will lead to unpredictable
 628      *         behavior)
 629      */
 630     private void checkOffset(Object o, long offset) {
 631         if (ADDRESS_SIZE == 4) {
 632             // Note: this will also check for negative offsets
 633             if (!is32BitClean(offset)) {
 634                 throw invalidInput();
 635             }
 636         } else if (offset < 0) {
 637             throw invalidInput();
 638         }
 639     }
 640 
 641     /**
 642      * Check the validity of a double-register pointer
 643      *
 644      * Note: This code deliberately does *not* check for NPE for (at
 645      * least) three reasons:
 646      *
 647      * 1) NPE is not just NULL/0 - there is a range of values all
 648      * resulting in an NPE, which is not trivial to check for
 649      *
 650      * 2) It is the responsibility of the callers of Unsafe methods
 651      * to verify the input, so throwing an exception here is not really
 652      * useful - passing in a NULL pointer is a critical error and the
 653      * must not expect an exception to be thrown anyway.
 654      *
 655      * 3) the actual operations will detect NULL pointers anyway by
 656      * means of traps and signals (like SIGSEGV).
 657      *
 658      * @param o Java heap object, or null
 659      * @param offset indication of where the variable resides in a Java heap
 660      *        object, if any, else a memory address locating the variable
 661      *        statically
 662      *
 663      * @throws RuntimeException if the pointer is invalid
 664      *         (<em>Note:</em> after optimization, invalid inputs may
 665      *         go undetected, which will lead to unpredictable
 666      *         behavior)
 667      */
 668     private void checkPointer(Object o, long offset) {
 669         if (o == null) {
 670             checkNativeAddress(offset);
 671         } else {
 672             checkOffset(o, offset);
 673         }
 674     }
 675 
 676     /**
 677      * Check if a type is a primitive array type
 678      *
 679      * @param c the type to check
 680      *
 681      * @return true if the type is a primitive array type
 682      */
 683     private void checkPrimitiveArray(Class<?> c) {
 684         Class<?> componentType = c.getComponentType();
 685         if (componentType == null || !componentType.isPrimitive()) {
 686             throw invalidInput();
 687         }
 688     }
 689 
 690     /**
 691      * Check that a pointer is a valid primitive array type pointer
 692      *
 693      * Note: pointers off-heap are considered to be primitive arrays
 694      *
 695      * @throws RuntimeException if the pointer is invalid
 696      *         (<em>Note:</em> after optimization, invalid inputs may
 697      *         go undetected, which will lead to unpredictable
 698      *         behavior)
 699      */
 700     private void checkPrimitivePointer(Object o, long offset) {
 701         checkPointer(o, offset);
 702 
 703         if (o != null) {
 704             // If on heap, it must be a primitive array
 705             checkPrimitiveArray(o.getClass());
 706         }
 707     }
 708 
 709 
 710     /// wrappers for malloc, realloc, free:
 711 
 712     /**
 713      * Round up allocation size to a multiple of HeapWordSize.
 714      */
 715     private long alignToHeapWordSize(long bytes) {
 716         if (bytes >= 0) {
 717             return (bytes + ADDRESS_SIZE - 1) & ~(ADDRESS_SIZE - 1);
 718         } else {
 719             throw invalidInput();
 720         }
 721     }
 722 
 723     /**
 724      * Allocates a new block of native memory, of the given size in bytes.  The
 725      * contents of the memory are uninitialized; they will generally be
 726      * garbage.  The resulting native pointer will be zero if and only if the
 727      * requested size is zero.  The resulting native pointer will be aligned for
 728      * all value types.   Dispose of this memory by calling {@link #freeMemory}
 729      * or resize it with {@link #reallocateMemory}.
 730      *
 731      * <em>Note:</em> It is the responsibility of the caller to make
 732      * sure arguments are checked before the methods are called. While
 733      * some rudimentary checks are performed on the input, the checks
 734      * are best effort and when performance is an overriding priority,
 735      * as when methods of this class are optimized by the runtime
 736      * compiler, some or all checks (if any) may be elided. Hence, the
 737      * caller must not rely on the checks and corresponding
 738      * exceptions!
 739      *
 740      * @throws RuntimeException if the size is negative or too large
 741      *         for the native size_t type
 742      *
 743      * @throws OutOfMemoryError if the allocation is refused by the system
 744      *
 745      * @see #getByte(long)
 746      * @see #putByte(long, byte)
 747      */
 748     public long allocateMemory(long bytes) {
 749         bytes = alignToHeapWordSize(bytes);
 750 
 751         allocateMemoryChecks(bytes);
 752 
 753         if (bytes == 0) {
 754             return 0;
 755         }
 756 
 757         long p = allocateMemory0(bytes);
 758         if (p == 0) {
 759             throw new OutOfMemoryError("Unable to allocate " + bytes + " bytes");
 760         }
 761 
 762         return p;
 763     }
 764 
 765     /**
 766      * Validate the arguments to allocateMemory
 767      *
 768      * @throws RuntimeException if the arguments are invalid
 769      *         (<em>Note:</em> after optimization, invalid inputs may
 770      *         go undetected, which will lead to unpredictable
 771      *         behavior)
 772      */
 773     private void allocateMemoryChecks(long bytes) {
 774         checkSize(bytes);
 775     }
 776 
 777     /**
 778      * Resizes a new block of native memory, to the given size in bytes.  The
 779      * contents of the new block past the size of the old block are
 780      * uninitialized; they will generally be garbage.  The resulting native
 781      * pointer will be zero if and only if the requested size is zero.  The
 782      * resulting native pointer will be aligned for all value types.  Dispose
 783      * of this memory by calling {@link #freeMemory}, or resize it with {@link
 784      * #reallocateMemory}.  The address passed to this method may be null, in
 785      * which case an allocation will be performed.
 786      *
 787      * <em>Note:</em> It is the responsibility of the caller to make
 788      * sure arguments are checked before the methods are called. While
 789      * some rudimentary checks are performed on the input, the checks
 790      * are best effort and when performance is an overriding priority,
 791      * as when methods of this class are optimized by the runtime
 792      * compiler, some or all checks (if any) may be elided. Hence, the
 793      * caller must not rely on the checks and corresponding
 794      * exceptions!
 795      *
 796      * @throws RuntimeException if the size is negative or too large
 797      *         for the native size_t type
 798      *
 799      * @throws OutOfMemoryError if the allocation is refused by the system
 800      *
 801      * @see #allocateMemory
 802      */
 803     public long reallocateMemory(long address, long bytes) {
 804         bytes = alignToHeapWordSize(bytes);
 805 
 806         reallocateMemoryChecks(address, bytes);
 807 
 808         if (bytes == 0) {
 809             freeMemory(address);
 810             return 0;
 811         }
 812 
 813         long p = (address == 0) ? allocateMemory0(bytes) : reallocateMemory0(address, bytes);
 814         if (p == 0) {
 815             throw new OutOfMemoryError("Unable to allocate " + bytes + " bytes");
 816         }
 817 
 818         return p;
 819     }
 820 
 821     /**
 822      * Validate the arguments to reallocateMemory
 823      *
 824      * @throws RuntimeException if the arguments are invalid
 825      *         (<em>Note:</em> after optimization, invalid inputs may
 826      *         go undetected, which will lead to unpredictable
 827      *         behavior)
 828      */
 829     private void reallocateMemoryChecks(long address, long bytes) {
 830         checkPointer(null, address);
 831         checkSize(bytes);
 832     }
 833 
 834     /**
 835      * Sets all bytes in a given block of memory to a fixed value
 836      * (usually zero).
 837      *
 838      * <p>This method determines a block's base address by means of two parameters,
 839      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 840      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 841      * the offset supplies an absolute base address.
 842      *
 843      * <p>The stores are in coherent (atomic) units of a size determined
 844      * by the address and length parameters.  If the effective address and
 845      * length are all even modulo 8, the stores take place in 'long' units.
 846      * If the effective address and length are (resp.) even modulo 4 or 2,
 847      * the stores take place in units of 'int' or 'short'.
 848      *
 849      * <em>Note:</em> It is the responsibility of the caller to make
 850      * sure arguments are checked before the methods are called. While
 851      * some rudimentary checks are performed on the input, the checks
 852      * are best effort and when performance is an overriding priority,
 853      * as when methods of this class are optimized by the runtime
 854      * compiler, some or all checks (if any) may be elided. Hence, the
 855      * caller must not rely on the checks and corresponding
 856      * exceptions!
 857      *
 858      * @throws RuntimeException if any of the arguments is invalid
 859      *
 860      * @since 1.7
 861      */
 862     public void setMemory(Object o, long offset, long bytes, byte value) {
 863         setMemoryChecks(o, offset, bytes, value);
 864 
 865         if (bytes == 0) {
 866             return;
 867         }
 868 
 869         setMemory0(o, offset, bytes, value);
 870     }
 871 
 872     /**
 873      * Sets all bytes in a given block of memory to a fixed value
 874      * (usually zero).  This provides a <em>single-register</em> addressing mode,
 875      * as discussed in {@link #getInt(Object,long)}.
 876      *
 877      * <p>Equivalent to {@code setMemory(null, address, bytes, value)}.
 878      */
 879     public void setMemory(long address, long bytes, byte value) {
 880         setMemory(null, address, bytes, value);
 881     }
 882 
 883     /**
 884      * Validate the arguments to setMemory
 885      *
 886      * @throws RuntimeException if the arguments are invalid
 887      *         (<em>Note:</em> after optimization, invalid inputs may
 888      *         go undetected, which will lead to unpredictable
 889      *         behavior)
 890      */
 891     private void setMemoryChecks(Object o, long offset, long bytes, byte value) {
 892         checkPrimitivePointer(o, offset);
 893         checkSize(bytes);
 894     }
 895 
 896     /**
 897      * Sets all bytes in a given block of memory to a copy of another
 898      * block.
 899      *
 900      * <p>This method determines each block's base address by means of two parameters,
 901      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 902      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 903      * the offset supplies an absolute base address.
 904      *
 905      * <p>The transfers are in coherent (atomic) units of a size determined
 906      * by the address and length parameters.  If the effective addresses and
 907      * length are all even modulo 8, the transfer takes place in 'long' units.
 908      * If the effective addresses and length are (resp.) even modulo 4 or 2,
 909      * the transfer takes place in units of 'int' or 'short'.
 910      *
 911      * <em>Note:</em> It is the responsibility of the caller to make
 912      * sure arguments are checked before the methods are called. While
 913      * some rudimentary checks are performed on the input, the checks
 914      * are best effort and when performance is an overriding priority,
 915      * as when methods of this class are optimized by the runtime
 916      * compiler, some or all checks (if any) may be elided. Hence, the
 917      * caller must not rely on the checks and corresponding
 918      * exceptions!
 919      *
 920      * @throws RuntimeException if any of the arguments is invalid
 921      *
 922      * @since 1.7
 923      */
 924     public void copyMemory(Object srcBase, long srcOffset,
 925                            Object destBase, long destOffset,
 926                            long bytes) {
 927         copyMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes);
 928 
 929         if (bytes == 0) {
 930             return;
 931         }
 932 
 933         copyMemory0(srcBase, srcOffset, destBase, destOffset, bytes);
 934     }
 935 
 936     /**
 937      * Sets all bytes in a given block of memory to a copy of another
 938      * block.  This provides a <em>single-register</em> addressing mode,
 939      * as discussed in {@link #getInt(Object,long)}.
 940      *
 941      * Equivalent to {@code copyMemory(null, srcAddress, null, destAddress, bytes)}.
 942      */
 943     public void copyMemory(long srcAddress, long destAddress, long bytes) {
 944         copyMemory(null, srcAddress, null, destAddress, bytes);
 945     }
 946 
 947     /**
 948      * Validate the arguments to copyMemory
 949      *
 950      * @throws RuntimeException if any of the arguments is invalid
 951      *         (<em>Note:</em> after optimization, invalid inputs may
 952      *         go undetected, which will lead to unpredictable
 953      *         behavior)
 954      */
 955     private void copyMemoryChecks(Object srcBase, long srcOffset,
 956                                   Object destBase, long destOffset,
 957                                   long bytes) {
 958         checkSize(bytes);
 959         checkPrimitivePointer(srcBase, srcOffset);
 960         checkPrimitivePointer(destBase, destOffset);
 961     }
 962 
 963     /**
 964      * Copies all elements from one block of memory to another block,
 965      * *unconditionally* byte swapping the elements on the fly.
 966      *
 967      * <p>This method determines each block's base address by means of two parameters,
 968      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 969      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 970      * the offset supplies an absolute base address.
 971      *
 972      * <em>Note:</em> It is the responsibility of the caller to make
 973      * sure arguments are checked before the methods are called. While
 974      * some rudimentary checks are performed on the input, the checks
 975      * are best effort and when performance is an overriding priority,
 976      * as when methods of this class are optimized by the runtime
 977      * compiler, some or all checks (if any) may be elided. Hence, the
 978      * caller must not rely on the checks and corresponding
 979      * exceptions!
 980      *
 981      * @throws RuntimeException if any of the arguments is invalid
 982      *
 983      * @since 9
 984      */
 985     public void copySwapMemory(Object srcBase, long srcOffset,
 986                                Object destBase, long destOffset,
 987                                long bytes, long elemSize) {
 988         copySwapMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
 989 
 990         if (bytes == 0) {
 991             return;
 992         }
 993 
 994         copySwapMemory0(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
 995     }
 996 
 997     private void copySwapMemoryChecks(Object srcBase, long srcOffset,
 998                                       Object destBase, long destOffset,
 999                                       long bytes, long elemSize) {
1000         checkSize(bytes);
1001 
1002         if (elemSize != 2 && elemSize != 4 && elemSize != 8) {
1003             throw invalidInput();
1004         }
1005         if (bytes % elemSize != 0) {
1006             throw invalidInput();
1007         }
1008 
1009         checkPrimitivePointer(srcBase, srcOffset);
1010         checkPrimitivePointer(destBase, destOffset);
1011     }
1012 
1013     /**
1014      * Copies all elements from one block of memory to another block, byte swapping the
1015      * elements on the fly.
1016      *
1017      * This provides a <em>single-register</em> addressing mode, as
1018      * discussed in {@link #getInt(Object,long)}.
1019      *
1020      * Equivalent to {@code copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize)}.
1021      */
1022     public void copySwapMemory(long srcAddress, long destAddress, long bytes, long elemSize) {
1023         copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize);
1024     }
1025 
1026     /**
1027      * Disposes of a block of native memory, as obtained from {@link
1028      * #allocateMemory} or {@link #reallocateMemory}.  The address passed to
1029      * this method may be null, in which case no action is taken.
1030      *
1031      * <em>Note:</em> It is the responsibility of the caller to make
1032      * sure arguments are checked before the methods are called. While
1033      * some rudimentary checks are performed on the input, the checks
1034      * are best effort and when performance is an overriding priority,
1035      * as when methods of this class are optimized by the runtime
1036      * compiler, some or all checks (if any) may be elided. Hence, the
1037      * caller must not rely on the checks and corresponding
1038      * exceptions!
1039      *
1040      * @throws RuntimeException if any of the arguments is invalid
1041      *
1042      * @see #allocateMemory
1043      */
1044     public void freeMemory(long address) {
1045         freeMemoryChecks(address);
1046 
1047         if (address == 0) {
1048             return;
1049         }
1050 
1051         freeMemory0(address);
1052     }
1053 
1054     /**
1055      * Validate the arguments to freeMemory
1056      *
1057      * @throws RuntimeException if the arguments are invalid
1058      *         (<em>Note:</em> after optimization, invalid inputs may
1059      *         go undetected, which will lead to unpredictable
1060      *         behavior)
1061      */
1062     private void freeMemoryChecks(long address) {
1063         checkPointer(null, address);
1064     }
1065 
1066     /**
1067      * Ensure writeback of a specified virtual memory address range
1068      * from cache to physical memory. All bytes in the address range
1069      * are guaranteed to have been written back to physical memory on
1070      * return from this call i.e. subsequently executed store
1071      * instructions are guaranteed not to be visible before the
1072      * writeback is completed.
1073      *
1074      * @param address
1075      *        the lowest byte address that must be guaranteed written
1076      *        back to memory. bytes at lower addresses may also be
1077      *        written back.
1078      *
1079      * @param length
1080      *        the length in bytes of the region starting at address
1081      *        that must be guaranteed written back to memory.
1082      *
1083      * @throws RuntimeException if memory writeback is not supported
1084      *         on the current hardware of if the arguments are invalid.
1085      *         (<em>Note:</em> after optimization, invalid inputs may
1086      *         go undetected, which will lead to unpredictable
1087      *         behavior)
1088      *
1089      * @since 14
1090      */
1091 
1092     public void writebackMemory(long address, long length) {
1093         checkWritebackEnabled();
1094         checkWritebackMemory(address, length);
1095 
1096         // perform any required pre-writeback barrier
1097         writebackPreSync0();
1098 
1099         // write back one cache line at a time
1100         long line = dataCacheLineAlignDown(address);
1101         long end = address + length;
1102         while (line < end) {
1103             writeback0(line);
1104             line += dataCacheLineFlushSize();
1105         }
1106 
1107         // perform any required post-writeback barrier
1108         writebackPostSync0();
1109     }
1110 
1111     /**
1112      * Validate the arguments to writebackMemory
1113      *
1114      * @throws RuntimeException if the arguments are invalid
1115      *         (<em>Note:</em> after optimization, invalid inputs may
1116      *         go undetected, which will lead to unpredictable
1117      *         behavior)
1118      */
1119     private void checkWritebackMemory(long address, long length) {
1120         checkNativeAddress(address);
1121         checkSize(length);
1122     }
1123 
1124     /**
1125      * Validate that the current hardware supports memory writeback.
1126      * (<em>Note:</em> this is a belt and braces check.  Clients are
1127      * expected to test whether writeback is enabled by calling
1128      * ({@link isWritebackEnabled #isWritebackEnabled} and avoid
1129      * calling method {@link writeback #writeback} if it is disabled).
1130      *
1131      *
1132      * @throws RuntimeException if memory writeback is not supported
1133      */
1134     private void checkWritebackEnabled() {
1135         if (!isWritebackEnabled()) {
1136             throw new RuntimeException("writebackMemory not enabled!");
1137         }
1138     }
1139 
1140     /**
1141      * force writeback of an individual cache line.
1142      *
1143      * @param address
1144      *        the start address of the cache line to be written back
1145      */
1146     @IntrinsicCandidate
1147     private native void writeback0(long address);
1148 
1149      /**
1150       * Serialize writeback operations relative to preceding memory writes.
1151       */
1152     @IntrinsicCandidate
1153     private native void writebackPreSync0();
1154 
1155      /**
1156       * Serialize writeback operations relative to following memory writes.
1157       */
1158     @IntrinsicCandidate
1159     private native void writebackPostSync0();
1160 
1161     /// random queries
1162 
1163     /**
1164      * This constant differs from all results that will ever be returned from
1165      * {@link #staticFieldOffset}, {@link #objectFieldOffset},
1166      * or {@link #arrayBaseOffset}.
1167      */
1168     public static final int INVALID_FIELD_OFFSET = -1;
1169 
1170     /**
1171      * Reports the location of a given field in the storage allocation of its
1172      * class.  Do not expect to perform any sort of arithmetic on this offset;
1173      * it is just a cookie which is passed to the unsafe heap memory accessors.
1174      *
1175      * <p>Any given field will always have the same offset and base, and no
1176      * two distinct fields of the same class will ever have the same offset
1177      * and base.
1178      *
1179      * <p>As of 1.4.1, offsets for fields are represented as long values,
1180      * although the Sun JVM does not use the most significant 32 bits.
1181      * However, JVM implementations which store static fields at absolute
1182      * addresses can use long offsets and null base pointers to express
1183      * the field locations in a form usable by {@link #getInt(Object,long)}.
1184      * Therefore, code which will be ported to such JVMs on 64-bit platforms
1185      * must preserve all bits of static field offsets.
1186      * @see #getInt(Object, long)
1187      */
1188     public long objectFieldOffset(Field f) {
1189         if (f == null) {
1190             throw new NullPointerException();
1191         }
1192 
1193         return objectFieldOffset0(f);
1194     }
1195 
1196     /**
1197      * Reports the location of the field with a given name in the storage
1198      * allocation of its class.
1199      *
1200      * @throws NullPointerException if any parameter is {@code null}.
1201      * @throws InternalError if there is no field named {@code name} declared
1202      *         in class {@code c}, i.e., if {@code c.getDeclaredField(name)}
1203      *         would throw {@code java.lang.NoSuchFieldException}.
1204      *
1205      * @see #objectFieldOffset(Field)
1206      */
1207     public long objectFieldOffset(Class<?> c, String name) {
1208         if (c == null || name == null) {
1209             throw new NullPointerException();
1210         }
1211 
1212         return objectFieldOffset1(c, name);
1213     }
1214 
1215     /**
1216      * Reports the location of a given static field, in conjunction with {@link
1217      * #staticFieldBase}.
1218      * <p>Do not expect to perform any sort of arithmetic on this offset;
1219      * it is just a cookie which is passed to the unsafe heap memory accessors.
1220      *
1221      * <p>Any given field will always have the same offset, and no two distinct
1222      * fields of the same class will ever have the same offset.
1223      *
1224      * <p>As of 1.4.1, offsets for fields are represented as long values,
1225      * although the Sun JVM does not use the most significant 32 bits.
1226      * It is hard to imagine a JVM technology which needs more than
1227      * a few bits to encode an offset within a non-array object,
1228      * However, for consistency with other methods in this class,
1229      * this method reports its result as a long value.
1230      * @see #getInt(Object, long)
1231      */
1232     public long staticFieldOffset(Field f) {
1233         if (f == null) {
1234             throw new NullPointerException();
1235         }
1236 
1237         return staticFieldOffset0(f);
1238     }
1239 
1240     /**
1241      * Reports the location of a given static field, in conjunction with {@link
1242      * #staticFieldOffset}.
1243      * <p>Fetch the base "Object", if any, with which static fields of the
1244      * given class can be accessed via methods like {@link #getInt(Object,
1245      * long)}.  This value may be null.  This value may refer to an object
1246      * which is a "cookie", not guaranteed to be a real Object, and it should
1247      * not be used in any way except as argument to the get and put routines in
1248      * this class.
1249      */
1250     public Object staticFieldBase(Field f) {
1251         if (f == null) {
1252             throw new NullPointerException();
1253         }
1254 
1255         return staticFieldBase0(f);
1256     }
1257 
1258     /**
1259      * Detects if the given class may need to be initialized. This is often
1260      * needed in conjunction with obtaining the static field base of a
1261      * class.
1262      * @return false only if a call to {@code ensureClassInitialized} would have no effect
1263      */
1264     public boolean shouldBeInitialized(Class<?> c) {
1265         if (c == null) {
1266             throw new NullPointerException();
1267         }
1268 
1269         return shouldBeInitialized0(c);
1270     }
1271 
1272     /**
1273      * Ensures the given class has been initialized (see JVMS-5.5 for details).
1274      * This is often needed in conjunction with obtaining the static field base
1275      * of a class.
1276      *
1277      * The call returns when either class {@code c} is fully initialized or
1278      * class {@code c} is being initialized and the call is performed from
1279      * the initializing thread. In the latter case a subsequent call to
1280      * {@link #shouldBeInitialized} will return {@code true}.
1281      */
1282     public void ensureClassInitialized(Class<?> c) {
1283         if (c == null) {
1284             throw new NullPointerException();
1285         }
1286 
1287         ensureClassInitialized0(c);
1288     }
1289 
1290     /**
1291      * Reports the offset of the first element in the storage allocation of a
1292      * given array class.  If {@link #arrayIndexScale} returns a non-zero value
1293      * for the same class, you may use that scale factor, together with this
1294      * base offset, to form new offsets to access elements of arrays of the
1295      * given class.
1296      *
1297      * @see #getInt(Object, long)
1298      * @see #putInt(Object, long, int)
1299      */
1300     public int arrayBaseOffset(Class<?> arrayClass) {
1301         if (arrayClass == null) {
1302             throw new NullPointerException();
1303         }
1304 
1305         return arrayBaseOffset0(arrayClass);
1306     }
1307 
1308 
1309     /** The value of {@code arrayBaseOffset(boolean[].class)} */
1310     public static final int ARRAY_BOOLEAN_BASE_OFFSET
1311             = theUnsafe.arrayBaseOffset(boolean[].class);
1312 
1313     /** The value of {@code arrayBaseOffset(byte[].class)} */
1314     public static final int ARRAY_BYTE_BASE_OFFSET
1315             = theUnsafe.arrayBaseOffset(byte[].class);
1316 
1317     /** The value of {@code arrayBaseOffset(short[].class)} */
1318     public static final int ARRAY_SHORT_BASE_OFFSET
1319             = theUnsafe.arrayBaseOffset(short[].class);
1320 
1321     /** The value of {@code arrayBaseOffset(char[].class)} */
1322     public static final int ARRAY_CHAR_BASE_OFFSET
1323             = theUnsafe.arrayBaseOffset(char[].class);
1324 
1325     /** The value of {@code arrayBaseOffset(int[].class)} */
1326     public static final int ARRAY_INT_BASE_OFFSET
1327             = theUnsafe.arrayBaseOffset(int[].class);
1328 
1329     /** The value of {@code arrayBaseOffset(long[].class)} */
1330     public static final int ARRAY_LONG_BASE_OFFSET
1331             = theUnsafe.arrayBaseOffset(long[].class);
1332 
1333     /** The value of {@code arrayBaseOffset(float[].class)} */
1334     public static final int ARRAY_FLOAT_BASE_OFFSET
1335             = theUnsafe.arrayBaseOffset(float[].class);
1336 
1337     /** The value of {@code arrayBaseOffset(double[].class)} */
1338     public static final int ARRAY_DOUBLE_BASE_OFFSET
1339             = theUnsafe.arrayBaseOffset(double[].class);
1340 
1341     /** The value of {@code arrayBaseOffset(Object[].class)} */
1342     public static final int ARRAY_OBJECT_BASE_OFFSET
1343             = theUnsafe.arrayBaseOffset(Object[].class);
1344 
1345     /**
1346      * Reports the scale factor for addressing elements in the storage
1347      * allocation of a given array class.  However, arrays of "narrow" types
1348      * will generally not work properly with accessors like {@link
1349      * #getByte(Object, long)}, so the scale factor for such classes is reported
1350      * as zero.
1351      *
1352      * @see #arrayBaseOffset
1353      * @see #getInt(Object, long)
1354      * @see #putInt(Object, long, int)
1355      */
1356     public int arrayIndexScale(Class<?> arrayClass) {
1357         if (arrayClass == null) {
1358             throw new NullPointerException();
1359         }
1360 
1361         return arrayIndexScale0(arrayClass);
1362     }
1363 
1364     /**
1365      * Return the size of the object in the heap.
1366      * @param o an object
1367      * @return the objects's size
1368      * @since Valhalla
1369      */
1370     public long getObjectSize(Object o) {
1371         if (o == null)
1372             throw new NullPointerException();
1373         return getObjectSize0(o);
1374     }
1375 
1376     /** The value of {@code arrayIndexScale(boolean[].class)} */
1377     public static final int ARRAY_BOOLEAN_INDEX_SCALE
1378             = theUnsafe.arrayIndexScale(boolean[].class);
1379 
1380     /** The value of {@code arrayIndexScale(byte[].class)} */
1381     public static final int ARRAY_BYTE_INDEX_SCALE
1382             = theUnsafe.arrayIndexScale(byte[].class);
1383 
1384     /** The value of {@code arrayIndexScale(short[].class)} */
1385     public static final int ARRAY_SHORT_INDEX_SCALE
1386             = theUnsafe.arrayIndexScale(short[].class);
1387 
1388     /** The value of {@code arrayIndexScale(char[].class)} */
1389     public static final int ARRAY_CHAR_INDEX_SCALE
1390             = theUnsafe.arrayIndexScale(char[].class);
1391 
1392     /** The value of {@code arrayIndexScale(int[].class)} */
1393     public static final int ARRAY_INT_INDEX_SCALE
1394             = theUnsafe.arrayIndexScale(int[].class);
1395 
1396     /** The value of {@code arrayIndexScale(long[].class)} */
1397     public static final int ARRAY_LONG_INDEX_SCALE
1398             = theUnsafe.arrayIndexScale(long[].class);
1399 
1400     /** The value of {@code arrayIndexScale(float[].class)} */
1401     public static final int ARRAY_FLOAT_INDEX_SCALE
1402             = theUnsafe.arrayIndexScale(float[].class);
1403 
1404     /** The value of {@code arrayIndexScale(double[].class)} */
1405     public static final int ARRAY_DOUBLE_INDEX_SCALE
1406             = theUnsafe.arrayIndexScale(double[].class);
1407 
1408     /** The value of {@code arrayIndexScale(Object[].class)} */
1409     public static final int ARRAY_OBJECT_INDEX_SCALE
1410             = theUnsafe.arrayIndexScale(Object[].class);
1411 
1412     /**
1413      * Reports the size in bytes of a native pointer, as stored via {@link
1414      * #putAddress}.  This value will be either 4 or 8.  Note that the sizes of
1415      * other primitive types (as stored in native memory blocks) is determined
1416      * fully by their information content.
1417      */
1418     public int addressSize() {
1419         return ADDRESS_SIZE;
1420     }
1421 
1422     /** The value of {@code addressSize()} */
1423     public static final int ADDRESS_SIZE = ADDRESS_SIZE0;
1424 
1425     /**
1426      * Reports the size in bytes of a native memory page (whatever that is).
1427      * This value will always be a power of two.
1428      */
1429     public int pageSize() { return PAGE_SIZE; }
1430 
1431     /**
1432      * Reports the size in bytes of a data cache line written back by
1433      * the hardware cache line flush operation available to the JVM or
1434      * 0 if data cache line flushing is not enabled.
1435      */
1436     public int dataCacheLineFlushSize() { return DATA_CACHE_LINE_FLUSH_SIZE; }
1437 
1438     /**
1439      * Rounds down address to a data cache line boundary as
1440      * determined by {@link #dataCacheLineFlushSize}
1441      * @return the rounded down address
1442      */
1443     public long dataCacheLineAlignDown(long address) {
1444         return (address & ~(DATA_CACHE_LINE_FLUSH_SIZE - 1));
1445     }
1446 
1447     /**
1448      * Returns true if data cache line writeback
1449      */
1450     public static boolean isWritebackEnabled() { return DATA_CACHE_LINE_FLUSH_SIZE != 0; }
1451 
1452     /// random trusted operations from JNI:
1453 
1454     /**
1455      * Tells the VM to define a class, without security checks.  By default, the
1456      * class loader and protection domain come from the caller's class.
1457      */
1458     public Class<?> defineClass(String name, byte[] b, int off, int len,
1459                                 ClassLoader loader,
1460                                 ProtectionDomain protectionDomain) {
1461         if (b == null) {
1462             throw new NullPointerException();
1463         }
1464         if (len < 0) {
1465             throw new ArrayIndexOutOfBoundsException();
1466         }
1467 
1468         return defineClass0(name, b, off, len, loader, protectionDomain);
1469     }
1470 
1471     public native Class<?> defineClass0(String name, byte[] b, int off, int len,
1472                                         ClassLoader loader,
1473                                         ProtectionDomain protectionDomain);
1474 
1475     /**
1476      * Allocates an instance but does not run any constructor.
1477      * Initializes the class if it has not yet been.
1478      */
1479     @IntrinsicCandidate
1480     public native Object allocateInstance(Class<?> cls)
1481         throws InstantiationException;
1482 
1483     /**
1484      * Allocates an array of a given type, but does not do zeroing.
1485      * <p>
1486      * This method should only be used in the very rare cases where a high-performance code
1487      * overwrites the destination array completely, and compilers cannot assist in zeroing elimination.
1488      * In an overwhelming majority of cases, a normal Java allocation should be used instead.
1489      * <p>
1490      * Users of this method are <b>required</b> to overwrite the initial (garbage) array contents
1491      * before allowing untrusted code, or code in other threads, to observe the reference
1492      * to the newly allocated array. In addition, the publication of the array reference must be
1493      * safe according to the Java Memory Model requirements.
1494      * <p>
1495      * The safest approach to deal with an uninitialized array is to keep the reference to it in local
1496      * variable at least until the initialization is complete, and then publish it <b>once</b>, either
1497      * by writing it to a <em>volatile</em> field, or storing it into a <em>final</em> field in constructor,
1498      * or issuing a {@link #storeFence} before publishing the reference.
1499      * <p>
1500      * @implnote This method can only allocate primitive arrays, to avoid garbage reference
1501      * elements that could break heap integrity.
1502      *
1503      * @param componentType array component type to allocate
1504      * @param length array size to allocate
1505      * @throws IllegalArgumentException if component type is null, or not a primitive class;
1506      *                                  or the length is negative
1507      */
1508     public Object allocateUninitializedArray(Class<?> componentType, int length) {
1509        if (componentType == null) {
1510            throw new IllegalArgumentException("Component type is null");
1511        }
1512        if (!componentType.isPrimitive()) {
1513            throw new IllegalArgumentException("Component type is not primitive");
1514        }
1515        if (length < 0) {
1516            throw new IllegalArgumentException("Negative length");
1517        }
1518        return allocateUninitializedArray0(componentType, length);
1519     }
1520 
1521     @IntrinsicCandidate
1522     private Object allocateUninitializedArray0(Class<?> componentType, int length) {
1523        // These fallbacks provide zeroed arrays, but intrinsic is not required to
1524        // return the zeroed arrays.
1525        if (componentType == byte.class)    return new byte[length];
1526        if (componentType == boolean.class) return new boolean[length];
1527        if (componentType == short.class)   return new short[length];
1528        if (componentType == char.class)    return new char[length];
1529        if (componentType == int.class)     return new int[length];
1530        if (componentType == float.class)   return new float[length];
1531        if (componentType == long.class)    return new long[length];
1532        if (componentType == double.class)  return new double[length];
1533        return null;
1534     }
1535 
1536     /** Throws the exception without telling the verifier. */
1537     public native void throwException(Throwable ee);
1538 
1539     /**
1540      * Atomically updates Java variable to {@code x} if it is currently
1541      * holding {@code expected}.
1542      *
1543      * <p>This operation has memory semantics of a {@code volatile} read
1544      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1545      *
1546      * @return {@code true} if successful
1547      */
1548     @IntrinsicCandidate
1549     public final native boolean compareAndSetReference(Object o, long offset,
1550                                                        Object expected,
1551                                                        Object x);
1552 
1553     private final boolean isValueObject(Object o) {
1554         return o != null && o.getClass().isValue();
1555     }
1556 
1557     /*
1558      * For value type, CAS should do substitutability test as opposed
1559      * to two pointers comparison.
1560      *
1561      * TODO: replace global lock workaround with the proper support for
1562      * atomic access to value objects and loosely consistent values.
1563      */
1564     public final <V> boolean compareAndSetReference(Object o, long offset,
1565                                                     Class<?> type,
1566                                                     V expected,
1567                                                     V x) {
1568         if (type.isValue() || isValueObject(expected)) {
1569             synchronized (valueLock) {
1570                 Object witness = getReference(o, offset);
1571                 if (witness == expected) {
1572                     putReference(o, offset, x);
1573                     return true;
1574                 } else {
1575                     return false;
1576                 }
1577             }
1578         } else {
1579             return compareAndSetReference(o, offset, expected, x);
1580         }
1581     }
1582 
1583     @ForceInline
1584     public final <V> boolean compareAndSetValue(Object o, long offset,
1585                                                 Class<?> valueType,
1586                                                 V expected,
1587                                                 V x) {
1588         synchronized (valueLock) {
1589             Object witness = getValue(o, offset, valueType);
1590             if (witness == expected) {
1591                 putValue(o, offset, valueType, x);
1592                 return true;
1593             }
1594             else {
1595                 return false;
1596             }
1597         }
1598     }
1599 
1600     @IntrinsicCandidate
1601     public final native Object compareAndExchangeReference(Object o, long offset,
1602                                                            Object expected,
1603                                                            Object x);
1604 
1605     public final <V> Object compareAndExchangeReference(Object o, long offset,
1606                                                         Class<?> valueType,
1607                                                         V expected,
1608                                                         V x) {
1609         if (valueType.isValue() || isValueObject(expected)) {
1610             synchronized (valueLock) {
1611                 Object witness = getReference(o, offset);
1612                 if (witness == expected) {
1613                     putReference(o, offset, x);
1614                 }
1615                 return witness;
1616             }
1617         } else {
1618             return compareAndExchangeReference(o, offset, expected, x);
1619         }
1620     }
1621 
1622     @ForceInline
1623     public final <V> Object compareAndExchangeValue(Object o, long offset,
1624                                                     Class<?> valueType,
1625                                                     V expected,
1626                                                     V x) {
1627         synchronized (valueLock) {
1628             Object witness = getValue(o, offset, valueType);
1629             if (witness == expected) {
1630                 putValue(o, offset, valueType, x);
1631             }
1632             return witness;
1633         }
1634     }
1635 
1636     @IntrinsicCandidate
1637     public final Object compareAndExchangeReferenceAcquire(Object o, long offset,
1638                                                            Object expected,
1639                                                            Object x) {
1640         return compareAndExchangeReference(o, offset, expected, x);
1641     }
1642 
1643     public final <V> Object compareAndExchangeReferenceAcquire(Object o, long offset,
1644                                                                Class<?> valueType,
1645                                                                V expected,
1646                                                                V x) {
1647         return compareAndExchangeReference(o, offset, valueType, expected, x);
1648     }
1649 
1650     @ForceInline
1651     public final <V> Object compareAndExchangeValueAcquire(Object o, long offset,
1652                                                            Class<?> valueType,
1653                                                            V expected,
1654                                                            V x) {
1655         return compareAndExchangeValue(o, offset, valueType, expected, x);
1656     }
1657 
1658     @IntrinsicCandidate
1659     public final Object compareAndExchangeReferenceRelease(Object o, long offset,
1660                                                            Object expected,
1661                                                            Object x) {
1662         return compareAndExchangeReference(o, offset, expected, x);
1663     }
1664 
1665     public final <V> Object compareAndExchangeReferenceRelease(Object o, long offset,
1666                                                                Class<?> valueType,
1667                                                                V expected,
1668                                                                V x) {
1669         return compareAndExchangeReference(o, offset, valueType, expected, x);
1670     }
1671 
1672     @ForceInline
1673     public final <V> Object compareAndExchangeValueRelease(Object o, long offset,
1674                                                            Class<?> valueType,
1675                                                            V expected,
1676                                                            V x) {
1677         return compareAndExchangeValue(o, offset, valueType, expected, x);
1678     }
1679 
1680     @IntrinsicCandidate
1681     public final boolean weakCompareAndSetReferencePlain(Object o, long offset,
1682                                                          Object expected,
1683                                                          Object x) {
1684         return compareAndSetReference(o, offset, expected, x);
1685     }
1686 
1687     public final <V> boolean weakCompareAndSetReferencePlain(Object o, long offset,
1688                                                              Class<?> valueType,
1689                                                              V expected,
1690                                                              V x) {
1691         if (valueType.isValue() || isValueObject(expected)) {
1692             return compareAndSetReference(o, offset, valueType, expected, x);
1693         } else {
1694             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1695         }
1696     }
1697 
1698     @ForceInline
1699     public final <V> boolean weakCompareAndSetValuePlain(Object o, long offset,
1700                                                          Class<?> valueType,
1701                                                          V expected,
1702                                                          V x) {
1703         return compareAndSetValue(o, offset, valueType, expected, x);
1704     }
1705 
1706     @IntrinsicCandidate
1707     public final boolean weakCompareAndSetReferenceAcquire(Object o, long offset,
1708                                                            Object expected,
1709                                                            Object x) {
1710         return compareAndSetReference(o, offset, expected, x);
1711     }
1712 
1713     public final <V> boolean weakCompareAndSetReferenceAcquire(Object o, long offset,
1714                                                                Class<?> valueType,
1715                                                                V expected,
1716                                                                V x) {
1717         if (valueType.isValue() || isValueObject(expected)) {
1718             return compareAndSetReference(o, offset, valueType, expected, x);
1719         } else {
1720             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1721         }
1722     }
1723 
1724     @ForceInline
1725     public final <V> boolean weakCompareAndSetValueAcquire(Object o, long offset,
1726                                                            Class<?> valueType,
1727                                                            V expected,
1728                                                            V x) {
1729         return compareAndSetValue(o, offset, valueType, expected, x);
1730     }
1731 
1732     @IntrinsicCandidate
1733     public final boolean weakCompareAndSetReferenceRelease(Object o, long offset,
1734                                                            Object expected,
1735                                                            Object x) {
1736         return compareAndSetReference(o, offset, expected, x);
1737     }
1738 
1739     public final <V> boolean weakCompareAndSetReferenceRelease(Object o, long offset,
1740                                                                Class<?> valueType,
1741                                                                V expected,
1742                                                                V x) {
1743         if (valueType.isValue() || isValueObject(expected)) {
1744             return compareAndSetReference(o, offset, valueType, expected, x);
1745         } else {
1746             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1747         }
1748     }
1749 
1750     @ForceInline
1751     public final <V> boolean weakCompareAndSetValueRelease(Object o, long offset,
1752                                                            Class<?> valueType,
1753                                                            V expected,
1754                                                            V x) {
1755         return compareAndSetValue(o, offset, valueType, expected, x);
1756     }
1757 
1758     @IntrinsicCandidate
1759     public final boolean weakCompareAndSetReference(Object o, long offset,
1760                                                     Object expected,
1761                                                     Object x) {
1762         return compareAndSetReference(o, offset, expected, x);
1763     }
1764 
1765     public final <V> boolean weakCompareAndSetReference(Object o, long offset,
1766                                                         Class<?> valueType,
1767                                                         V expected,
1768                                                         V x) {
1769         if (valueType.isValue() || isValueObject(expected)) {
1770             return compareAndSetReference(o, offset, valueType, expected, x);
1771         } else {
1772             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1773         }
1774     }
1775 
1776     @ForceInline
1777     public final <V> boolean weakCompareAndSetValue(Object o, long offset,
1778                                                     Class<?> valueType,
1779                                                     V expected,
1780                                                     V x) {
1781         return compareAndSetValue(o, offset, valueType, expected, x);
1782     }
1783 
1784     /**
1785      * Atomically updates Java variable to {@code x} if it is currently
1786      * holding {@code expected}.
1787      *
1788      * <p>This operation has memory semantics of a {@code volatile} read
1789      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1790      *
1791      * @return {@code true} if successful
1792      */
1793     @IntrinsicCandidate
1794     public final native boolean compareAndSetInt(Object o, long offset,
1795                                                  int expected,
1796                                                  int x);
1797 
1798     @IntrinsicCandidate
1799     public final native int compareAndExchangeInt(Object o, long offset,
1800                                                   int expected,
1801                                                   int x);
1802 
1803     @IntrinsicCandidate
1804     public final int compareAndExchangeIntAcquire(Object o, long offset,
1805                                                          int expected,
1806                                                          int x) {
1807         return compareAndExchangeInt(o, offset, expected, x);
1808     }
1809 
1810     @IntrinsicCandidate
1811     public final int compareAndExchangeIntRelease(Object o, long offset,
1812                                                          int expected,
1813                                                          int x) {
1814         return compareAndExchangeInt(o, offset, expected, x);
1815     }
1816 
1817     @IntrinsicCandidate
1818     public final boolean weakCompareAndSetIntPlain(Object o, long offset,
1819                                                    int expected,
1820                                                    int x) {
1821         return compareAndSetInt(o, offset, expected, x);
1822     }
1823 
1824     @IntrinsicCandidate
1825     public final boolean weakCompareAndSetIntAcquire(Object o, long offset,
1826                                                      int expected,
1827                                                      int x) {
1828         return compareAndSetInt(o, offset, expected, x);
1829     }
1830 
1831     @IntrinsicCandidate
1832     public final boolean weakCompareAndSetIntRelease(Object o, long offset,
1833                                                      int expected,
1834                                                      int x) {
1835         return compareAndSetInt(o, offset, expected, x);
1836     }
1837 
1838     @IntrinsicCandidate
1839     public final boolean weakCompareAndSetInt(Object o, long offset,
1840                                               int expected,
1841                                               int x) {
1842         return compareAndSetInt(o, offset, expected, x);
1843     }
1844 
1845     @IntrinsicCandidate
1846     public final byte compareAndExchangeByte(Object o, long offset,
1847                                              byte expected,
1848                                              byte x) {
1849         long wordOffset = offset & ~3;
1850         int shift = (int) (offset & 3) << 3;
1851         if (BIG_ENDIAN) {
1852             shift = 24 - shift;
1853         }
1854         int mask           = 0xFF << shift;
1855         int maskedExpected = (expected & 0xFF) << shift;
1856         int maskedX        = (x & 0xFF) << shift;
1857         int fullWord;
1858         do {
1859             fullWord = getIntVolatile(o, wordOffset);
1860             if ((fullWord & mask) != maskedExpected)
1861                 return (byte) ((fullWord & mask) >> shift);
1862         } while (!weakCompareAndSetInt(o, wordOffset,
1863                                                 fullWord, (fullWord & ~mask) | maskedX));
1864         return expected;
1865     }
1866 
1867     @IntrinsicCandidate
1868     public final boolean compareAndSetByte(Object o, long offset,
1869                                            byte expected,
1870                                            byte x) {
1871         return compareAndExchangeByte(o, offset, expected, x) == expected;
1872     }
1873 
1874     @IntrinsicCandidate
1875     public final boolean weakCompareAndSetByte(Object o, long offset,
1876                                                byte expected,
1877                                                byte x) {
1878         return compareAndSetByte(o, offset, expected, x);
1879     }
1880 
1881     @IntrinsicCandidate
1882     public final boolean weakCompareAndSetByteAcquire(Object o, long offset,
1883                                                       byte expected,
1884                                                       byte x) {
1885         return weakCompareAndSetByte(o, offset, expected, x);
1886     }
1887 
1888     @IntrinsicCandidate
1889     public final boolean weakCompareAndSetByteRelease(Object o, long offset,
1890                                                       byte expected,
1891                                                       byte x) {
1892         return weakCompareAndSetByte(o, offset, expected, x);
1893     }
1894 
1895     @IntrinsicCandidate
1896     public final boolean weakCompareAndSetBytePlain(Object o, long offset,
1897                                                     byte expected,
1898                                                     byte x) {
1899         return weakCompareAndSetByte(o, offset, expected, x);
1900     }
1901 
1902     @IntrinsicCandidate
1903     public final byte compareAndExchangeByteAcquire(Object o, long offset,
1904                                                     byte expected,
1905                                                     byte x) {
1906         return compareAndExchangeByte(o, offset, expected, x);
1907     }
1908 
1909     @IntrinsicCandidate
1910     public final byte compareAndExchangeByteRelease(Object o, long offset,
1911                                                     byte expected,
1912                                                     byte x) {
1913         return compareAndExchangeByte(o, offset, expected, x);
1914     }
1915 
1916     @IntrinsicCandidate
1917     public final short compareAndExchangeShort(Object o, long offset,
1918                                                short expected,
1919                                                short x) {
1920         if ((offset & 3) == 3) {
1921             throw new IllegalArgumentException("Update spans the word, not supported");
1922         }
1923         long wordOffset = offset & ~3;
1924         int shift = (int) (offset & 3) << 3;
1925         if (BIG_ENDIAN) {
1926             shift = 16 - shift;
1927         }
1928         int mask           = 0xFFFF << shift;
1929         int maskedExpected = (expected & 0xFFFF) << shift;
1930         int maskedX        = (x & 0xFFFF) << shift;
1931         int fullWord;
1932         do {
1933             fullWord = getIntVolatile(o, wordOffset);
1934             if ((fullWord & mask) != maskedExpected) {
1935                 return (short) ((fullWord & mask) >> shift);
1936             }
1937         } while (!weakCompareAndSetInt(o, wordOffset,
1938                                                 fullWord, (fullWord & ~mask) | maskedX));
1939         return expected;
1940     }
1941 
1942     @IntrinsicCandidate
1943     public final boolean compareAndSetShort(Object o, long offset,
1944                                             short expected,
1945                                             short x) {
1946         return compareAndExchangeShort(o, offset, expected, x) == expected;
1947     }
1948 
1949     @IntrinsicCandidate
1950     public final boolean weakCompareAndSetShort(Object o, long offset,
1951                                                 short expected,
1952                                                 short x) {
1953         return compareAndSetShort(o, offset, expected, x);
1954     }
1955 
1956     @IntrinsicCandidate
1957     public final boolean weakCompareAndSetShortAcquire(Object o, long offset,
1958                                                        short expected,
1959                                                        short x) {
1960         return weakCompareAndSetShort(o, offset, expected, x);
1961     }
1962 
1963     @IntrinsicCandidate
1964     public final boolean weakCompareAndSetShortRelease(Object o, long offset,
1965                                                        short expected,
1966                                                        short x) {
1967         return weakCompareAndSetShort(o, offset, expected, x);
1968     }
1969 
1970     @IntrinsicCandidate
1971     public final boolean weakCompareAndSetShortPlain(Object o, long offset,
1972                                                      short expected,
1973                                                      short x) {
1974         return weakCompareAndSetShort(o, offset, expected, x);
1975     }
1976 
1977 
1978     @IntrinsicCandidate
1979     public final short compareAndExchangeShortAcquire(Object o, long offset,
1980                                                      short expected,
1981                                                      short x) {
1982         return compareAndExchangeShort(o, offset, expected, x);
1983     }
1984 
1985     @IntrinsicCandidate
1986     public final short compareAndExchangeShortRelease(Object o, long offset,
1987                                                     short expected,
1988                                                     short x) {
1989         return compareAndExchangeShort(o, offset, expected, x);
1990     }
1991 
1992     @ForceInline
1993     private char s2c(short s) {
1994         return (char) s;
1995     }
1996 
1997     @ForceInline
1998     private short c2s(char s) {
1999         return (short) s;
2000     }
2001 
2002     @ForceInline
2003     public final boolean compareAndSetChar(Object o, long offset,
2004                                            char expected,
2005                                            char x) {
2006         return compareAndSetShort(o, offset, c2s(expected), c2s(x));
2007     }
2008 
2009     @ForceInline
2010     public final char compareAndExchangeChar(Object o, long offset,
2011                                              char expected,
2012                                              char x) {
2013         return s2c(compareAndExchangeShort(o, offset, c2s(expected), c2s(x)));
2014     }
2015 
2016     @ForceInline
2017     public final char compareAndExchangeCharAcquire(Object o, long offset,
2018                                             char expected,
2019                                             char x) {
2020         return s2c(compareAndExchangeShortAcquire(o, offset, c2s(expected), c2s(x)));
2021     }
2022 
2023     @ForceInline
2024     public final char compareAndExchangeCharRelease(Object o, long offset,
2025                                             char expected,
2026                                             char x) {
2027         return s2c(compareAndExchangeShortRelease(o, offset, c2s(expected), c2s(x)));
2028     }
2029 
2030     @ForceInline
2031     public final boolean weakCompareAndSetChar(Object o, long offset,
2032                                                char expected,
2033                                                char x) {
2034         return weakCompareAndSetShort(o, offset, c2s(expected), c2s(x));
2035     }
2036 
2037     @ForceInline
2038     public final boolean weakCompareAndSetCharAcquire(Object o, long offset,
2039                                                       char expected,
2040                                                       char x) {
2041         return weakCompareAndSetShortAcquire(o, offset, c2s(expected), c2s(x));
2042     }
2043 
2044     @ForceInline
2045     public final boolean weakCompareAndSetCharRelease(Object o, long offset,
2046                                                       char expected,
2047                                                       char x) {
2048         return weakCompareAndSetShortRelease(o, offset, c2s(expected), c2s(x));
2049     }
2050 
2051     @ForceInline
2052     public final boolean weakCompareAndSetCharPlain(Object o, long offset,
2053                                                     char expected,
2054                                                     char x) {
2055         return weakCompareAndSetShortPlain(o, offset, c2s(expected), c2s(x));
2056     }
2057 
2058     /**
2059      * The JVM converts integral values to boolean values using two
2060      * different conventions, byte testing against zero and truncation
2061      * to least-significant bit.
2062      *
2063      * <p>The JNI documents specify that, at least for returning
2064      * values from native methods, a Java boolean value is converted
2065      * to the value-set 0..1 by first truncating to a byte (0..255 or
2066      * maybe -128..127) and then testing against zero. Thus, Java
2067      * booleans in non-Java data structures are by convention
2068      * represented as 8-bit containers containing either zero (for
2069      * false) or any non-zero value (for true).
2070      *
2071      * <p>Java booleans in the heap are also stored in bytes, but are
2072      * strongly normalized to the value-set 0..1 (i.e., they are
2073      * truncated to the least-significant bit).
2074      *
2075      * <p>The main reason for having different conventions for
2076      * conversion is performance: Truncation to the least-significant
2077      * bit can be usually implemented with fewer (machine)
2078      * instructions than byte testing against zero.
2079      *
2080      * <p>A number of Unsafe methods load boolean values from the heap
2081      * as bytes. Unsafe converts those values according to the JNI
2082      * rules (i.e, using the "testing against zero" convention). The
2083      * method {@code byte2bool} implements that conversion.
2084      *
2085      * @param b the byte to be converted to boolean
2086      * @return the result of the conversion
2087      */
2088     @ForceInline
2089     private boolean byte2bool(byte b) {
2090         return b != 0;
2091     }
2092 
2093     /**
2094      * Convert a boolean value to a byte. The return value is strongly
2095      * normalized to the value-set 0..1 (i.e., the value is truncated
2096      * to the least-significant bit). See {@link #byte2bool(byte)} for
2097      * more details on conversion conventions.
2098      *
2099      * @param b the boolean to be converted to byte (and then normalized)
2100      * @return the result of the conversion
2101      */
2102     @ForceInline
2103     private byte bool2byte(boolean b) {
2104         return b ? (byte)1 : (byte)0;
2105     }
2106 
2107     @ForceInline
2108     public final boolean compareAndSetBoolean(Object o, long offset,
2109                                               boolean expected,
2110                                               boolean x) {
2111         return compareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
2112     }
2113 
2114     @ForceInline
2115     public final boolean compareAndExchangeBoolean(Object o, long offset,
2116                                                    boolean expected,
2117                                                    boolean x) {
2118         return byte2bool(compareAndExchangeByte(o, offset, bool2byte(expected), bool2byte(x)));
2119     }
2120 
2121     @ForceInline
2122     public final boolean compareAndExchangeBooleanAcquire(Object o, long offset,
2123                                                     boolean expected,
2124                                                     boolean x) {
2125         return byte2bool(compareAndExchangeByteAcquire(o, offset, bool2byte(expected), bool2byte(x)));
2126     }
2127 
2128     @ForceInline
2129     public final boolean compareAndExchangeBooleanRelease(Object o, long offset,
2130                                                        boolean expected,
2131                                                        boolean x) {
2132         return byte2bool(compareAndExchangeByteRelease(o, offset, bool2byte(expected), bool2byte(x)));
2133     }
2134 
2135     @ForceInline
2136     public final boolean weakCompareAndSetBoolean(Object o, long offset,
2137                                                   boolean expected,
2138                                                   boolean x) {
2139         return weakCompareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
2140     }
2141 
2142     @ForceInline
2143     public final boolean weakCompareAndSetBooleanAcquire(Object o, long offset,
2144                                                          boolean expected,
2145                                                          boolean x) {
2146         return weakCompareAndSetByteAcquire(o, offset, bool2byte(expected), bool2byte(x));
2147     }
2148 
2149     @ForceInline
2150     public final boolean weakCompareAndSetBooleanRelease(Object o, long offset,
2151                                                          boolean expected,
2152                                                          boolean x) {
2153         return weakCompareAndSetByteRelease(o, offset, bool2byte(expected), bool2byte(x));
2154     }
2155 
2156     @ForceInline
2157     public final boolean weakCompareAndSetBooleanPlain(Object o, long offset,
2158                                                        boolean expected,
2159                                                        boolean x) {
2160         return weakCompareAndSetBytePlain(o, offset, bool2byte(expected), bool2byte(x));
2161     }
2162 
2163     /**
2164      * Atomically updates Java variable to {@code x} if it is currently
2165      * holding {@code expected}.
2166      *
2167      * <p>This operation has memory semantics of a {@code volatile} read
2168      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2169      *
2170      * @return {@code true} if successful
2171      */
2172     @ForceInline
2173     public final boolean compareAndSetFloat(Object o, long offset,
2174                                             float expected,
2175                                             float x) {
2176         return compareAndSetInt(o, offset,
2177                                  Float.floatToRawIntBits(expected),
2178                                  Float.floatToRawIntBits(x));
2179     }
2180 
2181     @ForceInline
2182     public final float compareAndExchangeFloat(Object o, long offset,
2183                                                float expected,
2184                                                float x) {
2185         int w = compareAndExchangeInt(o, offset,
2186                                       Float.floatToRawIntBits(expected),
2187                                       Float.floatToRawIntBits(x));
2188         return Float.intBitsToFloat(w);
2189     }
2190 
2191     @ForceInline
2192     public final float compareAndExchangeFloatAcquire(Object o, long offset,
2193                                                   float expected,
2194                                                   float x) {
2195         int w = compareAndExchangeIntAcquire(o, offset,
2196                                              Float.floatToRawIntBits(expected),
2197                                              Float.floatToRawIntBits(x));
2198         return Float.intBitsToFloat(w);
2199     }
2200 
2201     @ForceInline
2202     public final float compareAndExchangeFloatRelease(Object o, long offset,
2203                                                   float expected,
2204                                                   float x) {
2205         int w = compareAndExchangeIntRelease(o, offset,
2206                                              Float.floatToRawIntBits(expected),
2207                                              Float.floatToRawIntBits(x));
2208         return Float.intBitsToFloat(w);
2209     }
2210 
2211     @ForceInline
2212     public final boolean weakCompareAndSetFloatPlain(Object o, long offset,
2213                                                      float expected,
2214                                                      float x) {
2215         return weakCompareAndSetIntPlain(o, offset,
2216                                      Float.floatToRawIntBits(expected),
2217                                      Float.floatToRawIntBits(x));
2218     }
2219 
2220     @ForceInline
2221     public final boolean weakCompareAndSetFloatAcquire(Object o, long offset,
2222                                                        float expected,
2223                                                        float x) {
2224         return weakCompareAndSetIntAcquire(o, offset,
2225                                             Float.floatToRawIntBits(expected),
2226                                             Float.floatToRawIntBits(x));
2227     }
2228 
2229     @ForceInline
2230     public final boolean weakCompareAndSetFloatRelease(Object o, long offset,
2231                                                        float expected,
2232                                                        float x) {
2233         return weakCompareAndSetIntRelease(o, offset,
2234                                             Float.floatToRawIntBits(expected),
2235                                             Float.floatToRawIntBits(x));
2236     }
2237 
2238     @ForceInline
2239     public final boolean weakCompareAndSetFloat(Object o, long offset,
2240                                                 float expected,
2241                                                 float x) {
2242         return weakCompareAndSetInt(o, offset,
2243                                              Float.floatToRawIntBits(expected),
2244                                              Float.floatToRawIntBits(x));
2245     }
2246 
2247     /**
2248      * Atomically updates Java variable to {@code x} if it is currently
2249      * holding {@code expected}.
2250      *
2251      * <p>This operation has memory semantics of a {@code volatile} read
2252      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2253      *
2254      * @return {@code true} if successful
2255      */
2256     @ForceInline
2257     public final boolean compareAndSetDouble(Object o, long offset,
2258                                              double expected,
2259                                              double x) {
2260         return compareAndSetLong(o, offset,
2261                                  Double.doubleToRawLongBits(expected),
2262                                  Double.doubleToRawLongBits(x));
2263     }
2264 
2265     @ForceInline
2266     public final double compareAndExchangeDouble(Object o, long offset,
2267                                                  double expected,
2268                                                  double x) {
2269         long w = compareAndExchangeLong(o, offset,
2270                                         Double.doubleToRawLongBits(expected),
2271                                         Double.doubleToRawLongBits(x));
2272         return Double.longBitsToDouble(w);
2273     }
2274 
2275     @ForceInline
2276     public final double compareAndExchangeDoubleAcquire(Object o, long offset,
2277                                                         double expected,
2278                                                         double x) {
2279         long w = compareAndExchangeLongAcquire(o, offset,
2280                                                Double.doubleToRawLongBits(expected),
2281                                                Double.doubleToRawLongBits(x));
2282         return Double.longBitsToDouble(w);
2283     }
2284 
2285     @ForceInline
2286     public final double compareAndExchangeDoubleRelease(Object o, long offset,
2287                                                         double expected,
2288                                                         double x) {
2289         long w = compareAndExchangeLongRelease(o, offset,
2290                                                Double.doubleToRawLongBits(expected),
2291                                                Double.doubleToRawLongBits(x));
2292         return Double.longBitsToDouble(w);
2293     }
2294 
2295     @ForceInline
2296     public final boolean weakCompareAndSetDoublePlain(Object o, long offset,
2297                                                       double expected,
2298                                                       double x) {
2299         return weakCompareAndSetLongPlain(o, offset,
2300                                      Double.doubleToRawLongBits(expected),
2301                                      Double.doubleToRawLongBits(x));
2302     }
2303 
2304     @ForceInline
2305     public final boolean weakCompareAndSetDoubleAcquire(Object o, long offset,
2306                                                         double expected,
2307                                                         double x) {
2308         return weakCompareAndSetLongAcquire(o, offset,
2309                                              Double.doubleToRawLongBits(expected),
2310                                              Double.doubleToRawLongBits(x));
2311     }
2312 
2313     @ForceInline
2314     public final boolean weakCompareAndSetDoubleRelease(Object o, long offset,
2315                                                         double expected,
2316                                                         double x) {
2317         return weakCompareAndSetLongRelease(o, offset,
2318                                              Double.doubleToRawLongBits(expected),
2319                                              Double.doubleToRawLongBits(x));
2320     }
2321 
2322     @ForceInline
2323     public final boolean weakCompareAndSetDouble(Object o, long offset,
2324                                                  double expected,
2325                                                  double x) {
2326         return weakCompareAndSetLong(o, offset,
2327                                               Double.doubleToRawLongBits(expected),
2328                                               Double.doubleToRawLongBits(x));
2329     }
2330 
2331     /**
2332      * Atomically updates Java variable to {@code x} if it is currently
2333      * holding {@code expected}.
2334      *
2335      * <p>This operation has memory semantics of a {@code volatile} read
2336      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2337      *
2338      * @return {@code true} if successful
2339      */
2340     @IntrinsicCandidate
2341     public final native boolean compareAndSetLong(Object o, long offset,
2342                                                   long expected,
2343                                                   long x);
2344 
2345     @IntrinsicCandidate
2346     public final native long compareAndExchangeLong(Object o, long offset,
2347                                                     long expected,
2348                                                     long x);
2349 
2350     @IntrinsicCandidate
2351     public final long compareAndExchangeLongAcquire(Object o, long offset,
2352                                                            long expected,
2353                                                            long x) {
2354         return compareAndExchangeLong(o, offset, expected, x);
2355     }
2356 
2357     @IntrinsicCandidate
2358     public final long compareAndExchangeLongRelease(Object o, long offset,
2359                                                            long expected,
2360                                                            long x) {
2361         return compareAndExchangeLong(o, offset, expected, x);
2362     }
2363 
2364     @IntrinsicCandidate
2365     public final boolean weakCompareAndSetLongPlain(Object o, long offset,
2366                                                     long expected,
2367                                                     long x) {
2368         return compareAndSetLong(o, offset, expected, x);
2369     }
2370 
2371     @IntrinsicCandidate
2372     public final boolean weakCompareAndSetLongAcquire(Object o, long offset,
2373                                                       long expected,
2374                                                       long x) {
2375         return compareAndSetLong(o, offset, expected, x);
2376     }
2377 
2378     @IntrinsicCandidate
2379     public final boolean weakCompareAndSetLongRelease(Object o, long offset,
2380                                                       long expected,
2381                                                       long x) {
2382         return compareAndSetLong(o, offset, expected, x);
2383     }
2384 
2385     @IntrinsicCandidate
2386     public final boolean weakCompareAndSetLong(Object o, long offset,
2387                                                long expected,
2388                                                long x) {
2389         return compareAndSetLong(o, offset, expected, x);
2390     }
2391 
2392     /**
2393      * Fetches a reference value from a given Java variable, with volatile
2394      * load semantics. Otherwise identical to {@link #getReference(Object, long)}
2395      */
2396     @IntrinsicCandidate
2397     public native Object getReferenceVolatile(Object o, long offset);
2398 
2399     /**
2400      * Global lock for atomic and volatile strength access to any value of
2401      * a value type.  This is a temporary workaround until better localized
2402      * atomic access mechanisms are supported for value class and primitive class.
2403      */
2404     private static final Object valueLock = new Object();
2405 
2406     public final <V> Object getValueVolatile(Object base, long offset, Class<?> valueType) {
2407         synchronized (valueLock) {
2408             return getValue(base, offset, valueType);
2409         }
2410     }
2411 
2412     /**
2413      * Stores a reference value into a given Java variable, with
2414      * volatile store semantics. Otherwise identical to {@link #putReference(Object, long, Object)}
2415      */
2416     @IntrinsicCandidate
2417     public native void putReferenceVolatile(Object o, long offset, Object x);
2418 
2419     public final <V> void putValueVolatile(Object o, long offset, Class<?> valueType, V x) {
2420         synchronized (valueLock) {
2421             putValue(o, offset, valueType, x);
2422         }
2423     }
2424 
2425     /** Volatile version of {@link #getInt(Object, long)}  */
2426     @IntrinsicCandidate
2427     public native int     getIntVolatile(Object o, long offset);
2428 
2429     /** Volatile version of {@link #putInt(Object, long, int)}  */
2430     @IntrinsicCandidate
2431     public native void    putIntVolatile(Object o, long offset, int x);
2432 
2433     /** Volatile version of {@link #getBoolean(Object, long)}  */
2434     @IntrinsicCandidate
2435     public native boolean getBooleanVolatile(Object o, long offset);
2436 
2437     /** Volatile version of {@link #putBoolean(Object, long, boolean)}  */
2438     @IntrinsicCandidate
2439     public native void    putBooleanVolatile(Object o, long offset, boolean x);
2440 
2441     /** Volatile version of {@link #getByte(Object, long)}  */
2442     @IntrinsicCandidate
2443     public native byte    getByteVolatile(Object o, long offset);
2444 
2445     /** Volatile version of {@link #putByte(Object, long, byte)}  */
2446     @IntrinsicCandidate
2447     public native void    putByteVolatile(Object o, long offset, byte x);
2448 
2449     /** Volatile version of {@link #getShort(Object, long)}  */
2450     @IntrinsicCandidate
2451     public native short   getShortVolatile(Object o, long offset);
2452 
2453     /** Volatile version of {@link #putShort(Object, long, short)}  */
2454     @IntrinsicCandidate
2455     public native void    putShortVolatile(Object o, long offset, short x);
2456 
2457     /** Volatile version of {@link #getChar(Object, long)}  */
2458     @IntrinsicCandidate
2459     public native char    getCharVolatile(Object o, long offset);
2460 
2461     /** Volatile version of {@link #putChar(Object, long, char)}  */
2462     @IntrinsicCandidate
2463     public native void    putCharVolatile(Object o, long offset, char x);
2464 
2465     /** Volatile version of {@link #getLong(Object, long)}  */
2466     @IntrinsicCandidate
2467     public native long    getLongVolatile(Object o, long offset);
2468 
2469     /** Volatile version of {@link #putLong(Object, long, long)}  */
2470     @IntrinsicCandidate
2471     public native void    putLongVolatile(Object o, long offset, long x);
2472 
2473     /** Volatile version of {@link #getFloat(Object, long)}  */
2474     @IntrinsicCandidate
2475     public native float   getFloatVolatile(Object o, long offset);
2476 
2477     /** Volatile version of {@link #putFloat(Object, long, float)}  */
2478     @IntrinsicCandidate
2479     public native void    putFloatVolatile(Object o, long offset, float x);
2480 
2481     /** Volatile version of {@link #getDouble(Object, long)}  */
2482     @IntrinsicCandidate
2483     public native double  getDoubleVolatile(Object o, long offset);
2484 
2485     /** Volatile version of {@link #putDouble(Object, long, double)}  */
2486     @IntrinsicCandidate
2487     public native void    putDoubleVolatile(Object o, long offset, double x);
2488 
2489 
2490 
2491     /** Acquire version of {@link #getReferenceVolatile(Object, long)} */
2492     @IntrinsicCandidate
2493     public final Object getReferenceAcquire(Object o, long offset) {
2494         return getReferenceVolatile(o, offset);
2495     }
2496 
2497     public final <V> Object getValueAcquire(Object base, long offset, Class<?> valueType) {
2498         return getValueVolatile(base, offset, valueType);
2499     }
2500 
2501     /** Acquire version of {@link #getBooleanVolatile(Object, long)} */
2502     @IntrinsicCandidate
2503     public final boolean getBooleanAcquire(Object o, long offset) {
2504         return getBooleanVolatile(o, offset);
2505     }
2506 
2507     /** Acquire version of {@link #getByteVolatile(Object, long)} */
2508     @IntrinsicCandidate
2509     public final byte getByteAcquire(Object o, long offset) {
2510         return getByteVolatile(o, offset);
2511     }
2512 
2513     /** Acquire version of {@link #getShortVolatile(Object, long)} */
2514     @IntrinsicCandidate
2515     public final short getShortAcquire(Object o, long offset) {
2516         return getShortVolatile(o, offset);
2517     }
2518 
2519     /** Acquire version of {@link #getCharVolatile(Object, long)} */
2520     @IntrinsicCandidate
2521     public final char getCharAcquire(Object o, long offset) {
2522         return getCharVolatile(o, offset);
2523     }
2524 
2525     /** Acquire version of {@link #getIntVolatile(Object, long)} */
2526     @IntrinsicCandidate
2527     public final int getIntAcquire(Object o, long offset) {
2528         return getIntVolatile(o, offset);
2529     }
2530 
2531     /** Acquire version of {@link #getFloatVolatile(Object, long)} */
2532     @IntrinsicCandidate
2533     public final float getFloatAcquire(Object o, long offset) {
2534         return getFloatVolatile(o, offset);
2535     }
2536 
2537     /** Acquire version of {@link #getLongVolatile(Object, long)} */
2538     @IntrinsicCandidate
2539     public final long getLongAcquire(Object o, long offset) {
2540         return getLongVolatile(o, offset);
2541     }
2542 
2543     /** Acquire version of {@link #getDoubleVolatile(Object, long)} */
2544     @IntrinsicCandidate
2545     public final double getDoubleAcquire(Object o, long offset) {
2546         return getDoubleVolatile(o, offset);
2547     }
2548 
2549     /*
2550      * Versions of {@link #putReferenceVolatile(Object, long, Object)}
2551      * that do not guarantee immediate visibility of the store to
2552      * other threads. This method is generally only useful if the
2553      * underlying field is a Java volatile (or if an array cell, one
2554      * that is otherwise only accessed using volatile accesses).
2555      *
2556      * Corresponds to C11 atomic_store_explicit(..., memory_order_release).
2557      */
2558 
2559     /** Release version of {@link #putReferenceVolatile(Object, long, Object)} */
2560     @IntrinsicCandidate
2561     public final void putReferenceRelease(Object o, long offset, Object x) {
2562         putReferenceVolatile(o, offset, x);
2563     }
2564 
2565     public final <V> void putValueRelease(Object o, long offset, Class<?> valueType, V x) {
2566         putValueVolatile(o, offset, valueType, x);
2567     }
2568 
2569     /** Release version of {@link #putBooleanVolatile(Object, long, boolean)} */
2570     @IntrinsicCandidate
2571     public final void putBooleanRelease(Object o, long offset, boolean x) {
2572         putBooleanVolatile(o, offset, x);
2573     }
2574 
2575     /** Release version of {@link #putByteVolatile(Object, long, byte)} */
2576     @IntrinsicCandidate
2577     public final void putByteRelease(Object o, long offset, byte x) {
2578         putByteVolatile(o, offset, x);
2579     }
2580 
2581     /** Release version of {@link #putShortVolatile(Object, long, short)} */
2582     @IntrinsicCandidate
2583     public final void putShortRelease(Object o, long offset, short x) {
2584         putShortVolatile(o, offset, x);
2585     }
2586 
2587     /** Release version of {@link #putCharVolatile(Object, long, char)} */
2588     @IntrinsicCandidate
2589     public final void putCharRelease(Object o, long offset, char x) {
2590         putCharVolatile(o, offset, x);
2591     }
2592 
2593     /** Release version of {@link #putIntVolatile(Object, long, int)} */
2594     @IntrinsicCandidate
2595     public final void putIntRelease(Object o, long offset, int x) {
2596         putIntVolatile(o, offset, x);
2597     }
2598 
2599     /** Release version of {@link #putFloatVolatile(Object, long, float)} */
2600     @IntrinsicCandidate
2601     public final void putFloatRelease(Object o, long offset, float x) {
2602         putFloatVolatile(o, offset, x);
2603     }
2604 
2605     /** Release version of {@link #putLongVolatile(Object, long, long)} */
2606     @IntrinsicCandidate
2607     public final void putLongRelease(Object o, long offset, long x) {
2608         putLongVolatile(o, offset, x);
2609     }
2610 
2611     /** Release version of {@link #putDoubleVolatile(Object, long, double)} */
2612     @IntrinsicCandidate
2613     public final void putDoubleRelease(Object o, long offset, double x) {
2614         putDoubleVolatile(o, offset, x);
2615     }
2616 
2617     // ------------------------------ Opaque --------------------------------------
2618 
2619     /** Opaque version of {@link #getReferenceVolatile(Object, long)} */
2620     @IntrinsicCandidate
2621     public final Object getReferenceOpaque(Object o, long offset) {
2622         return getReferenceVolatile(o, offset);
2623     }
2624 
2625     public final <V> Object getValueOpaque(Object base, long offset, Class<?> valueType) {
2626         return getValueVolatile(base, offset, valueType);
2627     }
2628 
2629     /** Opaque version of {@link #getBooleanVolatile(Object, long)} */
2630     @IntrinsicCandidate
2631     public final boolean getBooleanOpaque(Object o, long offset) {
2632         return getBooleanVolatile(o, offset);
2633     }
2634 
2635     /** Opaque version of {@link #getByteVolatile(Object, long)} */
2636     @IntrinsicCandidate
2637     public final byte getByteOpaque(Object o, long offset) {
2638         return getByteVolatile(o, offset);
2639     }
2640 
2641     /** Opaque version of {@link #getShortVolatile(Object, long)} */
2642     @IntrinsicCandidate
2643     public final short getShortOpaque(Object o, long offset) {
2644         return getShortVolatile(o, offset);
2645     }
2646 
2647     /** Opaque version of {@link #getCharVolatile(Object, long)} */
2648     @IntrinsicCandidate
2649     public final char getCharOpaque(Object o, long offset) {
2650         return getCharVolatile(o, offset);
2651     }
2652 
2653     /** Opaque version of {@link #getIntVolatile(Object, long)} */
2654     @IntrinsicCandidate
2655     public final int getIntOpaque(Object o, long offset) {
2656         return getIntVolatile(o, offset);
2657     }
2658 
2659     /** Opaque version of {@link #getFloatVolatile(Object, long)} */
2660     @IntrinsicCandidate
2661     public final float getFloatOpaque(Object o, long offset) {
2662         return getFloatVolatile(o, offset);
2663     }
2664 
2665     /** Opaque version of {@link #getLongVolatile(Object, long)} */
2666     @IntrinsicCandidate
2667     public final long getLongOpaque(Object o, long offset) {
2668         return getLongVolatile(o, offset);
2669     }
2670 
2671     /** Opaque version of {@link #getDoubleVolatile(Object, long)} */
2672     @IntrinsicCandidate
2673     public final double getDoubleOpaque(Object o, long offset) {
2674         return getDoubleVolatile(o, offset);
2675     }
2676 
2677     /** Opaque version of {@link #putReferenceVolatile(Object, long, Object)} */
2678     @IntrinsicCandidate
2679     public final void putReferenceOpaque(Object o, long offset, Object x) {
2680         putReferenceVolatile(o, offset, x);
2681     }
2682 
2683     public final <V> void putValueOpaque(Object o, long offset, Class<?> valueType, V x) {
2684         putValueVolatile(o, offset, valueType, x);
2685     }
2686 
2687     /** Opaque version of {@link #putBooleanVolatile(Object, long, boolean)} */
2688     @IntrinsicCandidate
2689     public final void putBooleanOpaque(Object o, long offset, boolean x) {
2690         putBooleanVolatile(o, offset, x);
2691     }
2692 
2693     /** Opaque version of {@link #putByteVolatile(Object, long, byte)} */
2694     @IntrinsicCandidate
2695     public final void putByteOpaque(Object o, long offset, byte x) {
2696         putByteVolatile(o, offset, x);
2697     }
2698 
2699     /** Opaque version of {@link #putShortVolatile(Object, long, short)} */
2700     @IntrinsicCandidate
2701     public final void putShortOpaque(Object o, long offset, short x) {
2702         putShortVolatile(o, offset, x);
2703     }
2704 
2705     /** Opaque version of {@link #putCharVolatile(Object, long, char)} */
2706     @IntrinsicCandidate
2707     public final void putCharOpaque(Object o, long offset, char x) {
2708         putCharVolatile(o, offset, x);
2709     }
2710 
2711     /** Opaque version of {@link #putIntVolatile(Object, long, int)} */
2712     @IntrinsicCandidate
2713     public final void putIntOpaque(Object o, long offset, int x) {
2714         putIntVolatile(o, offset, x);
2715     }
2716 
2717     /** Opaque version of {@link #putFloatVolatile(Object, long, float)} */
2718     @IntrinsicCandidate
2719     public final void putFloatOpaque(Object o, long offset, float x) {
2720         putFloatVolatile(o, offset, x);
2721     }
2722 
2723     /** Opaque version of {@link #putLongVolatile(Object, long, long)} */
2724     @IntrinsicCandidate
2725     public final void putLongOpaque(Object o, long offset, long x) {
2726         putLongVolatile(o, offset, x);
2727     }
2728 
2729     /** Opaque version of {@link #putDoubleVolatile(Object, long, double)} */
2730     @IntrinsicCandidate
2731     public final void putDoubleOpaque(Object o, long offset, double x) {
2732         putDoubleVolatile(o, offset, x);
2733     }
2734 
2735     /**
2736      * Unblocks the given thread blocked on {@code park}, or, if it is
2737      * not blocked, causes the subsequent call to {@code park} not to
2738      * block.  Note: this operation is "unsafe" solely because the
2739      * caller must somehow ensure that the thread has not been
2740      * destroyed. Nothing special is usually required to ensure this
2741      * when called from Java (in which there will ordinarily be a live
2742      * reference to the thread) but this is not nearly-automatically
2743      * so when calling from native code.
2744      *
2745      * @param thread the thread to unpark.
2746      */
2747     @IntrinsicCandidate
2748     public native void unpark(Object thread);
2749 
2750     /**
2751      * Blocks current thread, returning when a balancing
2752      * {@code unpark} occurs, or a balancing {@code unpark} has
2753      * already occurred, or the thread is interrupted, or, if not
2754      * absolute and time is not zero, the given time nanoseconds have
2755      * elapsed, or if absolute, the given deadline in milliseconds
2756      * since Epoch has passed, or spuriously (i.e., returning for no
2757      * "reason"). Note: This operation is in the Unsafe class only
2758      * because {@code unpark} is, so it would be strange to place it
2759      * elsewhere.
2760      */
2761     @IntrinsicCandidate
2762     public native void park(boolean isAbsolute, long time);
2763 
2764     /**
2765      * Gets the load average in the system run queue assigned
2766      * to the available processors averaged over various periods of time.
2767      * This method retrieves the given {@code nelem} samples and
2768      * assigns to the elements of the given {@code loadavg} array.
2769      * The system imposes a maximum of 3 samples, representing
2770      * averages over the last 1,  5,  and  15 minutes, respectively.
2771      *
2772      * @param loadavg an array of double of size nelems
2773      * @param nelems the number of samples to be retrieved and
2774      *        must be 1 to 3.
2775      *
2776      * @return the number of samples actually retrieved; or -1
2777      *         if the load average is unobtainable.
2778      */
2779     public int getLoadAverage(double[] loadavg, int nelems) {
2780         if (nelems < 0 || nelems > 3 || nelems > loadavg.length) {
2781             throw new ArrayIndexOutOfBoundsException();
2782         }
2783 
2784         return getLoadAverage0(loadavg, nelems);
2785     }
2786 
2787     // The following contain CAS-based Java implementations used on
2788     // platforms not supporting native instructions
2789 
2790     /**
2791      * Atomically adds the given value to the current value of a field
2792      * or array element within the given object {@code o}
2793      * at the given {@code offset}.
2794      *
2795      * @param o object/array to update the field/element in
2796      * @param offset field/element offset
2797      * @param delta the value to add
2798      * @return the previous value
2799      * @since 1.8
2800      */
2801     @IntrinsicCandidate
2802     public final int getAndAddInt(Object o, long offset, int delta) {
2803         int v;
2804         do {
2805             v = getIntVolatile(o, offset);
2806         } while (!weakCompareAndSetInt(o, offset, v, v + delta));
2807         return v;
2808     }
2809 
2810     @ForceInline
2811     public final int getAndAddIntRelease(Object o, long offset, int delta) {
2812         int v;
2813         do {
2814             v = getInt(o, offset);
2815         } while (!weakCompareAndSetIntRelease(o, offset, v, v + delta));
2816         return v;
2817     }
2818 
2819     @ForceInline
2820     public final int getAndAddIntAcquire(Object o, long offset, int delta) {
2821         int v;
2822         do {
2823             v = getIntAcquire(o, offset);
2824         } while (!weakCompareAndSetIntAcquire(o, offset, v, v + delta));
2825         return v;
2826     }
2827 
2828     /**
2829      * Atomically adds the given value to the current value of a field
2830      * or array element within the given object {@code o}
2831      * at the given {@code offset}.
2832      *
2833      * @param o object/array to update the field/element in
2834      * @param offset field/element offset
2835      * @param delta the value to add
2836      * @return the previous value
2837      * @since 1.8
2838      */
2839     @IntrinsicCandidate
2840     public final long getAndAddLong(Object o, long offset, long delta) {
2841         long v;
2842         do {
2843             v = getLongVolatile(o, offset);
2844         } while (!weakCompareAndSetLong(o, offset, v, v + delta));
2845         return v;
2846     }
2847 
2848     @ForceInline
2849     public final long getAndAddLongRelease(Object o, long offset, long delta) {
2850         long v;
2851         do {
2852             v = getLong(o, offset);
2853         } while (!weakCompareAndSetLongRelease(o, offset, v, v + delta));
2854         return v;
2855     }
2856 
2857     @ForceInline
2858     public final long getAndAddLongAcquire(Object o, long offset, long delta) {
2859         long v;
2860         do {
2861             v = getLongAcquire(o, offset);
2862         } while (!weakCompareAndSetLongAcquire(o, offset, v, v + delta));
2863         return v;
2864     }
2865 
2866     @IntrinsicCandidate
2867     public final byte getAndAddByte(Object o, long offset, byte delta) {
2868         byte v;
2869         do {
2870             v = getByteVolatile(o, offset);
2871         } while (!weakCompareAndSetByte(o, offset, v, (byte) (v + delta)));
2872         return v;
2873     }
2874 
2875     @ForceInline
2876     public final byte getAndAddByteRelease(Object o, long offset, byte delta) {
2877         byte v;
2878         do {
2879             v = getByte(o, offset);
2880         } while (!weakCompareAndSetByteRelease(o, offset, v, (byte) (v + delta)));
2881         return v;
2882     }
2883 
2884     @ForceInline
2885     public final byte getAndAddByteAcquire(Object o, long offset, byte delta) {
2886         byte v;
2887         do {
2888             v = getByteAcquire(o, offset);
2889         } while (!weakCompareAndSetByteAcquire(o, offset, v, (byte) (v + delta)));
2890         return v;
2891     }
2892 
2893     @IntrinsicCandidate
2894     public final short getAndAddShort(Object o, long offset, short delta) {
2895         short v;
2896         do {
2897             v = getShortVolatile(o, offset);
2898         } while (!weakCompareAndSetShort(o, offset, v, (short) (v + delta)));
2899         return v;
2900     }
2901 
2902     @ForceInline
2903     public final short getAndAddShortRelease(Object o, long offset, short delta) {
2904         short v;
2905         do {
2906             v = getShort(o, offset);
2907         } while (!weakCompareAndSetShortRelease(o, offset, v, (short) (v + delta)));
2908         return v;
2909     }
2910 
2911     @ForceInline
2912     public final short getAndAddShortAcquire(Object o, long offset, short delta) {
2913         short v;
2914         do {
2915             v = getShortAcquire(o, offset);
2916         } while (!weakCompareAndSetShortAcquire(o, offset, v, (short) (v + delta)));
2917         return v;
2918     }
2919 
2920     @ForceInline
2921     public final char getAndAddChar(Object o, long offset, char delta) {
2922         return (char) getAndAddShort(o, offset, (short) delta);
2923     }
2924 
2925     @ForceInline
2926     public final char getAndAddCharRelease(Object o, long offset, char delta) {
2927         return (char) getAndAddShortRelease(o, offset, (short) delta);
2928     }
2929 
2930     @ForceInline
2931     public final char getAndAddCharAcquire(Object o, long offset, char delta) {
2932         return (char) getAndAddShortAcquire(o, offset, (short) delta);
2933     }
2934 
2935     @ForceInline
2936     public final float getAndAddFloat(Object o, long offset, float delta) {
2937         int expectedBits;
2938         float v;
2939         do {
2940             // Load and CAS with the raw bits to avoid issues with NaNs and
2941             // possible bit conversion from signaling NaNs to quiet NaNs that
2942             // may result in the loop not terminating.
2943             expectedBits = getIntVolatile(o, offset);
2944             v = Float.intBitsToFloat(expectedBits);
2945         } while (!weakCompareAndSetInt(o, offset,
2946                                                 expectedBits, Float.floatToRawIntBits(v + delta)));
2947         return v;
2948     }
2949 
2950     @ForceInline
2951     public final float getAndAddFloatRelease(Object o, long offset, float delta) {
2952         int expectedBits;
2953         float v;
2954         do {
2955             // Load and CAS with the raw bits to avoid issues with NaNs and
2956             // possible bit conversion from signaling NaNs to quiet NaNs that
2957             // may result in the loop not terminating.
2958             expectedBits = getInt(o, offset);
2959             v = Float.intBitsToFloat(expectedBits);
2960         } while (!weakCompareAndSetIntRelease(o, offset,
2961                                                expectedBits, Float.floatToRawIntBits(v + delta)));
2962         return v;
2963     }
2964 
2965     @ForceInline
2966     public final float getAndAddFloatAcquire(Object o, long offset, float delta) {
2967         int expectedBits;
2968         float v;
2969         do {
2970             // Load and CAS with the raw bits to avoid issues with NaNs and
2971             // possible bit conversion from signaling NaNs to quiet NaNs that
2972             // may result in the loop not terminating.
2973             expectedBits = getIntAcquire(o, offset);
2974             v = Float.intBitsToFloat(expectedBits);
2975         } while (!weakCompareAndSetIntAcquire(o, offset,
2976                                                expectedBits, Float.floatToRawIntBits(v + delta)));
2977         return v;
2978     }
2979 
2980     @ForceInline
2981     public final double getAndAddDouble(Object o, long offset, double delta) {
2982         long expectedBits;
2983         double v;
2984         do {
2985             // Load and CAS with the raw bits to avoid issues with NaNs and
2986             // possible bit conversion from signaling NaNs to quiet NaNs that
2987             // may result in the loop not terminating.
2988             expectedBits = getLongVolatile(o, offset);
2989             v = Double.longBitsToDouble(expectedBits);
2990         } while (!weakCompareAndSetLong(o, offset,
2991                                                  expectedBits, Double.doubleToRawLongBits(v + delta)));
2992         return v;
2993     }
2994 
2995     @ForceInline
2996     public final double getAndAddDoubleRelease(Object o, long offset, double delta) {
2997         long expectedBits;
2998         double v;
2999         do {
3000             // Load and CAS with the raw bits to avoid issues with NaNs and
3001             // possible bit conversion from signaling NaNs to quiet NaNs that
3002             // may result in the loop not terminating.
3003             expectedBits = getLong(o, offset);
3004             v = Double.longBitsToDouble(expectedBits);
3005         } while (!weakCompareAndSetLongRelease(o, offset,
3006                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
3007         return v;
3008     }
3009 
3010     @ForceInline
3011     public final double getAndAddDoubleAcquire(Object o, long offset, double delta) {
3012         long expectedBits;
3013         double v;
3014         do {
3015             // Load and CAS with the raw bits to avoid issues with NaNs and
3016             // possible bit conversion from signaling NaNs to quiet NaNs that
3017             // may result in the loop not terminating.
3018             expectedBits = getLongAcquire(o, offset);
3019             v = Double.longBitsToDouble(expectedBits);
3020         } while (!weakCompareAndSetLongAcquire(o, offset,
3021                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
3022         return v;
3023     }
3024 
3025     /**
3026      * Atomically exchanges the given value with the current value of
3027      * a field or array element within the given object {@code o}
3028      * at the given {@code offset}.
3029      *
3030      * @param o object/array to update the field/element in
3031      * @param offset field/element offset
3032      * @param newValue new value
3033      * @return the previous value
3034      * @since 1.8
3035      */
3036     @IntrinsicCandidate
3037     public final int getAndSetInt(Object o, long offset, int newValue) {
3038         int v;
3039         do {
3040             v = getIntVolatile(o, offset);
3041         } while (!weakCompareAndSetInt(o, offset, v, newValue));
3042         return v;
3043     }
3044 
3045     @ForceInline
3046     public final int getAndSetIntRelease(Object o, long offset, int newValue) {
3047         int v;
3048         do {
3049             v = getInt(o, offset);
3050         } while (!weakCompareAndSetIntRelease(o, offset, v, newValue));
3051         return v;
3052     }
3053 
3054     @ForceInline
3055     public final int getAndSetIntAcquire(Object o, long offset, int newValue) {
3056         int v;
3057         do {
3058             v = getIntAcquire(o, offset);
3059         } while (!weakCompareAndSetIntAcquire(o, offset, v, newValue));
3060         return v;
3061     }
3062 
3063     /**
3064      * Atomically exchanges the given value with the current value of
3065      * a field or array element within the given object {@code o}
3066      * at the given {@code offset}.
3067      *
3068      * @param o object/array to update the field/element in
3069      * @param offset field/element offset
3070      * @param newValue new value
3071      * @return the previous value
3072      * @since 1.8
3073      */
3074     @IntrinsicCandidate
3075     public final long getAndSetLong(Object o, long offset, long newValue) {
3076         long v;
3077         do {
3078             v = getLongVolatile(o, offset);
3079         } while (!weakCompareAndSetLong(o, offset, v, newValue));
3080         return v;
3081     }
3082 
3083     @ForceInline
3084     public final long getAndSetLongRelease(Object o, long offset, long newValue) {
3085         long v;
3086         do {
3087             v = getLong(o, offset);
3088         } while (!weakCompareAndSetLongRelease(o, offset, v, newValue));
3089         return v;
3090     }
3091 
3092     @ForceInline
3093     public final long getAndSetLongAcquire(Object o, long offset, long newValue) {
3094         long v;
3095         do {
3096             v = getLongAcquire(o, offset);
3097         } while (!weakCompareAndSetLongAcquire(o, offset, v, newValue));
3098         return v;
3099     }
3100 
3101     /**
3102      * Atomically exchanges the given reference value with the current
3103      * reference value of a field or array element within the given
3104      * object {@code o} at the given {@code offset}.
3105      *
3106      * @param o object/array to update the field/element in
3107      * @param offset field/element offset
3108      * @param newValue new value
3109      * @return the previous value
3110      * @since 1.8
3111      */
3112     @IntrinsicCandidate
3113     public final Object getAndSetReference(Object o, long offset, Object newValue) {
3114         Object v;
3115         do {
3116             v = getReferenceVolatile(o, offset);
3117         } while (!weakCompareAndSetReference(o, offset, v, newValue));
3118         return v;
3119     }
3120 
3121     @SuppressWarnings("unchecked")
3122     public final <V> Object getAndSetValue(Object o, long offset, Class<?> valueType, V newValue) {
3123         synchronized (valueLock) {
3124             Object oldValue = getValue(o, offset, valueType);
3125             putValue(o, offset, valueType, newValue);
3126             return oldValue;
3127         }
3128     }
3129 
3130     @ForceInline
3131     public final Object getAndSetReferenceRelease(Object o, long offset, Object newValue) {
3132         Object v;
3133         do {
3134             v = getReference(o, offset);
3135         } while (!weakCompareAndSetReferenceRelease(o, offset, v, newValue));
3136         return v;
3137     }
3138 
3139     @ForceInline
3140     public final <V> Object getAndSetValueRelease(Object o, long offset, Class<?> valueType, V newValue) {
3141         return getAndSetValue(o, offset, valueType, newValue);
3142     }
3143 
3144     @ForceInline
3145     public final Object getAndSetReferenceAcquire(Object o, long offset, Object newValue) {
3146         Object v;
3147         do {
3148             v = getReferenceAcquire(o, offset);
3149         } while (!weakCompareAndSetReferenceAcquire(o, offset, v, newValue));
3150         return v;
3151     }
3152 
3153     @ForceInline
3154     public final <V> Object getAndSetValueAcquire(Object o, long offset, Class<?> valueType, V newValue) {
3155         return getAndSetValue(o, offset, valueType, newValue);
3156     }
3157 
3158     @IntrinsicCandidate
3159     public final byte getAndSetByte(Object o, long offset, byte newValue) {
3160         byte v;
3161         do {
3162             v = getByteVolatile(o, offset);
3163         } while (!weakCompareAndSetByte(o, offset, v, newValue));
3164         return v;
3165     }
3166 
3167     @ForceInline
3168     public final byte getAndSetByteRelease(Object o, long offset, byte newValue) {
3169         byte v;
3170         do {
3171             v = getByte(o, offset);
3172         } while (!weakCompareAndSetByteRelease(o, offset, v, newValue));
3173         return v;
3174     }
3175 
3176     @ForceInline
3177     public final byte getAndSetByteAcquire(Object o, long offset, byte newValue) {
3178         byte v;
3179         do {
3180             v = getByteAcquire(o, offset);
3181         } while (!weakCompareAndSetByteAcquire(o, offset, v, newValue));
3182         return v;
3183     }
3184 
3185     @ForceInline
3186     public final boolean getAndSetBoolean(Object o, long offset, boolean newValue) {
3187         return byte2bool(getAndSetByte(o, offset, bool2byte(newValue)));
3188     }
3189 
3190     @ForceInline
3191     public final boolean getAndSetBooleanRelease(Object o, long offset, boolean newValue) {
3192         return byte2bool(getAndSetByteRelease(o, offset, bool2byte(newValue)));
3193     }
3194 
3195     @ForceInline
3196     public final boolean getAndSetBooleanAcquire(Object o, long offset, boolean newValue) {
3197         return byte2bool(getAndSetByteAcquire(o, offset, bool2byte(newValue)));
3198     }
3199 
3200     @IntrinsicCandidate
3201     public final short getAndSetShort(Object o, long offset, short newValue) {
3202         short v;
3203         do {
3204             v = getShortVolatile(o, offset);
3205         } while (!weakCompareAndSetShort(o, offset, v, newValue));
3206         return v;
3207     }
3208 
3209     @ForceInline
3210     public final short getAndSetShortRelease(Object o, long offset, short newValue) {
3211         short v;
3212         do {
3213             v = getShort(o, offset);
3214         } while (!weakCompareAndSetShortRelease(o, offset, v, newValue));
3215         return v;
3216     }
3217 
3218     @ForceInline
3219     public final short getAndSetShortAcquire(Object o, long offset, short newValue) {
3220         short v;
3221         do {
3222             v = getShortAcquire(o, offset);
3223         } while (!weakCompareAndSetShortAcquire(o, offset, v, newValue));
3224         return v;
3225     }
3226 
3227     @ForceInline
3228     public final char getAndSetChar(Object o, long offset, char newValue) {
3229         return s2c(getAndSetShort(o, offset, c2s(newValue)));
3230     }
3231 
3232     @ForceInline
3233     public final char getAndSetCharRelease(Object o, long offset, char newValue) {
3234         return s2c(getAndSetShortRelease(o, offset, c2s(newValue)));
3235     }
3236 
3237     @ForceInline
3238     public final char getAndSetCharAcquire(Object o, long offset, char newValue) {
3239         return s2c(getAndSetShortAcquire(o, offset, c2s(newValue)));
3240     }
3241 
3242     @ForceInline
3243     public final float getAndSetFloat(Object o, long offset, float newValue) {
3244         int v = getAndSetInt(o, offset, Float.floatToRawIntBits(newValue));
3245         return Float.intBitsToFloat(v);
3246     }
3247 
3248     @ForceInline
3249     public final float getAndSetFloatRelease(Object o, long offset, float newValue) {
3250         int v = getAndSetIntRelease(o, offset, Float.floatToRawIntBits(newValue));
3251         return Float.intBitsToFloat(v);
3252     }
3253 
3254     @ForceInline
3255     public final float getAndSetFloatAcquire(Object o, long offset, float newValue) {
3256         int v = getAndSetIntAcquire(o, offset, Float.floatToRawIntBits(newValue));
3257         return Float.intBitsToFloat(v);
3258     }
3259 
3260     @ForceInline
3261     public final double getAndSetDouble(Object o, long offset, double newValue) {
3262         long v = getAndSetLong(o, offset, Double.doubleToRawLongBits(newValue));
3263         return Double.longBitsToDouble(v);
3264     }
3265 
3266     @ForceInline
3267     public final double getAndSetDoubleRelease(Object o, long offset, double newValue) {
3268         long v = getAndSetLongRelease(o, offset, Double.doubleToRawLongBits(newValue));
3269         return Double.longBitsToDouble(v);
3270     }
3271 
3272     @ForceInline
3273     public final double getAndSetDoubleAcquire(Object o, long offset, double newValue) {
3274         long v = getAndSetLongAcquire(o, offset, Double.doubleToRawLongBits(newValue));
3275         return Double.longBitsToDouble(v);
3276     }
3277 
3278 
3279     // The following contain CAS-based Java implementations used on
3280     // platforms not supporting native instructions
3281 
3282     @ForceInline
3283     public final boolean getAndBitwiseOrBoolean(Object o, long offset, boolean mask) {
3284         return byte2bool(getAndBitwiseOrByte(o, offset, bool2byte(mask)));
3285     }
3286 
3287     @ForceInline
3288     public final boolean getAndBitwiseOrBooleanRelease(Object o, long offset, boolean mask) {
3289         return byte2bool(getAndBitwiseOrByteRelease(o, offset, bool2byte(mask)));
3290     }
3291 
3292     @ForceInline
3293     public final boolean getAndBitwiseOrBooleanAcquire(Object o, long offset, boolean mask) {
3294         return byte2bool(getAndBitwiseOrByteAcquire(o, offset, bool2byte(mask)));
3295     }
3296 
3297     @ForceInline
3298     public final boolean getAndBitwiseAndBoolean(Object o, long offset, boolean mask) {
3299         return byte2bool(getAndBitwiseAndByte(o, offset, bool2byte(mask)));
3300     }
3301 
3302     @ForceInline
3303     public final boolean getAndBitwiseAndBooleanRelease(Object o, long offset, boolean mask) {
3304         return byte2bool(getAndBitwiseAndByteRelease(o, offset, bool2byte(mask)));
3305     }
3306 
3307     @ForceInline
3308     public final boolean getAndBitwiseAndBooleanAcquire(Object o, long offset, boolean mask) {
3309         return byte2bool(getAndBitwiseAndByteAcquire(o, offset, bool2byte(mask)));
3310     }
3311 
3312     @ForceInline
3313     public final boolean getAndBitwiseXorBoolean(Object o, long offset, boolean mask) {
3314         return byte2bool(getAndBitwiseXorByte(o, offset, bool2byte(mask)));
3315     }
3316 
3317     @ForceInline
3318     public final boolean getAndBitwiseXorBooleanRelease(Object o, long offset, boolean mask) {
3319         return byte2bool(getAndBitwiseXorByteRelease(o, offset, bool2byte(mask)));
3320     }
3321 
3322     @ForceInline
3323     public final boolean getAndBitwiseXorBooleanAcquire(Object o, long offset, boolean mask) {
3324         return byte2bool(getAndBitwiseXorByteAcquire(o, offset, bool2byte(mask)));
3325     }
3326 
3327 
3328     @ForceInline
3329     public final byte getAndBitwiseOrByte(Object o, long offset, byte mask) {
3330         byte current;
3331         do {
3332             current = getByteVolatile(o, offset);
3333         } while (!weakCompareAndSetByte(o, offset,
3334                                                   current, (byte) (current | mask)));
3335         return current;
3336     }
3337 
3338     @ForceInline
3339     public final byte getAndBitwiseOrByteRelease(Object o, long offset, byte mask) {
3340         byte current;
3341         do {
3342             current = getByte(o, offset);
3343         } while (!weakCompareAndSetByteRelease(o, offset,
3344                                                  current, (byte) (current | mask)));
3345         return current;
3346     }
3347 
3348     @ForceInline
3349     public final byte getAndBitwiseOrByteAcquire(Object o, long offset, byte mask) {
3350         byte current;
3351         do {
3352             // Plain read, the value is a hint, the acquire CAS does the work
3353             current = getByte(o, offset);
3354         } while (!weakCompareAndSetByteAcquire(o, offset,
3355                                                  current, (byte) (current | mask)));
3356         return current;
3357     }
3358 
3359     @ForceInline
3360     public final byte getAndBitwiseAndByte(Object o, long offset, byte mask) {
3361         byte current;
3362         do {
3363             current = getByteVolatile(o, offset);
3364         } while (!weakCompareAndSetByte(o, offset,
3365                                                   current, (byte) (current & mask)));
3366         return current;
3367     }
3368 
3369     @ForceInline
3370     public final byte getAndBitwiseAndByteRelease(Object o, long offset, byte mask) {
3371         byte current;
3372         do {
3373             current = getByte(o, offset);
3374         } while (!weakCompareAndSetByteRelease(o, offset,
3375                                                  current, (byte) (current & mask)));
3376         return current;
3377     }
3378 
3379     @ForceInline
3380     public final byte getAndBitwiseAndByteAcquire(Object o, long offset, byte mask) {
3381         byte current;
3382         do {
3383             // Plain read, the value is a hint, the acquire CAS does the work
3384             current = getByte(o, offset);
3385         } while (!weakCompareAndSetByteAcquire(o, offset,
3386                                                  current, (byte) (current & mask)));
3387         return current;
3388     }
3389 
3390     @ForceInline
3391     public final byte getAndBitwiseXorByte(Object o, long offset, byte mask) {
3392         byte current;
3393         do {
3394             current = getByteVolatile(o, offset);
3395         } while (!weakCompareAndSetByte(o, offset,
3396                                                   current, (byte) (current ^ mask)));
3397         return current;
3398     }
3399 
3400     @ForceInline
3401     public final byte getAndBitwiseXorByteRelease(Object o, long offset, byte mask) {
3402         byte current;
3403         do {
3404             current = getByte(o, offset);
3405         } while (!weakCompareAndSetByteRelease(o, offset,
3406                                                  current, (byte) (current ^ mask)));
3407         return current;
3408     }
3409 
3410     @ForceInline
3411     public final byte getAndBitwiseXorByteAcquire(Object o, long offset, byte mask) {
3412         byte current;
3413         do {
3414             // Plain read, the value is a hint, the acquire CAS does the work
3415             current = getByte(o, offset);
3416         } while (!weakCompareAndSetByteAcquire(o, offset,
3417                                                  current, (byte) (current ^ mask)));
3418         return current;
3419     }
3420 
3421 
3422     @ForceInline
3423     public final char getAndBitwiseOrChar(Object o, long offset, char mask) {
3424         return s2c(getAndBitwiseOrShort(o, offset, c2s(mask)));
3425     }
3426 
3427     @ForceInline
3428     public final char getAndBitwiseOrCharRelease(Object o, long offset, char mask) {
3429         return s2c(getAndBitwiseOrShortRelease(o, offset, c2s(mask)));
3430     }
3431 
3432     @ForceInline
3433     public final char getAndBitwiseOrCharAcquire(Object o, long offset, char mask) {
3434         return s2c(getAndBitwiseOrShortAcquire(o, offset, c2s(mask)));
3435     }
3436 
3437     @ForceInline
3438     public final char getAndBitwiseAndChar(Object o, long offset, char mask) {
3439         return s2c(getAndBitwiseAndShort(o, offset, c2s(mask)));
3440     }
3441 
3442     @ForceInline
3443     public final char getAndBitwiseAndCharRelease(Object o, long offset, char mask) {
3444         return s2c(getAndBitwiseAndShortRelease(o, offset, c2s(mask)));
3445     }
3446 
3447     @ForceInline
3448     public final char getAndBitwiseAndCharAcquire(Object o, long offset, char mask) {
3449         return s2c(getAndBitwiseAndShortAcquire(o, offset, c2s(mask)));
3450     }
3451 
3452     @ForceInline
3453     public final char getAndBitwiseXorChar(Object o, long offset, char mask) {
3454         return s2c(getAndBitwiseXorShort(o, offset, c2s(mask)));
3455     }
3456 
3457     @ForceInline
3458     public final char getAndBitwiseXorCharRelease(Object o, long offset, char mask) {
3459         return s2c(getAndBitwiseXorShortRelease(o, offset, c2s(mask)));
3460     }
3461 
3462     @ForceInline
3463     public final char getAndBitwiseXorCharAcquire(Object o, long offset, char mask) {
3464         return s2c(getAndBitwiseXorShortAcquire(o, offset, c2s(mask)));
3465     }
3466 
3467 
3468     @ForceInline
3469     public final short getAndBitwiseOrShort(Object o, long offset, short mask) {
3470         short current;
3471         do {
3472             current = getShortVolatile(o, offset);
3473         } while (!weakCompareAndSetShort(o, offset,
3474                                                 current, (short) (current | mask)));
3475         return current;
3476     }
3477 
3478     @ForceInline
3479     public final short getAndBitwiseOrShortRelease(Object o, long offset, short mask) {
3480         short current;
3481         do {
3482             current = getShort(o, offset);
3483         } while (!weakCompareAndSetShortRelease(o, offset,
3484                                                current, (short) (current | mask)));
3485         return current;
3486     }
3487 
3488     @ForceInline
3489     public final short getAndBitwiseOrShortAcquire(Object o, long offset, short mask) {
3490         short current;
3491         do {
3492             // Plain read, the value is a hint, the acquire CAS does the work
3493             current = getShort(o, offset);
3494         } while (!weakCompareAndSetShortAcquire(o, offset,
3495                                                current, (short) (current | mask)));
3496         return current;
3497     }
3498 
3499     @ForceInline
3500     public final short getAndBitwiseAndShort(Object o, long offset, short mask) {
3501         short current;
3502         do {
3503             current = getShortVolatile(o, offset);
3504         } while (!weakCompareAndSetShort(o, offset,
3505                                                 current, (short) (current & mask)));
3506         return current;
3507     }
3508 
3509     @ForceInline
3510     public final short getAndBitwiseAndShortRelease(Object o, long offset, short mask) {
3511         short current;
3512         do {
3513             current = getShort(o, offset);
3514         } while (!weakCompareAndSetShortRelease(o, offset,
3515                                                current, (short) (current & mask)));
3516         return current;
3517     }
3518 
3519     @ForceInline
3520     public final short getAndBitwiseAndShortAcquire(Object o, long offset, short mask) {
3521         short current;
3522         do {
3523             // Plain read, the value is a hint, the acquire CAS does the work
3524             current = getShort(o, offset);
3525         } while (!weakCompareAndSetShortAcquire(o, offset,
3526                                                current, (short) (current & mask)));
3527         return current;
3528     }
3529 
3530     @ForceInline
3531     public final short getAndBitwiseXorShort(Object o, long offset, short mask) {
3532         short current;
3533         do {
3534             current = getShortVolatile(o, offset);
3535         } while (!weakCompareAndSetShort(o, offset,
3536                                                 current, (short) (current ^ mask)));
3537         return current;
3538     }
3539 
3540     @ForceInline
3541     public final short getAndBitwiseXorShortRelease(Object o, long offset, short mask) {
3542         short current;
3543         do {
3544             current = getShort(o, offset);
3545         } while (!weakCompareAndSetShortRelease(o, offset,
3546                                                current, (short) (current ^ mask)));
3547         return current;
3548     }
3549 
3550     @ForceInline
3551     public final short getAndBitwiseXorShortAcquire(Object o, long offset, short mask) {
3552         short current;
3553         do {
3554             // Plain read, the value is a hint, the acquire CAS does the work
3555             current = getShort(o, offset);
3556         } while (!weakCompareAndSetShortAcquire(o, offset,
3557                                                current, (short) (current ^ mask)));
3558         return current;
3559     }
3560 
3561 
3562     @ForceInline
3563     public final int getAndBitwiseOrInt(Object o, long offset, int mask) {
3564         int current;
3565         do {
3566             current = getIntVolatile(o, offset);
3567         } while (!weakCompareAndSetInt(o, offset,
3568                                                 current, current | mask));
3569         return current;
3570     }
3571 
3572     @ForceInline
3573     public final int getAndBitwiseOrIntRelease(Object o, long offset, int mask) {
3574         int current;
3575         do {
3576             current = getInt(o, offset);
3577         } while (!weakCompareAndSetIntRelease(o, offset,
3578                                                current, current | mask));
3579         return current;
3580     }
3581 
3582     @ForceInline
3583     public final int getAndBitwiseOrIntAcquire(Object o, long offset, int mask) {
3584         int current;
3585         do {
3586             // Plain read, the value is a hint, the acquire CAS does the work
3587             current = getInt(o, offset);
3588         } while (!weakCompareAndSetIntAcquire(o, offset,
3589                                                current, current | mask));
3590         return current;
3591     }
3592 
3593     /**
3594      * Atomically replaces the current value of a field or array element within
3595      * the given object with the result of bitwise AND between the current value
3596      * and mask.
3597      *
3598      * @param o object/array to update the field/element in
3599      * @param offset field/element offset
3600      * @param mask the mask value
3601      * @return the previous value
3602      * @since 9
3603      */
3604     @ForceInline
3605     public final int getAndBitwiseAndInt(Object o, long offset, int mask) {
3606         int current;
3607         do {
3608             current = getIntVolatile(o, offset);
3609         } while (!weakCompareAndSetInt(o, offset,
3610                                                 current, current & mask));
3611         return current;
3612     }
3613 
3614     @ForceInline
3615     public final int getAndBitwiseAndIntRelease(Object o, long offset, int mask) {
3616         int current;
3617         do {
3618             current = getInt(o, offset);
3619         } while (!weakCompareAndSetIntRelease(o, offset,
3620                                                current, current & mask));
3621         return current;
3622     }
3623 
3624     @ForceInline
3625     public final int getAndBitwiseAndIntAcquire(Object o, long offset, int mask) {
3626         int current;
3627         do {
3628             // Plain read, the value is a hint, the acquire CAS does the work
3629             current = getInt(o, offset);
3630         } while (!weakCompareAndSetIntAcquire(o, offset,
3631                                                current, current & mask));
3632         return current;
3633     }
3634 
3635     @ForceInline
3636     public final int getAndBitwiseXorInt(Object o, long offset, int mask) {
3637         int current;
3638         do {
3639             current = getIntVolatile(o, offset);
3640         } while (!weakCompareAndSetInt(o, offset,
3641                                                 current, current ^ mask));
3642         return current;
3643     }
3644 
3645     @ForceInline
3646     public final int getAndBitwiseXorIntRelease(Object o, long offset, int mask) {
3647         int current;
3648         do {
3649             current = getInt(o, offset);
3650         } while (!weakCompareAndSetIntRelease(o, offset,
3651                                                current, current ^ mask));
3652         return current;
3653     }
3654 
3655     @ForceInline
3656     public final int getAndBitwiseXorIntAcquire(Object o, long offset, int mask) {
3657         int current;
3658         do {
3659             // Plain read, the value is a hint, the acquire CAS does the work
3660             current = getInt(o, offset);
3661         } while (!weakCompareAndSetIntAcquire(o, offset,
3662                                                current, current ^ mask));
3663         return current;
3664     }
3665 
3666 
3667     @ForceInline
3668     public final long getAndBitwiseOrLong(Object o, long offset, long mask) {
3669         long current;
3670         do {
3671             current = getLongVolatile(o, offset);
3672         } while (!weakCompareAndSetLong(o, offset,
3673                                                 current, current | mask));
3674         return current;
3675     }
3676 
3677     @ForceInline
3678     public final long getAndBitwiseOrLongRelease(Object o, long offset, long mask) {
3679         long current;
3680         do {
3681             current = getLong(o, offset);
3682         } while (!weakCompareAndSetLongRelease(o, offset,
3683                                                current, current | mask));
3684         return current;
3685     }
3686 
3687     @ForceInline
3688     public final long getAndBitwiseOrLongAcquire(Object o, long offset, long mask) {
3689         long current;
3690         do {
3691             // Plain read, the value is a hint, the acquire CAS does the work
3692             current = getLong(o, offset);
3693         } while (!weakCompareAndSetLongAcquire(o, offset,
3694                                                current, current | mask));
3695         return current;
3696     }
3697 
3698     @ForceInline
3699     public final long getAndBitwiseAndLong(Object o, long offset, long mask) {
3700         long current;
3701         do {
3702             current = getLongVolatile(o, offset);
3703         } while (!weakCompareAndSetLong(o, offset,
3704                                                 current, current & mask));
3705         return current;
3706     }
3707 
3708     @ForceInline
3709     public final long getAndBitwiseAndLongRelease(Object o, long offset, long mask) {
3710         long current;
3711         do {
3712             current = getLong(o, offset);
3713         } while (!weakCompareAndSetLongRelease(o, offset,
3714                                                current, current & mask));
3715         return current;
3716     }
3717 
3718     @ForceInline
3719     public final long getAndBitwiseAndLongAcquire(Object o, long offset, long mask) {
3720         long current;
3721         do {
3722             // Plain read, the value is a hint, the acquire CAS does the work
3723             current = getLong(o, offset);
3724         } while (!weakCompareAndSetLongAcquire(o, offset,
3725                                                current, current & mask));
3726         return current;
3727     }
3728 
3729     @ForceInline
3730     public final long getAndBitwiseXorLong(Object o, long offset, long mask) {
3731         long current;
3732         do {
3733             current = getLongVolatile(o, offset);
3734         } while (!weakCompareAndSetLong(o, offset,
3735                                                 current, current ^ mask));
3736         return current;
3737     }
3738 
3739     @ForceInline
3740     public final long getAndBitwiseXorLongRelease(Object o, long offset, long mask) {
3741         long current;
3742         do {
3743             current = getLong(o, offset);
3744         } while (!weakCompareAndSetLongRelease(o, offset,
3745                                                current, current ^ mask));
3746         return current;
3747     }
3748 
3749     @ForceInline
3750     public final long getAndBitwiseXorLongAcquire(Object o, long offset, long mask) {
3751         long current;
3752         do {
3753             // Plain read, the value is a hint, the acquire CAS does the work
3754             current = getLong(o, offset);
3755         } while (!weakCompareAndSetLongAcquire(o, offset,
3756                                                current, current ^ mask));
3757         return current;
3758     }
3759 
3760 
3761 
3762     /**
3763      * Ensures that loads before the fence will not be reordered with loads and
3764      * stores after the fence; a "LoadLoad plus LoadStore barrier".
3765      *
3766      * Corresponds to C11 atomic_thread_fence(memory_order_acquire)
3767      * (an "acquire fence").
3768      *
3769      * Provides a LoadLoad barrier followed by a LoadStore barrier.
3770      *
3771      * @since 1.8
3772      */
3773     @IntrinsicCandidate
3774     public final void loadFence() {
3775         // If loadFence intrinsic is not available, fall back to full fence.
3776         fullFence();
3777     }
3778 
3779     /**
3780      * Ensures that loads and stores before the fence will not be reordered with
3781      * stores after the fence; a "StoreStore plus LoadStore barrier".
3782      *
3783      * Corresponds to C11 atomic_thread_fence(memory_order_release)
3784      * (a "release fence").
3785      *
3786      * Provides a StoreStore barrier followed by a LoadStore barrier.
3787      *
3788      * @since 1.8
3789      */
3790     @IntrinsicCandidate
3791     public final void storeFence() {
3792         // If storeFence intrinsic is not available, fall back to full fence.
3793         fullFence();
3794     }
3795 
3796     /**
3797      * Ensures that loads and stores before the fence will not be reordered
3798      * with loads and stores after the fence.  Implies the effects of both
3799      * loadFence() and storeFence(), and in addition, the effect of a StoreLoad
3800      * barrier.
3801      *
3802      * Corresponds to C11 atomic_thread_fence(memory_order_seq_cst).
3803      * @since 1.8
3804      */
3805     @IntrinsicCandidate
3806     public native void fullFence();
3807 
3808     /**
3809      * Ensures that loads before the fence will not be reordered with
3810      * loads after the fence.
3811      *
3812      * @implNote
3813      * This method is operationally equivalent to {@link #loadFence()}.
3814      *
3815      * @since 9
3816      */
3817     public final void loadLoadFence() {
3818         loadFence();
3819     }
3820 
3821     /**
3822      * Ensures that stores before the fence will not be reordered with
3823      * stores after the fence.
3824      *
3825      * @since 9
3826      */
3827     @IntrinsicCandidate
3828     public final void storeStoreFence() {
3829         // If storeStoreFence intrinsic is not available, fall back to storeFence.
3830         storeFence();
3831     }
3832 
3833     /**
3834      * Throws IllegalAccessError; for use by the VM for access control
3835      * error support.
3836      * @since 1.8
3837      */
3838     private static void throwIllegalAccessError() {
3839         throw new IllegalAccessError();
3840     }
3841 
3842     /**
3843      * Throws NoSuchMethodError; for use by the VM for redefinition support.
3844      * @since 13
3845      */
3846     private static void throwNoSuchMethodError() {
3847         throw new NoSuchMethodError();
3848     }
3849 
3850     /**
3851      * @return Returns true if the native byte ordering of this
3852      * platform is big-endian, false if it is little-endian.
3853      */
3854     public final boolean isBigEndian() { return BIG_ENDIAN; }
3855 
3856     /**
3857      * @return Returns true if this platform is capable of performing
3858      * accesses at addresses which are not aligned for the type of the
3859      * primitive type being accessed, false otherwise.
3860      */
3861     public final boolean unalignedAccess() { return UNALIGNED_ACCESS; }
3862 
3863     /**
3864      * Fetches a value at some byte offset into a given Java object.
3865      * More specifically, fetches a value within the given object
3866      * <code>o</code> at the given offset, or (if <code>o</code> is
3867      * null) from the memory address whose numerical value is the
3868      * given offset.  <p>
3869      *
3870      * The specification of this method is the same as {@link
3871      * #getLong(Object, long)} except that the offset does not need to
3872      * have been obtained from {@link #objectFieldOffset} on the
3873      * {@link java.lang.reflect.Field} of some Java field.  The value
3874      * in memory is raw data, and need not correspond to any Java
3875      * variable.  Unless <code>o</code> is null, the value accessed
3876      * must be entirely within the allocated object.  The endianness
3877      * of the value in memory is the endianness of the native platform.
3878      *
3879      * <p> The read will be atomic with respect to the largest power
3880      * of two that divides the GCD of the offset and the storage size.
3881      * For example, getLongUnaligned will make atomic reads of 2-, 4-,
3882      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
3883      * respectively.  There are no other guarantees of atomicity.
3884      * <p>
3885      * 8-byte atomicity is only guaranteed on platforms on which
3886      * support atomic accesses to longs.
3887      *
3888      * @param o Java heap object in which the value resides, if any, else
3889      *        null
3890      * @param offset The offset in bytes from the start of the object
3891      * @return the value fetched from the indicated object
3892      * @throws RuntimeException No defined exceptions are thrown, not even
3893      *         {@link NullPointerException}
3894      * @since 9
3895      */
3896     @IntrinsicCandidate
3897     public final long getLongUnaligned(Object o, long offset) {
3898         if ((offset & 7) == 0) {
3899             return getLong(o, offset);
3900         } else if ((offset & 3) == 0) {
3901             return makeLong(getInt(o, offset),
3902                             getInt(o, offset + 4));
3903         } else if ((offset & 1) == 0) {
3904             return makeLong(getShort(o, offset),
3905                             getShort(o, offset + 2),
3906                             getShort(o, offset + 4),
3907                             getShort(o, offset + 6));
3908         } else {
3909             return makeLong(getByte(o, offset),
3910                             getByte(o, offset + 1),
3911                             getByte(o, offset + 2),
3912                             getByte(o, offset + 3),
3913                             getByte(o, offset + 4),
3914                             getByte(o, offset + 5),
3915                             getByte(o, offset + 6),
3916                             getByte(o, offset + 7));
3917         }
3918     }
3919     /**
3920      * As {@link #getLongUnaligned(Object, long)} but with an
3921      * additional argument which specifies the endianness of the value
3922      * as stored in memory.
3923      *
3924      * @param o Java heap object in which the variable resides
3925      * @param offset The offset in bytes from the start of the object
3926      * @param bigEndian The endianness of the value
3927      * @return the value fetched from the indicated object
3928      * @since 9
3929      */
3930     public final long getLongUnaligned(Object o, long offset, boolean bigEndian) {
3931         return convEndian(bigEndian, getLongUnaligned(o, offset));
3932     }
3933 
3934     /** @see #getLongUnaligned(Object, long) */
3935     @IntrinsicCandidate
3936     public final int getIntUnaligned(Object o, long offset) {
3937         if ((offset & 3) == 0) {
3938             return getInt(o, offset);
3939         } else if ((offset & 1) == 0) {
3940             return makeInt(getShort(o, offset),
3941                            getShort(o, offset + 2));
3942         } else {
3943             return makeInt(getByte(o, offset),
3944                            getByte(o, offset + 1),
3945                            getByte(o, offset + 2),
3946                            getByte(o, offset + 3));
3947         }
3948     }
3949     /** @see #getLongUnaligned(Object, long, boolean) */
3950     public final int getIntUnaligned(Object o, long offset, boolean bigEndian) {
3951         return convEndian(bigEndian, getIntUnaligned(o, offset));
3952     }
3953 
3954     /** @see #getLongUnaligned(Object, long) */
3955     @IntrinsicCandidate
3956     public final short getShortUnaligned(Object o, long offset) {
3957         if ((offset & 1) == 0) {
3958             return getShort(o, offset);
3959         } else {
3960             return makeShort(getByte(o, offset),
3961                              getByte(o, offset + 1));
3962         }
3963     }
3964     /** @see #getLongUnaligned(Object, long, boolean) */
3965     public final short getShortUnaligned(Object o, long offset, boolean bigEndian) {
3966         return convEndian(bigEndian, getShortUnaligned(o, offset));
3967     }
3968 
3969     /** @see #getLongUnaligned(Object, long) */
3970     @IntrinsicCandidate
3971     public final char getCharUnaligned(Object o, long offset) {
3972         if ((offset & 1) == 0) {
3973             return getChar(o, offset);
3974         } else {
3975             return (char)makeShort(getByte(o, offset),
3976                                    getByte(o, offset + 1));
3977         }
3978     }
3979 
3980     /** @see #getLongUnaligned(Object, long, boolean) */
3981     public final char getCharUnaligned(Object o, long offset, boolean bigEndian) {
3982         return convEndian(bigEndian, getCharUnaligned(o, offset));
3983     }
3984 
3985     /**
3986      * Stores a value at some byte offset into a given Java object.
3987      * <p>
3988      * The specification of this method is the same as {@link
3989      * #getLong(Object, long)} except that the offset does not need to
3990      * have been obtained from {@link #objectFieldOffset} on the
3991      * {@link java.lang.reflect.Field} of some Java field.  The value
3992      * in memory is raw data, and need not correspond to any Java
3993      * variable.  The endianness of the value in memory is the
3994      * endianness of the native platform.
3995      * <p>
3996      * The write will be atomic with respect to the largest power of
3997      * two that divides the GCD of the offset and the storage size.
3998      * For example, putLongUnaligned will make atomic writes of 2-, 4-,
3999      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
4000      * respectively.  There are no other guarantees of atomicity.
4001      * <p>
4002      * 8-byte atomicity is only guaranteed on platforms on which
4003      * support atomic accesses to longs.
4004      *
4005      * @param o Java heap object in which the value resides, if any, else
4006      *        null
4007      * @param offset The offset in bytes from the start of the object
4008      * @param x the value to store
4009      * @throws RuntimeException No defined exceptions are thrown, not even
4010      *         {@link NullPointerException}
4011      * @since 9
4012      */
4013     @IntrinsicCandidate
4014     public final void putLongUnaligned(Object o, long offset, long x) {
4015         if ((offset & 7) == 0) {
4016             putLong(o, offset, x);
4017         } else if ((offset & 3) == 0) {
4018             putLongParts(o, offset,
4019                          (int)(x >> 0),
4020                          (int)(x >>> 32));
4021         } else if ((offset & 1) == 0) {
4022             putLongParts(o, offset,
4023                          (short)(x >>> 0),
4024                          (short)(x >>> 16),
4025                          (short)(x >>> 32),
4026                          (short)(x >>> 48));
4027         } else {
4028             putLongParts(o, offset,
4029                          (byte)(x >>> 0),
4030                          (byte)(x >>> 8),
4031                          (byte)(x >>> 16),
4032                          (byte)(x >>> 24),
4033                          (byte)(x >>> 32),
4034                          (byte)(x >>> 40),
4035                          (byte)(x >>> 48),
4036                          (byte)(x >>> 56));
4037         }
4038     }
4039 
4040     /**
4041      * As {@link #putLongUnaligned(Object, long, long)} but with an additional
4042      * argument which specifies the endianness of the value as stored in memory.
4043      * @param o Java heap object in which the value resides
4044      * @param offset The offset in bytes from the start of the object
4045      * @param x the value to store
4046      * @param bigEndian The endianness of the value
4047      * @throws RuntimeException No defined exceptions are thrown, not even
4048      *         {@link NullPointerException}
4049      * @since 9
4050      */
4051     public final void putLongUnaligned(Object o, long offset, long x, boolean bigEndian) {
4052         putLongUnaligned(o, offset, convEndian(bigEndian, x));
4053     }
4054 
4055     /** @see #putLongUnaligned(Object, long, long) */
4056     @IntrinsicCandidate
4057     public final void putIntUnaligned(Object o, long offset, int x) {
4058         if ((offset & 3) == 0) {
4059             putInt(o, offset, x);
4060         } else if ((offset & 1) == 0) {
4061             putIntParts(o, offset,
4062                         (short)(x >> 0),
4063                         (short)(x >>> 16));
4064         } else {
4065             putIntParts(o, offset,
4066                         (byte)(x >>> 0),
4067                         (byte)(x >>> 8),
4068                         (byte)(x >>> 16),
4069                         (byte)(x >>> 24));
4070         }
4071     }
4072     /** @see #putLongUnaligned(Object, long, long, boolean) */
4073     public final void putIntUnaligned(Object o, long offset, int x, boolean bigEndian) {
4074         putIntUnaligned(o, offset, convEndian(bigEndian, x));
4075     }
4076 
4077     /** @see #putLongUnaligned(Object, long, long) */
4078     @IntrinsicCandidate
4079     public final void putShortUnaligned(Object o, long offset, short x) {
4080         if ((offset & 1) == 0) {
4081             putShort(o, offset, x);
4082         } else {
4083             putShortParts(o, offset,
4084                           (byte)(x >>> 0),
4085                           (byte)(x >>> 8));
4086         }
4087     }
4088     /** @see #putLongUnaligned(Object, long, long, boolean) */
4089     public final void putShortUnaligned(Object o, long offset, short x, boolean bigEndian) {
4090         putShortUnaligned(o, offset, convEndian(bigEndian, x));
4091     }
4092 
4093     /** @see #putLongUnaligned(Object, long, long) */
4094     @IntrinsicCandidate
4095     public final void putCharUnaligned(Object o, long offset, char x) {
4096         putShortUnaligned(o, offset, (short)x);
4097     }
4098     /** @see #putLongUnaligned(Object, long, long, boolean) */
4099     public final void putCharUnaligned(Object o, long offset, char x, boolean bigEndian) {
4100         putCharUnaligned(o, offset, convEndian(bigEndian, x));
4101     }
4102 
4103     private static int pickPos(int top, int pos) { return BIG_ENDIAN ? top - pos : pos; }
4104 
4105     // These methods construct integers from bytes.  The byte ordering
4106     // is the native endianness of this platform.
4107     private static long makeLong(byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
4108         return ((toUnsignedLong(i0) << pickPos(56, 0))
4109               | (toUnsignedLong(i1) << pickPos(56, 8))
4110               | (toUnsignedLong(i2) << pickPos(56, 16))
4111               | (toUnsignedLong(i3) << pickPos(56, 24))
4112               | (toUnsignedLong(i4) << pickPos(56, 32))
4113               | (toUnsignedLong(i5) << pickPos(56, 40))
4114               | (toUnsignedLong(i6) << pickPos(56, 48))
4115               | (toUnsignedLong(i7) << pickPos(56, 56)));
4116     }
4117     private static long makeLong(short i0, short i1, short i2, short i3) {
4118         return ((toUnsignedLong(i0) << pickPos(48, 0))
4119               | (toUnsignedLong(i1) << pickPos(48, 16))
4120               | (toUnsignedLong(i2) << pickPos(48, 32))
4121               | (toUnsignedLong(i3) << pickPos(48, 48)));
4122     }
4123     private static long makeLong(int i0, int i1) {
4124         return (toUnsignedLong(i0) << pickPos(32, 0))
4125              | (toUnsignedLong(i1) << pickPos(32, 32));
4126     }
4127     private static int makeInt(short i0, short i1) {
4128         return (toUnsignedInt(i0) << pickPos(16, 0))
4129              | (toUnsignedInt(i1) << pickPos(16, 16));
4130     }
4131     private static int makeInt(byte i0, byte i1, byte i2, byte i3) {
4132         return ((toUnsignedInt(i0) << pickPos(24, 0))
4133               | (toUnsignedInt(i1) << pickPos(24, 8))
4134               | (toUnsignedInt(i2) << pickPos(24, 16))
4135               | (toUnsignedInt(i3) << pickPos(24, 24)));
4136     }
4137     private static short makeShort(byte i0, byte i1) {
4138         return (short)((toUnsignedInt(i0) << pickPos(8, 0))
4139                      | (toUnsignedInt(i1) << pickPos(8, 8)));
4140     }
4141 
4142     private static byte  pick(byte  le, byte  be) { return BIG_ENDIAN ? be : le; }
4143     private static short pick(short le, short be) { return BIG_ENDIAN ? be : le; }
4144     private static int   pick(int   le, int   be) { return BIG_ENDIAN ? be : le; }
4145 
4146     // These methods write integers to memory from smaller parts
4147     // provided by their caller.  The ordering in which these parts
4148     // are written is the native endianness of this platform.
4149     private void putLongParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
4150         putByte(o, offset + 0, pick(i0, i7));
4151         putByte(o, offset + 1, pick(i1, i6));
4152         putByte(o, offset + 2, pick(i2, i5));
4153         putByte(o, offset + 3, pick(i3, i4));
4154         putByte(o, offset + 4, pick(i4, i3));
4155         putByte(o, offset + 5, pick(i5, i2));
4156         putByte(o, offset + 6, pick(i6, i1));
4157         putByte(o, offset + 7, pick(i7, i0));
4158     }
4159     private void putLongParts(Object o, long offset, short i0, short i1, short i2, short i3) {
4160         putShort(o, offset + 0, pick(i0, i3));
4161         putShort(o, offset + 2, pick(i1, i2));
4162         putShort(o, offset + 4, pick(i2, i1));
4163         putShort(o, offset + 6, pick(i3, i0));
4164     }
4165     private void putLongParts(Object o, long offset, int i0, int i1) {
4166         putInt(o, offset + 0, pick(i0, i1));
4167         putInt(o, offset + 4, pick(i1, i0));
4168     }
4169     private void putIntParts(Object o, long offset, short i0, short i1) {
4170         putShort(o, offset + 0, pick(i0, i1));
4171         putShort(o, offset + 2, pick(i1, i0));
4172     }
4173     private void putIntParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3) {
4174         putByte(o, offset + 0, pick(i0, i3));
4175         putByte(o, offset + 1, pick(i1, i2));
4176         putByte(o, offset + 2, pick(i2, i1));
4177         putByte(o, offset + 3, pick(i3, i0));
4178     }
4179     private void putShortParts(Object o, long offset, byte i0, byte i1) {
4180         putByte(o, offset + 0, pick(i0, i1));
4181         putByte(o, offset + 1, pick(i1, i0));
4182     }
4183 
4184     // Zero-extend an integer
4185     private static int toUnsignedInt(byte n)    { return n & 0xff; }
4186     private static int toUnsignedInt(short n)   { return n & 0xffff; }
4187     private static long toUnsignedLong(byte n)  { return n & 0xffl; }
4188     private static long toUnsignedLong(short n) { return n & 0xffffl; }
4189     private static long toUnsignedLong(int n)   { return n & 0xffffffffl; }
4190 
4191     // Maybe byte-reverse an integer
4192     private static char convEndian(boolean big, char n)   { return big == BIG_ENDIAN ? n : Character.reverseBytes(n); }
4193     private static short convEndian(boolean big, short n) { return big == BIG_ENDIAN ? n : Short.reverseBytes(n)    ; }
4194     private static int convEndian(boolean big, int n)     { return big == BIG_ENDIAN ? n : Integer.reverseBytes(n)  ; }
4195     private static long convEndian(boolean big, long n)   { return big == BIG_ENDIAN ? n : Long.reverseBytes(n)     ; }
4196 
4197 
4198 
4199     private native long allocateMemory0(long bytes);
4200     private native long reallocateMemory0(long address, long bytes);
4201     private native void freeMemory0(long address);
4202     private native void setMemory0(Object o, long offset, long bytes, byte value);
4203     @IntrinsicCandidate
4204     private native void copyMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
4205     private native void copySwapMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes, long elemSize);
4206     private native long objectFieldOffset0(Field f);
4207     private native long objectFieldOffset1(Class<?> c, String name);
4208     private native long staticFieldOffset0(Field f);
4209     private native Object staticFieldBase0(Field f);
4210     private native boolean shouldBeInitialized0(Class<?> c);
4211     private native void ensureClassInitialized0(Class<?> c);
4212     private native int arrayBaseOffset0(Class<?> arrayClass);
4213     private native int arrayIndexScale0(Class<?> arrayClass);
4214     private native long getObjectSize0(Object o);
4215     private native int getLoadAverage0(double[] loadavg, int nelems);
4216 
4217 
4218     /**
4219      * Invokes the given direct byte buffer's cleaner, if any.
4220      *
4221      * @param directBuffer a direct byte buffer
4222      * @throws NullPointerException     if {@code directBuffer} is null
4223      * @throws IllegalArgumentException if {@code directBuffer} is non-direct,
4224      *                                  or is a {@link java.nio.Buffer#slice slice}, or is a
4225      *                                  {@link java.nio.Buffer#duplicate duplicate}
4226      */
4227     public void invokeCleaner(java.nio.ByteBuffer directBuffer) {
4228         if (!directBuffer.isDirect())
4229             throw new IllegalArgumentException("buffer is non-direct");
4230 
4231         DirectBuffer db = (DirectBuffer) directBuffer;
4232         if (db.attachment() != null)
4233             throw new IllegalArgumentException("duplicate or slice");
4234 
4235         Cleaner cleaner = db.cleaner();
4236         if (cleaner != null) {
4237             cleaner.clean();
4238         }
4239     }
4240 
4241     // The following deprecated methods are used by JSR 166.
4242 
4243     @Deprecated(since="12", forRemoval=true)
4244     public final Object getObject(Object o, long offset) {
4245         return getReference(o, offset);
4246     }
4247     @Deprecated(since="12", forRemoval=true)
4248     public final Object getObjectVolatile(Object o, long offset) {
4249         return getReferenceVolatile(o, offset);
4250     }
4251     @Deprecated(since="12", forRemoval=true)
4252     public final Object getObjectAcquire(Object o, long offset) {
4253         return getReferenceAcquire(o, offset);
4254     }
4255     @Deprecated(since="12", forRemoval=true)
4256     public final Object getObjectOpaque(Object o, long offset) {
4257         return getReferenceOpaque(o, offset);
4258     }
4259 
4260 
4261     @Deprecated(since="12", forRemoval=true)
4262     public final void putObject(Object o, long offset, Object x) {
4263         putReference(o, offset, x);
4264     }
4265     @Deprecated(since="12", forRemoval=true)
4266     public final void putObjectVolatile(Object o, long offset, Object x) {
4267         putReferenceVolatile(o, offset, x);
4268     }
4269     @Deprecated(since="12", forRemoval=true)
4270     public final void putObjectOpaque(Object o, long offset, Object x) {
4271         putReferenceOpaque(o, offset, x);
4272     }
4273     @Deprecated(since="12", forRemoval=true)
4274     public final void putObjectRelease(Object o, long offset, Object x) {
4275         putReferenceRelease(o, offset, x);
4276     }
4277 
4278 
4279     @Deprecated(since="12", forRemoval=true)
4280     public final Object getAndSetObject(Object o, long offset, Object newValue) {
4281         return getAndSetReference(o, offset, newValue);
4282     }
4283     @Deprecated(since="12", forRemoval=true)
4284     public final Object getAndSetObjectAcquire(Object o, long offset, Object newValue) {
4285         return getAndSetReferenceAcquire(o, offset, newValue);
4286     }
4287     @Deprecated(since="12", forRemoval=true)
4288     public final Object getAndSetObjectRelease(Object o, long offset, Object newValue) {
4289         return getAndSetReferenceRelease(o, offset, newValue);
4290     }
4291 
4292 
4293     @Deprecated(since="12", forRemoval=true)
4294     public final boolean compareAndSetObject(Object o, long offset, Object expected, Object x) {
4295         return compareAndSetReference(o, offset, expected, x);
4296     }
4297     @Deprecated(since="12", forRemoval=true)
4298     public final Object compareAndExchangeObject(Object o, long offset, Object expected, Object x) {
4299         return compareAndExchangeReference(o, offset, expected, x);
4300     }
4301     @Deprecated(since="12", forRemoval=true)
4302     public final Object compareAndExchangeObjectAcquire(Object o, long offset, Object expected, Object x) {
4303         return compareAndExchangeReferenceAcquire(o, offset, expected, x);
4304     }
4305     @Deprecated(since="12", forRemoval=true)
4306     public final Object compareAndExchangeObjectRelease(Object o, long offset, Object expected, Object x) {
4307         return compareAndExchangeReferenceRelease(o, offset, expected, x);
4308     }
4309 
4310 
4311     @Deprecated(since="12", forRemoval=true)
4312     public final boolean weakCompareAndSetObject(Object o, long offset, Object expected, Object x) {
4313         return weakCompareAndSetReference(o, offset, expected, x);
4314     }
4315     @Deprecated(since="12", forRemoval=true)
4316     public final boolean weakCompareAndSetObjectAcquire(Object o, long offset, Object expected, Object x) {
4317         return weakCompareAndSetReferenceAcquire(o, offset, expected, x);
4318     }
4319     @Deprecated(since="12", forRemoval=true)
4320     public final boolean weakCompareAndSetObjectPlain(Object o, long offset, Object expected, Object x) {
4321         return weakCompareAndSetReferencePlain(o, offset, expected, x);
4322     }
4323     @Deprecated(since="12", forRemoval=true)
4324     public final boolean weakCompareAndSetObjectRelease(Object o, long offset, Object expected, Object x) {
4325         return weakCompareAndSetReferenceRelease(o, offset, expected, x);
4326     }
4327 }