1 /*
   2  * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.internal.misc;
  27 
  28 import jdk.internal.ref.Cleaner;
  29 import jdk.internal.vm.annotation.ForceInline;
  30 import jdk.internal.vm.annotation.IntrinsicCandidate;
  31 import sun.nio.ch.DirectBuffer;
  32 
  33 import java.lang.reflect.Field;
  34 import java.security.ProtectionDomain;
  35 
  36 import static jdk.internal.misc.UnsafeConstants.*;
  37 
  38 /**
  39  * A collection of methods for performing low-level, unsafe operations.
  40  * Although the class and all methods are public, use of this class is
  41  * limited because only trusted code can obtain instances of it.
  42  *
  43  * <em>Note:</em> It is the responsibility of the caller to make sure
  44  * arguments are checked before methods of this class are
  45  * called. While some rudimentary checks are performed on the input,
  46  * the checks are best effort and when performance is an overriding
  47  * priority, as when methods of this class are optimized by the
  48  * runtime compiler, some or all checks (if any) may be elided. Hence,
  49  * the caller must not rely on the checks and corresponding
  50  * exceptions!
  51  *
  52  * @author John R. Rose
  53  * @see #getUnsafe
  54  */
  55 
  56 public final class Unsafe {
  57 
  58     private static native void registerNatives();
  59     static {
  60         runtimeSetup();
  61     }
  62 
  63     // Called from JVM when loading an AOT cache
  64     private static void runtimeSetup() {
  65         registerNatives();
  66     }
  67 
  68     private Unsafe() {}
  69 
  70     private static final Unsafe theUnsafe = new Unsafe();
  71 
  72     /**
  73      * Provides the caller with the capability of performing unsafe
  74      * operations.
  75      *
  76      * <p>The returned {@code Unsafe} object should be carefully guarded
  77      * by the caller, since it can be used to read and write data at arbitrary
  78      * memory addresses.  It must never be passed to untrusted code.
  79      *
  80      * <p>Most methods in this class are very low-level, and correspond to a
  81      * small number of hardware instructions (on typical machines).  Compilers
  82      * are encouraged to optimize these methods accordingly.
  83      *
  84      * <p>Here is a suggested idiom for using unsafe operations:
  85      *
  86      * <pre> {@code
  87      * class MyTrustedClass {
  88      *   private static final Unsafe unsafe = Unsafe.getUnsafe();
  89      *   ...
  90      *   private long myCountAddress = ...;
  91      *   public int getCount() { return unsafe.getByte(myCountAddress); }
  92      * }}</pre>
  93      *
  94      * (It may assist compilers to make the local variable {@code final}.)
  95      */
  96     public static Unsafe getUnsafe() {
  97         return theUnsafe;
  98     }
  99 
 100     //--- peek and poke operations
 101     // (compilers should optimize these to memory ops)
 102 
 103     // These work on object fields in the Java heap.
 104     // They will not work on elements of packed arrays.
 105 
 106     /**
 107      * Fetches a value from a given Java variable.
 108      * More specifically, fetches a field or array element within the given
 109      * object {@code o} at the given offset, or (if {@code o} is null)
 110      * from the memory address whose numerical value is the given offset.
 111      * <p>
 112      * The results are undefined unless one of the following cases is true:
 113      * <ul>
 114      * <li>The offset was obtained from {@link #objectFieldOffset} on
 115      * the {@link java.lang.reflect.Field} of some Java field and the object
 116      * referred to by {@code o} is of a class compatible with that
 117      * field's class.
 118      *
 119      * <li>The offset and object reference {@code o} (either null or
 120      * non-null) were both obtained via {@link #staticFieldOffset}
 121      * and {@link #staticFieldBase} (respectively) from the
 122      * reflective {@link Field} representation of some Java field.
 123      *
 124      * <li>The object referred to by {@code o} is an array, and the offset
 125      * is an integer of the form {@code B+N*S}, where {@code N} is
 126      * a valid index into the array, and {@code B} and {@code S} are
 127      * the values obtained by {@link #arrayBaseOffset} and {@link
 128      * #arrayIndexScale} (respectively) from the array's class.  The value
 129      * referred to is the {@code N}<em>th</em> element of the array.
 130      *
 131      * </ul>
 132      * <p>
 133      * If one of the above cases is true, the call references a specific Java
 134      * variable (field or array element).  However, the results are undefined
 135      * if that variable is not in fact of the type returned by this method.
 136      * <p>
 137      * This method refers to a variable by means of two parameters, and so
 138      * it provides (in effect) a <em>double-register</em> addressing mode
 139      * for Java variables.  When the object reference is null, this method
 140      * uses its offset as an absolute address.  This is similar in operation
 141      * to methods such as {@link #getInt(long)}, which provide (in effect) a
 142      * <em>single-register</em> addressing mode for non-Java variables.
 143      * However, because Java variables may have a different layout in memory
 144      * from non-Java variables, programmers should not assume that these
 145      * two addressing modes are ever equivalent.  Also, programmers should
 146      * remember that offsets from the double-register addressing mode cannot
 147      * be portably confused with longs used in the single-register addressing
 148      * mode.
 149      *
 150      * @param o Java heap object in which the variable resides, if any, else
 151      *        null
 152      * @param offset indication of where the variable resides in a Java heap
 153      *        object, if any, else a memory address locating the variable
 154      *        statically
 155      * @return the value fetched from the indicated Java variable
 156      * @throws RuntimeException No defined exceptions are thrown, not even
 157      *         {@link NullPointerException}
 158      */
 159     @IntrinsicCandidate
 160     public native int getInt(Object o, long offset);
 161 
 162     /**
 163      * Stores a value into a given Java variable.
 164      * <p>
 165      * The first two parameters are interpreted exactly as with
 166      * {@link #getInt(Object, long)} to refer to a specific
 167      * Java variable (field or array element).  The given value
 168      * is stored into that variable.
 169      * <p>
 170      * The variable must be of the same type as the method
 171      * parameter {@code x}.
 172      *
 173      * @param o Java heap object in which the variable resides, if any, else
 174      *        null
 175      * @param offset indication of where the variable resides in a Java heap
 176      *        object, if any, else a memory address locating the variable
 177      *        statically
 178      * @param x the value to store into the indicated Java variable
 179      * @throws RuntimeException No defined exceptions are thrown, not even
 180      *         {@link NullPointerException}
 181      */
 182     @IntrinsicCandidate
 183     public native void putInt(Object o, long offset, int x);
 184 
 185     /**
 186      * Fetches a reference value from a given Java variable.
 187      * @see #getInt(Object, long)
 188      */
 189     @IntrinsicCandidate
 190     public native Object getReference(Object o, long offset);
 191 
 192     /**
 193      * Stores a reference value into a given Java variable.
 194      * <p>
 195      * Unless the reference {@code x} being stored is either null
 196      * or matches the field type, the results are undefined.
 197      * If the reference {@code o} is non-null, card marks or
 198      * other store barriers for that object (if the VM requires them)
 199      * are updated.
 200      * @see #putInt(Object, long, int)
 201      */
 202     @IntrinsicCandidate
 203     public native void putReference(Object o, long offset, Object x);
 204 
 205     /** @see #getInt(Object, long) */
 206     @IntrinsicCandidate
 207     public native boolean getBoolean(Object o, long offset);
 208 
 209     /** @see #putInt(Object, long, int) */
 210     @IntrinsicCandidate
 211     public native void    putBoolean(Object o, long offset, boolean x);
 212 
 213     /** @see #getInt(Object, long) */
 214     @IntrinsicCandidate
 215     public native byte    getByte(Object o, long offset);
 216 
 217     /** @see #putInt(Object, long, int) */
 218     @IntrinsicCandidate
 219     public native void    putByte(Object o, long offset, byte x);
 220 
 221     /** @see #getInt(Object, long) */
 222     @IntrinsicCandidate
 223     public native short   getShort(Object o, long offset);
 224 
 225     /** @see #putInt(Object, long, int) */
 226     @IntrinsicCandidate
 227     public native void    putShort(Object o, long offset, short x);
 228 
 229     /** @see #getInt(Object, long) */
 230     @IntrinsicCandidate
 231     public native char    getChar(Object o, long offset);
 232 
 233     /** @see #putInt(Object, long, int) */
 234     @IntrinsicCandidate
 235     public native void    putChar(Object o, long offset, char x);
 236 
 237     /** @see #getInt(Object, long) */
 238     @IntrinsicCandidate
 239     public native long    getLong(Object o, long offset);
 240 
 241     /** @see #putInt(Object, long, int) */
 242     @IntrinsicCandidate
 243     public native void    putLong(Object o, long offset, long x);
 244 
 245     /** @see #getInt(Object, long) */
 246     @IntrinsicCandidate
 247     public native float   getFloat(Object o, long offset);
 248 
 249     /** @see #putInt(Object, long, int) */
 250     @IntrinsicCandidate
 251     public native void    putFloat(Object o, long offset, float x);
 252 
 253     /** @see #getInt(Object, long) */
 254     @IntrinsicCandidate
 255     public native double  getDouble(Object o, long offset);
 256 
 257     /** @see #putInt(Object, long, int) */
 258     @IntrinsicCandidate
 259     public native void    putDouble(Object o, long offset, double x);
 260 
 261     /**
 262      * Fetches a native pointer from a given memory address.  If the address is
 263      * zero, or does not point into a block obtained from {@link
 264      * #allocateMemory}, the results are undefined.
 265      *
 266      * <p>If the native pointer is less than 64 bits wide, it is extended as
 267      * an unsigned number to a Java long.  The pointer may be indexed by any
 268      * given byte offset, simply by adding that offset (as a simple integer) to
 269      * the long representing the pointer.  The number of bytes actually read
 270      * from the target address may be determined by consulting {@link
 271      * #addressSize}.
 272      *
 273      * @see #allocateMemory
 274      * @see #getInt(Object, long)
 275      */
 276     @ForceInline
 277     public long getAddress(Object o, long offset) {
 278         if (ADDRESS_SIZE == 4) {
 279             return Integer.toUnsignedLong(getInt(o, offset));
 280         } else {
 281             return getLong(o, offset);
 282         }
 283     }
 284 
 285     /**
 286      * Stores a native pointer into a given memory address.  If the address is
 287      * zero, or does not point into a block obtained from {@link
 288      * #allocateMemory}, the results are undefined.
 289      *
 290      * <p>The number of bytes actually written at the target address may be
 291      * determined by consulting {@link #addressSize}.
 292      *
 293      * @see #allocateMemory
 294      * @see #putInt(Object, long, int)
 295      */
 296     @ForceInline
 297     public void putAddress(Object o, long offset, long x) {
 298         if (ADDRESS_SIZE == 4) {
 299             putInt(o, offset, (int)x);
 300         } else {
 301             putLong(o, offset, x);
 302         }
 303     }
 304 
 305     // These read VM internal data.
 306 
 307     /**
 308      * Fetches an uncompressed reference value from a given native variable
 309      * ignoring the VM's compressed references mode.
 310      *
 311      * @param address a memory address locating the variable
 312      * @return the value fetched from the indicated native variable
 313      */
 314     public native Object getUncompressedObject(long address);
 315 
 316     // These work on values in the C heap.
 317 
 318     /**
 319      * Fetches a value from a given memory address.  If the address is zero, or
 320      * does not point into a block obtained from {@link #allocateMemory}, the
 321      * results are undefined.
 322      *
 323      * @see #allocateMemory
 324      */
 325     @ForceInline
 326     public byte getByte(long address) {
 327         return getByte(null, address);
 328     }
 329 
 330     /**
 331      * Stores a value into a given memory address.  If the address is zero, or
 332      * does not point into a block obtained from {@link #allocateMemory}, the
 333      * results are undefined.
 334      *
 335      * @see #getByte(long)
 336      */
 337     @ForceInline
 338     public void putByte(long address, byte x) {
 339         putByte(null, address, x);
 340     }
 341 
 342     /** @see #getByte(long) */
 343     @ForceInline
 344     public short getShort(long address) {
 345         return getShort(null, address);
 346     }
 347 
 348     /** @see #putByte(long, byte) */
 349     @ForceInline
 350     public void putShort(long address, short x) {
 351         putShort(null, address, x);
 352     }
 353 
 354     /** @see #getByte(long) */
 355     @ForceInline
 356     public char getChar(long address) {
 357         return getChar(null, address);
 358     }
 359 
 360     /** @see #putByte(long, byte) */
 361     @ForceInline
 362     public void putChar(long address, char x) {
 363         putChar(null, address, x);
 364     }
 365 
 366     /** @see #getByte(long) */
 367     @ForceInline
 368     public int getInt(long address) {
 369         return getInt(null, address);
 370     }
 371 
 372     /** @see #putByte(long, byte) */
 373     @ForceInline
 374     public void putInt(long address, int x) {
 375         putInt(null, address, x);
 376     }
 377 
 378     /** @see #getByte(long) */
 379     @ForceInline
 380     public long getLong(long address) {
 381         return getLong(null, address);
 382     }
 383 
 384     /** @see #putByte(long, byte) */
 385     @ForceInline
 386     public void putLong(long address, long x) {
 387         putLong(null, address, x);
 388     }
 389 
 390     /** @see #getByte(long) */
 391     @ForceInline
 392     public float getFloat(long address) {
 393         return getFloat(null, address);
 394     }
 395 
 396     /** @see #putByte(long, byte) */
 397     @ForceInline
 398     public void putFloat(long address, float x) {
 399         putFloat(null, address, x);
 400     }
 401 
 402     /** @see #getByte(long) */
 403     @ForceInline
 404     public double getDouble(long address) {
 405         return getDouble(null, address);
 406     }
 407 
 408     /** @see #putByte(long, byte) */
 409     @ForceInline
 410     public void putDouble(long address, double x) {
 411         putDouble(null, address, x);
 412     }
 413 
 414     /** @see #getAddress(Object, long) */
 415     @ForceInline
 416     public long getAddress(long address) {
 417         return getAddress(null, address);
 418     }
 419 
 420     /** @see #putAddress(Object, long, long) */
 421     @ForceInline
 422     public void putAddress(long address, long x) {
 423         putAddress(null, address, x);
 424     }
 425 
 426 
 427 
 428     //--- helper methods for validating various types of objects/values
 429 
 430     /**
 431      * Create an exception reflecting that some of the input was invalid
 432      *
 433      * <em>Note:</em> It is the responsibility of the caller to make
 434      * sure arguments are checked before the methods are called. While
 435      * some rudimentary checks are performed on the input, the checks
 436      * are best effort and when performance is an overriding priority,
 437      * as when methods of this class are optimized by the runtime
 438      * compiler, some or all checks (if any) may be elided. Hence, the
 439      * caller must not rely on the checks and corresponding
 440      * exceptions!
 441      *
 442      * @return an exception object
 443      */
 444     private RuntimeException invalidInput() {
 445         return new IllegalArgumentException();
 446     }
 447 
 448     /**
 449      * Check if a value is 32-bit clean (32 MSB are all zero)
 450      *
 451      * @param value the 64-bit value to check
 452      *
 453      * @return true if the value is 32-bit clean
 454      */
 455     private boolean is32BitClean(long value) {
 456         return value >>> 32 == 0;
 457     }
 458 
 459     /**
 460      * Check the validity of a size (the equivalent of a size_t)
 461      *
 462      * @throws RuntimeException if the size is invalid
 463      *         (<em>Note:</em> after optimization, invalid inputs may
 464      *         go undetected, which will lead to unpredictable
 465      *         behavior)
 466      */
 467     private void checkSize(long size) {
 468         if (ADDRESS_SIZE == 4) {
 469             // Note: this will also check for negative sizes
 470             if (!is32BitClean(size)) {
 471                 throw invalidInput();
 472             }
 473         } else if (size < 0) {
 474             throw invalidInput();
 475         }
 476     }
 477 
 478     /**
 479      * Check the validity of a native address (the equivalent of void*)
 480      *
 481      * @throws RuntimeException if the address is invalid
 482      *         (<em>Note:</em> after optimization, invalid inputs may
 483      *         go undetected, which will lead to unpredictable
 484      *         behavior)
 485      */
 486     private void checkNativeAddress(long address) {
 487         if (ADDRESS_SIZE == 4) {
 488             // Accept both zero and sign extended pointers. A valid
 489             // pointer will, after the +1 below, either have produced
 490             // the value 0x0 or 0x1. Masking off the low bit allows
 491             // for testing against 0.
 492             if ((((address >> 32) + 1) & ~1) != 0) {
 493                 throw invalidInput();
 494             }
 495         }
 496     }
 497 
 498     /**
 499      * Check the validity of an offset, relative to a base object
 500      *
 501      * @param o the base object
 502      * @param offset the offset to check
 503      *
 504      * @throws RuntimeException if the size is invalid
 505      *         (<em>Note:</em> after optimization, invalid inputs may
 506      *         go undetected, which will lead to unpredictable
 507      *         behavior)
 508      */
 509     private void checkOffset(Object o, long offset) {
 510         if (ADDRESS_SIZE == 4) {
 511             // Note: this will also check for negative offsets
 512             if (!is32BitClean(offset)) {
 513                 throw invalidInput();
 514             }
 515         } else if (offset < 0) {
 516             throw invalidInput();
 517         }
 518     }
 519 
 520     /**
 521      * Check the validity of a double-register pointer
 522      *
 523      * Note: This code deliberately does *not* check for NPE for (at
 524      * least) three reasons:
 525      *
 526      * 1) NPE is not just NULL/0 - there is a range of values all
 527      * resulting in an NPE, which is not trivial to check for
 528      *
 529      * 2) It is the responsibility of the callers of Unsafe methods
 530      * to verify the input, so throwing an exception here is not really
 531      * useful - passing in a NULL pointer is a critical error and the
 532      * must not expect an exception to be thrown anyway.
 533      *
 534      * 3) the actual operations will detect NULL pointers anyway by
 535      * means of traps and signals (like SIGSEGV).
 536      *
 537      * @param o Java heap object, or null
 538      * @param offset indication of where the variable resides in a Java heap
 539      *        object, if any, else a memory address locating the variable
 540      *        statically
 541      *
 542      * @throws RuntimeException if the pointer is invalid
 543      *         (<em>Note:</em> after optimization, invalid inputs may
 544      *         go undetected, which will lead to unpredictable
 545      *         behavior)
 546      */
 547     private void checkPointer(Object o, long offset) {
 548         if (o == null) {
 549             checkNativeAddress(offset);
 550         } else {
 551             checkOffset(o, offset);
 552         }
 553     }
 554 
 555     /**
 556      * Check if a type is a primitive array type
 557      *
 558      * @param c the type to check
 559      *
 560      * @return true if the type is a primitive array type
 561      */
 562     private void checkPrimitiveArray(Class<?> c) {
 563         Class<?> componentType = c.getComponentType();
 564         if (componentType == null || !componentType.isPrimitive()) {
 565             throw invalidInput();
 566         }
 567     }
 568 
 569     /**
 570      * Check that a pointer is a valid primitive array type pointer
 571      *
 572      * Note: pointers off-heap are considered to be primitive arrays
 573      *
 574      * @throws RuntimeException if the pointer is invalid
 575      *         (<em>Note:</em> after optimization, invalid inputs may
 576      *         go undetected, which will lead to unpredictable
 577      *         behavior)
 578      */
 579     private void checkPrimitivePointer(Object o, long offset) {
 580         checkPointer(o, offset);
 581 
 582         if (o != null) {
 583             // If on heap, it must be a primitive array
 584             checkPrimitiveArray(o.getClass());
 585         }
 586     }
 587 
 588 
 589     //--- wrappers for malloc, realloc, free:
 590 
 591     /**
 592      * Round up allocation size to a multiple of HeapWordSize.
 593      */
 594     private long alignToHeapWordSize(long bytes) {
 595         if (bytes >= 0) {
 596             return (bytes + ADDRESS_SIZE - 1) & ~(ADDRESS_SIZE - 1);
 597         } else {
 598             throw invalidInput();
 599         }
 600     }
 601 
 602     /**
 603      * Allocates a new block of native memory, of the given size in bytes.  The
 604      * contents of the memory are uninitialized; they will generally be
 605      * garbage.  The resulting native pointer will be zero if and only if the
 606      * requested size is zero.  The resulting native pointer will be aligned for
 607      * all value types.   Dispose of this memory by calling {@link #freeMemory}
 608      * or resize it with {@link #reallocateMemory}.
 609      *
 610      * <em>Note:</em> It is the responsibility of the caller to make
 611      * sure arguments are checked before the methods are called. While
 612      * some rudimentary checks are performed on the input, the checks
 613      * are best effort and when performance is an overriding priority,
 614      * as when methods of this class are optimized by the runtime
 615      * compiler, some or all checks (if any) may be elided. Hence, the
 616      * caller must not rely on the checks and corresponding
 617      * exceptions!
 618      *
 619      * @throws RuntimeException if the size is negative or too large
 620      *         for the native size_t type
 621      *
 622      * @throws OutOfMemoryError if the allocation is refused by the system
 623      *
 624      * @see #getByte(long)
 625      * @see #putByte(long, byte)
 626      */
 627     public long allocateMemory(long bytes) {
 628         bytes = alignToHeapWordSize(bytes);
 629 
 630         allocateMemoryChecks(bytes);
 631 
 632         if (bytes == 0) {
 633             return 0;
 634         }
 635 
 636         long p = allocateMemory0(bytes);
 637         if (p == 0) {
 638             throw new OutOfMemoryError("Unable to allocate " + bytes + " bytes");
 639         }
 640 
 641         return p;
 642     }
 643 
 644     /**
 645      * Validate the arguments to allocateMemory
 646      *
 647      * @throws RuntimeException if the arguments are invalid
 648      *         (<em>Note:</em> after optimization, invalid inputs may
 649      *         go undetected, which will lead to unpredictable
 650      *         behavior)
 651      */
 652     private void allocateMemoryChecks(long bytes) {
 653         checkSize(bytes);
 654     }
 655 
 656     /**
 657      * Resizes a new block of native memory, to the given size in bytes.  The
 658      * contents of the new block past the size of the old block are
 659      * uninitialized; they will generally be garbage.  The resulting native
 660      * pointer will be zero if and only if the requested size is zero.  The
 661      * resulting native pointer will be aligned for all value types.  Dispose
 662      * of this memory by calling {@link #freeMemory}, or resize it with {@link
 663      * #reallocateMemory}.  The address passed to this method may be null, in
 664      * which case an allocation will be performed.
 665      *
 666      * <em>Note:</em> It is the responsibility of the caller to make
 667      * sure arguments are checked before the methods are called. While
 668      * some rudimentary checks are performed on the input, the checks
 669      * are best effort and when performance is an overriding priority,
 670      * as when methods of this class are optimized by the runtime
 671      * compiler, some or all checks (if any) may be elided. Hence, the
 672      * caller must not rely on the checks and corresponding
 673      * exceptions!
 674      *
 675      * @throws RuntimeException if the size is negative or too large
 676      *         for the native size_t type
 677      *
 678      * @throws OutOfMemoryError if the allocation is refused by the system
 679      *
 680      * @see #allocateMemory
 681      */
 682     public long reallocateMemory(long address, long bytes) {
 683         bytes = alignToHeapWordSize(bytes);
 684 
 685         reallocateMemoryChecks(address, bytes);
 686 
 687         if (bytes == 0) {
 688             freeMemory(address);
 689             return 0;
 690         }
 691 
 692         long p = (address == 0) ? allocateMemory0(bytes) : reallocateMemory0(address, bytes);
 693         if (p == 0) {
 694             throw new OutOfMemoryError("Unable to allocate " + bytes + " bytes");
 695         }
 696 
 697         return p;
 698     }
 699 
 700     /**
 701      * Validate the arguments to reallocateMemory
 702      *
 703      * @throws RuntimeException if the arguments are invalid
 704      *         (<em>Note:</em> after optimization, invalid inputs may
 705      *         go undetected, which will lead to unpredictable
 706      *         behavior)
 707      */
 708     private void reallocateMemoryChecks(long address, long bytes) {
 709         checkPointer(null, address);
 710         checkSize(bytes);
 711     }
 712 
 713     /**
 714      * Sets all bytes in a given block of memory to a fixed value
 715      * (usually zero).
 716      *
 717      * <p>This method determines a block's base address by means of two parameters,
 718      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 719      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 720      * the offset supplies an absolute base address.
 721      *
 722      * <p>The stores are in coherent (atomic) units of a size determined
 723      * by the address and length parameters.  If the effective address and
 724      * length are all even modulo 8, the stores take place in 'long' units.
 725      * If the effective address and length are (resp.) even modulo 4 or 2,
 726      * the stores take place in units of 'int' or 'short'.
 727      *
 728      * <em>Note:</em> It is the responsibility of the caller to make
 729      * sure arguments are checked before the methods are called. While
 730      * some rudimentary checks are performed on the input, the checks
 731      * are best effort and when performance is an overriding priority,
 732      * as when methods of this class are optimized by the runtime
 733      * compiler, some or all checks (if any) may be elided. Hence, the
 734      * caller must not rely on the checks and corresponding
 735      * exceptions!
 736      *
 737      * @throws RuntimeException if any of the arguments is invalid
 738      *
 739      * @since 1.7
 740      */
 741     public void setMemory(Object o, long offset, long bytes, byte value) {
 742         setMemoryChecks(o, offset, bytes, value);
 743 
 744         if (bytes == 0) {
 745             return;
 746         }
 747 
 748         setMemory0(o, offset, bytes, value);
 749     }
 750 
 751     /**
 752      * Sets all bytes in a given block of memory to a fixed value
 753      * (usually zero).  This provides a <em>single-register</em> addressing mode,
 754      * as discussed in {@link #getInt(Object,long)}.
 755      *
 756      * <p>Equivalent to {@code setMemory(null, address, bytes, value)}.
 757      */
 758     public void setMemory(long address, long bytes, byte value) {
 759         setMemory(null, address, bytes, value);
 760     }
 761 
 762     /**
 763      * Validate the arguments to setMemory
 764      *
 765      * @throws RuntimeException if the arguments are invalid
 766      *         (<em>Note:</em> after optimization, invalid inputs may
 767      *         go undetected, which will lead to unpredictable
 768      *         behavior)
 769      */
 770     private void setMemoryChecks(Object o, long offset, long bytes, byte value) {
 771         checkPrimitivePointer(o, offset);
 772         checkSize(bytes);
 773     }
 774 
 775     /**
 776      * Sets all bytes in a given block of memory to a copy of another
 777      * block.
 778      *
 779      * <p>This method determines each block's base address by means of two parameters,
 780      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 781      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 782      * the offset supplies an absolute base address.
 783      *
 784      * <p>The transfers are in coherent (atomic) units of a size determined
 785      * by the address and length parameters.  If the effective addresses and
 786      * length are all even modulo 8, the transfer takes place in 'long' units.
 787      * If the effective addresses and length are (resp.) even modulo 4 or 2,
 788      * the transfer takes place in units of 'int' or 'short'.
 789      *
 790      * <em>Note:</em> It is the responsibility of the caller to make
 791      * sure arguments are checked before the methods are called. While
 792      * some rudimentary checks are performed on the input, the checks
 793      * are best effort and when performance is an overriding priority,
 794      * as when methods of this class are optimized by the runtime
 795      * compiler, some or all checks (if any) may be elided. Hence, the
 796      * caller must not rely on the checks and corresponding
 797      * exceptions!
 798      *
 799      * @throws RuntimeException if any of the arguments is invalid
 800      *
 801      * @since 1.7
 802      */
 803     public void copyMemory(Object srcBase, long srcOffset,
 804                            Object destBase, long destOffset,
 805                            long bytes) {
 806         copyMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes);
 807 
 808         if (bytes == 0) {
 809             return;
 810         }
 811 
 812         copyMemory0(srcBase, srcOffset, destBase, destOffset, bytes);
 813     }
 814 
 815     /**
 816      * Sets all bytes in a given block of memory to a copy of another
 817      * block.  This provides a <em>single-register</em> addressing mode,
 818      * as discussed in {@link #getInt(Object,long)}.
 819      *
 820      * Equivalent to {@code copyMemory(null, srcAddress, null, destAddress, bytes)}.
 821      */
 822     public void copyMemory(long srcAddress, long destAddress, long bytes) {
 823         copyMemory(null, srcAddress, null, destAddress, bytes);
 824     }
 825 
 826     /**
 827      * Validate the arguments to copyMemory
 828      *
 829      * @throws RuntimeException if any of the arguments is invalid
 830      *         (<em>Note:</em> after optimization, invalid inputs may
 831      *         go undetected, which will lead to unpredictable
 832      *         behavior)
 833      */
 834     private void copyMemoryChecks(Object srcBase, long srcOffset,
 835                                   Object destBase, long destOffset,
 836                                   long bytes) {
 837         checkSize(bytes);
 838         checkPrimitivePointer(srcBase, srcOffset);
 839         checkPrimitivePointer(destBase, destOffset);
 840     }
 841 
 842     /**
 843      * Copies all elements from one block of memory to another block,
 844      * *unconditionally* byte swapping the elements on the fly.
 845      *
 846      * <p>This method determines each block's base address by means of two parameters,
 847      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 848      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 849      * the offset supplies an absolute base address.
 850      *
 851      * <em>Note:</em> It is the responsibility of the caller to make
 852      * sure arguments are checked before the methods are called. While
 853      * some rudimentary checks are performed on the input, the checks
 854      * are best effort and when performance is an overriding priority,
 855      * as when methods of this class are optimized by the runtime
 856      * compiler, some or all checks (if any) may be elided. Hence, the
 857      * caller must not rely on the checks and corresponding
 858      * exceptions!
 859      *
 860      * @throws RuntimeException if any of the arguments is invalid
 861      *
 862      * @since 9
 863      */
 864     public void copySwapMemory(Object srcBase, long srcOffset,
 865                                Object destBase, long destOffset,
 866                                long bytes, long elemSize) {
 867         copySwapMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
 868 
 869         if (bytes == 0) {
 870             return;
 871         }
 872 
 873         copySwapMemory0(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
 874     }
 875 
 876     private void copySwapMemoryChecks(Object srcBase, long srcOffset,
 877                                       Object destBase, long destOffset,
 878                                       long bytes, long elemSize) {
 879         checkSize(bytes);
 880 
 881         if (elemSize != 2 && elemSize != 4 && elemSize != 8) {
 882             throw invalidInput();
 883         }
 884         if (bytes % elemSize != 0) {
 885             throw invalidInput();
 886         }
 887 
 888         checkPrimitivePointer(srcBase, srcOffset);
 889         checkPrimitivePointer(destBase, destOffset);
 890     }
 891 
 892     /**
 893      * Copies all elements from one block of memory to another block, byte swapping the
 894      * elements on the fly.
 895      *
 896      * This provides a <em>single-register</em> addressing mode, as
 897      * discussed in {@link #getInt(Object,long)}.
 898      *
 899      * Equivalent to {@code copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize)}.
 900      */
 901     public void copySwapMemory(long srcAddress, long destAddress, long bytes, long elemSize) {
 902         copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize);
 903     }
 904 
 905     /**
 906      * Disposes of a block of native memory, as obtained from {@link
 907      * #allocateMemory} or {@link #reallocateMemory}.  The address passed to
 908      * this method may be null, in which case no action is taken.
 909      *
 910      * <em>Note:</em> It is the responsibility of the caller to make
 911      * sure arguments are checked before the methods are called. While
 912      * some rudimentary checks are performed on the input, the checks
 913      * are best effort and when performance is an overriding priority,
 914      * as when methods of this class are optimized by the runtime
 915      * compiler, some or all checks (if any) may be elided. Hence, the
 916      * caller must not rely on the checks and corresponding
 917      * exceptions!
 918      *
 919      * @throws RuntimeException if any of the arguments is invalid
 920      *
 921      * @see #allocateMemory
 922      */
 923     public void freeMemory(long address) {
 924         freeMemoryChecks(address);
 925 
 926         if (address == 0) {
 927             return;
 928         }
 929 
 930         freeMemory0(address);
 931     }
 932 
 933     /**
 934      * Validate the arguments to freeMemory
 935      *
 936      * @throws RuntimeException if the arguments are invalid
 937      *         (<em>Note:</em> after optimization, invalid inputs may
 938      *         go undetected, which will lead to unpredictable
 939      *         behavior)
 940      */
 941     private void freeMemoryChecks(long address) {
 942         checkPointer(null, address);
 943     }
 944 
 945     /**
 946      * Ensure writeback of a specified virtual memory address range
 947      * from cache to physical memory. All bytes in the address range
 948      * are guaranteed to have been written back to physical memory on
 949      * return from this call i.e. subsequently executed store
 950      * instructions are guaranteed not to be visible before the
 951      * writeback is completed.
 952      *
 953      * @param address
 954      *        the lowest byte address that must be guaranteed written
 955      *        back to memory. bytes at lower addresses may also be
 956      *        written back.
 957      *
 958      * @param length
 959      *        the length in bytes of the region starting at address
 960      *        that must be guaranteed written back to memory.
 961      *
 962      * @throws RuntimeException if memory writeback is not supported
 963      *         on the current hardware of if the arguments are invalid.
 964      *         (<em>Note:</em> after optimization, invalid inputs may
 965      *         go undetected, which will lead to unpredictable
 966      *         behavior)
 967      *
 968      * @since 14
 969      */
 970 
 971     public void writebackMemory(long address, long length) {
 972         checkWritebackEnabled();
 973         checkWritebackMemory(address, length);
 974 
 975         // perform any required pre-writeback barrier
 976         writebackPreSync0();
 977 
 978         // write back one cache line at a time
 979         long line = dataCacheLineAlignDown(address);
 980         long end = address + length;
 981         while (line < end) {
 982             writeback0(line);
 983             line += dataCacheLineFlushSize();
 984         }
 985 
 986         // perform any required post-writeback barrier
 987         writebackPostSync0();
 988     }
 989 
 990     /**
 991      * Validate the arguments to writebackMemory
 992      *
 993      * @throws RuntimeException if the arguments are invalid
 994      *         (<em>Note:</em> after optimization, invalid inputs may
 995      *         go undetected, which will lead to unpredictable
 996      *         behavior)
 997      */
 998     private void checkWritebackMemory(long address, long length) {
 999         checkNativeAddress(address);
1000         checkSize(length);
1001     }
1002 
1003     /**
1004      * Validate that the current hardware supports memory writeback.
1005      * (<em>Note:</em> this is a belt and braces check.  Clients are
1006      * expected to test whether writeback is enabled by calling
1007      * ({@link isWritebackEnabled #isWritebackEnabled} and avoid
1008      * calling method {@link writeback #writeback} if it is disabled).
1009      *
1010      *
1011      * @throws RuntimeException if memory writeback is not supported
1012      */
1013     private void checkWritebackEnabled() {
1014         if (!isWritebackEnabled()) {
1015             throw new RuntimeException("writebackMemory not enabled!");
1016         }
1017     }
1018 
1019     /**
1020      * force writeback of an individual cache line.
1021      *
1022      * @param address
1023      *        the start address of the cache line to be written back
1024      */
1025     @IntrinsicCandidate
1026     private native void writeback0(long address);
1027 
1028      /**
1029       * Serialize writeback operations relative to preceding memory writes.
1030       */
1031     @IntrinsicCandidate
1032     private native void writebackPreSync0();
1033 
1034      /**
1035       * Serialize writeback operations relative to following memory writes.
1036       */
1037     @IntrinsicCandidate
1038     private native void writebackPostSync0();
1039 
1040     //--- random queries
1041 
1042     /**
1043      * This constant differs from all results that will ever be returned from
1044      * {@link #staticFieldOffset}, {@link #objectFieldOffset},
1045      * or {@link #arrayBaseOffset}.
1046      */
1047     public static final int INVALID_FIELD_OFFSET = -1;
1048 
1049     /**
1050      * Reports the location of a given field in the storage allocation of its
1051      * class.  Do not expect to perform any sort of arithmetic on this offset;
1052      * it is just a cookie which is passed to the unsafe heap memory accessors.
1053      *
1054      * <p>Any given field will always have the same offset and base, and no
1055      * two distinct fields of the same class will ever have the same offset
1056      * and base.
1057      *
1058      * <p>As of 1.4.1, offsets for fields are represented as long values,
1059      * although the Sun JVM does not use the most significant 32 bits.
1060      * However, JVM implementations which store static fields at absolute
1061      * addresses can use long offsets and null base pointers to express
1062      * the field locations in a form usable by {@link #getInt(Object,long)}.
1063      * Therefore, code which will be ported to such JVMs on 64-bit platforms
1064      * must preserve all bits of static field offsets.
1065      * @see #getInt(Object, long)
1066      */
1067     public long objectFieldOffset(Field f) {
1068         if (f == null) {
1069             throw new NullPointerException();
1070         }
1071 
1072         return objectFieldOffset0(f);
1073     }
1074 
1075     /**
1076      * Reports the location of the field with a given name in the storage
1077      * allocation of its class.
1078      *
1079      * @throws NullPointerException if any parameter is {@code null}.
1080      * @throws InternalError if there is no field named {@code name} declared
1081      *         in class {@code c}, i.e., if {@code c.getDeclaredField(name)}
1082      *         would throw {@code java.lang.NoSuchFieldException}.
1083      *
1084      * @see #objectFieldOffset(Field)
1085      */
1086     public long objectFieldOffset(Class<?> c, String name) {
1087         if (c == null || name == null) {
1088             throw new NullPointerException();
1089         }
1090 
1091         return objectFieldOffset1(c, name);
1092     }
1093 
1094     /**
1095      * Reports the location of a given static field, in conjunction with {@link
1096      * #staticFieldBase}.
1097      * <p>Do not expect to perform any sort of arithmetic on this offset;
1098      * it is just a cookie which is passed to the unsafe heap memory accessors.
1099      *
1100      * <p>Any given field will always have the same offset, and no two distinct
1101      * fields of the same class will ever have the same offset.
1102      *
1103      * <p>As of 1.4.1, offsets for fields are represented as long values,
1104      * although the Sun JVM does not use the most significant 32 bits.
1105      * It is hard to imagine a JVM technology which needs more than
1106      * a few bits to encode an offset within a non-array object,
1107      * However, for consistency with other methods in this class,
1108      * this method reports its result as a long value.
1109      * @see #getInt(Object, long)
1110      */
1111     public long staticFieldOffset(Field f) {
1112         if (f == null) {
1113             throw new NullPointerException();
1114         }
1115 
1116         return staticFieldOffset0(f);
1117     }
1118 
1119     /**
1120      * Reports the location of a given static field, in conjunction with {@link
1121      * #staticFieldOffset}.
1122      * <p>Fetch the base "Object", if any, with which static fields of the
1123      * given class can be accessed via methods like {@link #getInt(Object,
1124      * long)}.  This value may be null.  This value may refer to an object
1125      * which is a "cookie", not guaranteed to be a real Object, and it should
1126      * not be used in any way except as argument to the get and put routines in
1127      * this class.
1128      */
1129     public Object staticFieldBase(Field f) {
1130         if (f == null) {
1131             throw new NullPointerException();
1132         }
1133 
1134         return staticFieldBase0(f);
1135     }
1136 
1137     /**
1138      * Detects if the given class may need to be initialized. This is often
1139      * needed in conjunction with obtaining the static field base of a
1140      * class.
1141      * @return false only if a call to {@code ensureClassInitialized} would have no effect
1142      */
1143     public boolean shouldBeInitialized(Class<?> c) {
1144         if (c == null) {
1145             throw new NullPointerException();
1146         }
1147 
1148         return shouldBeInitialized0(c);
1149     }
1150 
1151     /**
1152      * Ensures the given class has been initialized (see JVMS-5.5 for details).
1153      * This is often needed in conjunction with obtaining the static field base
1154      * of a class.
1155      *
1156      * The call returns when either class {@code c} is fully initialized or
1157      * class {@code c} is being initialized and the call is performed from
1158      * the initializing thread. In the latter case a subsequent call to
1159      * {@link #shouldBeInitialized} will return {@code true}.
1160      */
1161     public void ensureClassInitialized(Class<?> c) {
1162         if (c == null) {
1163             throw new NullPointerException();
1164         }
1165 
1166         ensureClassInitialized0(c);
1167     }
1168 
1169     /**
1170      * Reports the offset of the first element in the storage allocation of a
1171      * given array class.  If {@link #arrayIndexScale} returns a non-zero value
1172      * for the same class, you may use that scale factor, together with this
1173      * base offset, to form new offsets to access elements of arrays of the
1174      * given class.
1175      *
1176      * @see #getInt(Object, long)
1177      * @see #putInt(Object, long, int)
1178      */
1179     public int arrayBaseOffset(Class<?> arrayClass) {
1180         if (arrayClass == null) {
1181             throw new NullPointerException();
1182         }
1183 
1184         return arrayBaseOffset0(arrayClass);
1185     }
1186 
1187 
1188     /** The value of {@code arrayBaseOffset(boolean[].class)} */
1189     public static final int ARRAY_BOOLEAN_BASE_OFFSET
1190             = theUnsafe.arrayBaseOffset(boolean[].class);
1191 
1192     /** The value of {@code arrayBaseOffset(byte[].class)} */
1193     public static final int ARRAY_BYTE_BASE_OFFSET
1194             = theUnsafe.arrayBaseOffset(byte[].class);
1195 
1196     /** The value of {@code arrayBaseOffset(short[].class)} */
1197     public static final int ARRAY_SHORT_BASE_OFFSET
1198             = theUnsafe.arrayBaseOffset(short[].class);
1199 
1200     /** The value of {@code arrayBaseOffset(char[].class)} */
1201     public static final int ARRAY_CHAR_BASE_OFFSET
1202             = theUnsafe.arrayBaseOffset(char[].class);
1203 
1204     /** The value of {@code arrayBaseOffset(int[].class)} */
1205     public static final int ARRAY_INT_BASE_OFFSET
1206             = theUnsafe.arrayBaseOffset(int[].class);
1207 
1208     /** The value of {@code arrayBaseOffset(long[].class)} */
1209     public static final int ARRAY_LONG_BASE_OFFSET
1210             = theUnsafe.arrayBaseOffset(long[].class);
1211 
1212     /** The value of {@code arrayBaseOffset(float[].class)} */
1213     public static final int ARRAY_FLOAT_BASE_OFFSET
1214             = theUnsafe.arrayBaseOffset(float[].class);
1215 
1216     /** The value of {@code arrayBaseOffset(double[].class)} */
1217     public static final int ARRAY_DOUBLE_BASE_OFFSET
1218             = theUnsafe.arrayBaseOffset(double[].class);
1219 
1220     /** The value of {@code arrayBaseOffset(Object[].class)} */
1221     public static final int ARRAY_OBJECT_BASE_OFFSET
1222             = theUnsafe.arrayBaseOffset(Object[].class);
1223 
1224     /**
1225      * Reports the scale factor for addressing elements in the storage
1226      * allocation of a given array class.  However, arrays of "narrow" types
1227      * will generally not work properly with accessors like {@link
1228      * #getByte(Object, long)}, so the scale factor for such classes is reported
1229      * as zero.
1230      *
1231      * @see #arrayBaseOffset
1232      * @see #getInt(Object, long)
1233      * @see #putInt(Object, long, int)
1234      */
1235     public int arrayIndexScale(Class<?> arrayClass) {
1236         if (arrayClass == null) {
1237             throw new NullPointerException();
1238         }
1239 
1240         return arrayIndexScale0(arrayClass);
1241     }
1242 
1243 
1244     /** The value of {@code arrayIndexScale(boolean[].class)} */
1245     public static final int ARRAY_BOOLEAN_INDEX_SCALE
1246             = theUnsafe.arrayIndexScale(boolean[].class);
1247 
1248     /** The value of {@code arrayIndexScale(byte[].class)} */
1249     public static final int ARRAY_BYTE_INDEX_SCALE
1250             = theUnsafe.arrayIndexScale(byte[].class);
1251 
1252     /** The value of {@code arrayIndexScale(short[].class)} */
1253     public static final int ARRAY_SHORT_INDEX_SCALE
1254             = theUnsafe.arrayIndexScale(short[].class);
1255 
1256     /** The value of {@code arrayIndexScale(char[].class)} */
1257     public static final int ARRAY_CHAR_INDEX_SCALE
1258             = theUnsafe.arrayIndexScale(char[].class);
1259 
1260     /** The value of {@code arrayIndexScale(int[].class)} */
1261     public static final int ARRAY_INT_INDEX_SCALE
1262             = theUnsafe.arrayIndexScale(int[].class);
1263 
1264     /** The value of {@code arrayIndexScale(long[].class)} */
1265     public static final int ARRAY_LONG_INDEX_SCALE
1266             = theUnsafe.arrayIndexScale(long[].class);
1267 
1268     /** The value of {@code arrayIndexScale(float[].class)} */
1269     public static final int ARRAY_FLOAT_INDEX_SCALE
1270             = theUnsafe.arrayIndexScale(float[].class);
1271 
1272     /** The value of {@code arrayIndexScale(double[].class)} */
1273     public static final int ARRAY_DOUBLE_INDEX_SCALE
1274             = theUnsafe.arrayIndexScale(double[].class);
1275 
1276     /** The value of {@code arrayIndexScale(Object[].class)} */
1277     public static final int ARRAY_OBJECT_INDEX_SCALE
1278             = theUnsafe.arrayIndexScale(Object[].class);
1279 
1280     /**
1281      * Reports the size in bytes of a native pointer, as stored via {@link
1282      * #putAddress}.  This value will be either 4 or 8.  Note that the sizes of
1283      * other primitive types (as stored in native memory blocks) is determined
1284      * fully by their information content.
1285      */
1286     public int addressSize() {
1287         return ADDRESS_SIZE;
1288     }
1289 
1290     /** The value of {@code addressSize()} */
1291     public static final int ADDRESS_SIZE = ADDRESS_SIZE0;
1292 
1293     /**
1294      * Reports the size in bytes of a native memory page (whatever that is).
1295      * This value will always be a power of two.
1296      */
1297     public int pageSize() { return PAGE_SIZE; }
1298 
1299     /**
1300      * Reports the size in bytes of a data cache line written back by
1301      * the hardware cache line flush operation available to the JVM or
1302      * 0 if data cache line flushing is not enabled.
1303      */
1304     public int dataCacheLineFlushSize() { return DATA_CACHE_LINE_FLUSH_SIZE; }
1305 
1306     /**
1307      * Rounds down address to a data cache line boundary as
1308      * determined by {@link #dataCacheLineFlushSize}
1309      * @return the rounded down address
1310      */
1311     public long dataCacheLineAlignDown(long address) {
1312         return (address & ~(DATA_CACHE_LINE_FLUSH_SIZE - 1));
1313     }
1314 
1315     /**
1316      * Returns true if data cache line writeback
1317      */
1318     public static boolean isWritebackEnabled() { return DATA_CACHE_LINE_FLUSH_SIZE != 0; }
1319 
1320     //--- random trusted operations from JNI:
1321 
1322     /**
1323      * Tells the VM to define a class, without security checks.  By default, the
1324      * class loader and protection domain come from the caller's class.
1325      */
1326     public Class<?> defineClass(String name, byte[] b, int off, int len,
1327                                 ClassLoader loader,
1328                                 ProtectionDomain protectionDomain) {
1329         if (b == null) {
1330             throw new NullPointerException();
1331         }
1332         if (len < 0) {
1333             throw new ArrayIndexOutOfBoundsException();
1334         }
1335 
1336         return defineClass0(name, b, off, len, loader, protectionDomain);
1337     }
1338 
1339     public native Class<?> defineClass0(String name, byte[] b, int off, int len,
1340                                         ClassLoader loader,
1341                                         ProtectionDomain protectionDomain);
1342 
1343     /**
1344      * Allocates an instance but does not run any constructor.
1345      * Initializes the class if it has not yet been.
1346      */
1347     @IntrinsicCandidate
1348     public native Object allocateInstance(Class<?> cls)
1349         throws InstantiationException;
1350 
1351     /**
1352      * Allocates an array of a given type, but does not do zeroing.
1353      * <p>
1354      * This method should only be used in the very rare cases where a high-performance code
1355      * overwrites the destination array completely, and compilers cannot assist in zeroing elimination.
1356      * In an overwhelming majority of cases, a normal Java allocation should be used instead.
1357      * <p>
1358      * Users of this method are <b>required</b> to overwrite the initial (garbage) array contents
1359      * before allowing untrusted code, or code in other threads, to observe the reference
1360      * to the newly allocated array. In addition, the publication of the array reference must be
1361      * safe according to the Java Memory Model requirements.
1362      * <p>
1363      * The safest approach to deal with an uninitialized array is to keep the reference to it in local
1364      * variable at least until the initialization is complete, and then publish it <b>once</b>, either
1365      * by writing it to a <em>volatile</em> field, or storing it into a <em>final</em> field in constructor,
1366      * or issuing a {@link #storeFence} before publishing the reference.
1367      * <p>
1368      * @implnote This method can only allocate primitive arrays, to avoid garbage reference
1369      * elements that could break heap integrity.
1370      *
1371      * @param componentType array component type to allocate
1372      * @param length array size to allocate
1373      * @throws IllegalArgumentException if component type is null, or not a primitive class;
1374      *                                  or the length is negative
1375      */
1376     public Object allocateUninitializedArray(Class<?> componentType, int length) {
1377        if (componentType == null) {
1378            throw new IllegalArgumentException("Component type is null");
1379        }
1380        if (!componentType.isPrimitive()) {
1381            throw new IllegalArgumentException("Component type is not primitive");
1382        }
1383        if (length < 0) {
1384            throw new IllegalArgumentException("Negative length");
1385        }
1386        return allocateUninitializedArray0(componentType, length);
1387     }
1388 
1389     @IntrinsicCandidate
1390     private Object allocateUninitializedArray0(Class<?> componentType, int length) {
1391        // These fallbacks provide zeroed arrays, but intrinsic is not required to
1392        // return the zeroed arrays.
1393        if (componentType == byte.class)    return new byte[length];
1394        if (componentType == boolean.class) return new boolean[length];
1395        if (componentType == short.class)   return new short[length];
1396        if (componentType == char.class)    return new char[length];
1397        if (componentType == int.class)     return new int[length];
1398        if (componentType == float.class)   return new float[length];
1399        if (componentType == long.class)    return new long[length];
1400        if (componentType == double.class)  return new double[length];
1401        return null;
1402     }
1403 
1404     /** Throws the exception without telling the verifier. */
1405     public native void throwException(Throwable ee);
1406 
1407     /**
1408      * Atomically updates Java variable to {@code x} if it is currently
1409      * holding {@code expected}.
1410      *
1411      * <p>This operation has memory semantics of a {@code volatile} read
1412      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1413      *
1414      * @return {@code true} if successful
1415      */
1416     @IntrinsicCandidate
1417     public final native boolean compareAndSetReference(Object o, long offset,
1418                                                        Object expected,
1419                                                        Object x);
1420 
1421     @IntrinsicCandidate
1422     public final native Object compareAndExchangeReference(Object o, long offset,
1423                                                            Object expected,
1424                                                            Object x);
1425 
1426     @IntrinsicCandidate
1427     public final Object compareAndExchangeReferenceAcquire(Object o, long offset,
1428                                                            Object expected,
1429                                                            Object x) {
1430         return compareAndExchangeReference(o, offset, expected, x);
1431     }
1432 
1433     @IntrinsicCandidate
1434     public final Object compareAndExchangeReferenceRelease(Object o, long offset,
1435                                                            Object expected,
1436                                                            Object x) {
1437         return compareAndExchangeReference(o, offset, expected, x);
1438     }
1439 
1440     @IntrinsicCandidate
1441     public final boolean weakCompareAndSetReferencePlain(Object o, long offset,
1442                                                          Object expected,
1443                                                          Object x) {
1444         return compareAndSetReference(o, offset, expected, x);
1445     }
1446 
1447     @IntrinsicCandidate
1448     public final boolean weakCompareAndSetReferenceAcquire(Object o, long offset,
1449                                                            Object expected,
1450                                                            Object x) {
1451         return compareAndSetReference(o, offset, expected, x);
1452     }
1453 
1454     @IntrinsicCandidate
1455     public final boolean weakCompareAndSetReferenceRelease(Object o, long offset,
1456                                                            Object expected,
1457                                                            Object x) {
1458         return compareAndSetReference(o, offset, expected, x);
1459     }
1460 
1461     @IntrinsicCandidate
1462     public final boolean weakCompareAndSetReference(Object o, long offset,
1463                                                     Object expected,
1464                                                     Object x) {
1465         return compareAndSetReference(o, offset, expected, x);
1466     }
1467 
1468     /**
1469      * Atomically updates Java variable to {@code x} if it is currently
1470      * holding {@code expected}.
1471      *
1472      * <p>This operation has memory semantics of a {@code volatile} read
1473      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1474      *
1475      * @return {@code true} if successful
1476      */
1477     @IntrinsicCandidate
1478     public final native boolean compareAndSetInt(Object o, long offset,
1479                                                  int expected,
1480                                                  int x);
1481 
1482     @IntrinsicCandidate
1483     public final native int compareAndExchangeInt(Object o, long offset,
1484                                                   int expected,
1485                                                   int x);
1486 
1487     @IntrinsicCandidate
1488     public final int compareAndExchangeIntAcquire(Object o, long offset,
1489                                                          int expected,
1490                                                          int x) {
1491         return compareAndExchangeInt(o, offset, expected, x);
1492     }
1493 
1494     @IntrinsicCandidate
1495     public final int compareAndExchangeIntRelease(Object o, long offset,
1496                                                          int expected,
1497                                                          int x) {
1498         return compareAndExchangeInt(o, offset, expected, x);
1499     }
1500 
1501     @IntrinsicCandidate
1502     public final boolean weakCompareAndSetIntPlain(Object o, long offset,
1503                                                    int expected,
1504                                                    int x) {
1505         return compareAndSetInt(o, offset, expected, x);
1506     }
1507 
1508     @IntrinsicCandidate
1509     public final boolean weakCompareAndSetIntAcquire(Object o, long offset,
1510                                                      int expected,
1511                                                      int x) {
1512         return compareAndSetInt(o, offset, expected, x);
1513     }
1514 
1515     @IntrinsicCandidate
1516     public final boolean weakCompareAndSetIntRelease(Object o, long offset,
1517                                                      int expected,
1518                                                      int x) {
1519         return compareAndSetInt(o, offset, expected, x);
1520     }
1521 
1522     @IntrinsicCandidate
1523     public final boolean weakCompareAndSetInt(Object o, long offset,
1524                                               int expected,
1525                                               int x) {
1526         return compareAndSetInt(o, offset, expected, x);
1527     }
1528 
1529     @IntrinsicCandidate
1530     public final byte compareAndExchangeByte(Object o, long offset,
1531                                              byte expected,
1532                                              byte x) {
1533         long wordOffset = offset & ~3;
1534         int shift = (int) (offset & 3) << 3;
1535         if (BIG_ENDIAN) {
1536             shift = 24 - shift;
1537         }
1538         int mask           = 0xFF << shift;
1539         int maskedExpected = (expected & 0xFF) << shift;
1540         int maskedX        = (x & 0xFF) << shift;
1541         int fullWord;
1542         do {
1543             fullWord = getIntVolatile(o, wordOffset);
1544             if ((fullWord & mask) != maskedExpected)
1545                 return (byte) ((fullWord & mask) >> shift);
1546         } while (!weakCompareAndSetInt(o, wordOffset,
1547                                                 fullWord, (fullWord & ~mask) | maskedX));
1548         return expected;
1549     }
1550 
1551     @IntrinsicCandidate
1552     public final boolean compareAndSetByte(Object o, long offset,
1553                                            byte expected,
1554                                            byte x) {
1555         return compareAndExchangeByte(o, offset, expected, x) == expected;
1556     }
1557 
1558     @IntrinsicCandidate
1559     public final boolean weakCompareAndSetByte(Object o, long offset,
1560                                                byte expected,
1561                                                byte x) {
1562         return compareAndSetByte(o, offset, expected, x);
1563     }
1564 
1565     @IntrinsicCandidate
1566     public final boolean weakCompareAndSetByteAcquire(Object o, long offset,
1567                                                       byte expected,
1568                                                       byte x) {
1569         return weakCompareAndSetByte(o, offset, expected, x);
1570     }
1571 
1572     @IntrinsicCandidate
1573     public final boolean weakCompareAndSetByteRelease(Object o, long offset,
1574                                                       byte expected,
1575                                                       byte x) {
1576         return weakCompareAndSetByte(o, offset, expected, x);
1577     }
1578 
1579     @IntrinsicCandidate
1580     public final boolean weakCompareAndSetBytePlain(Object o, long offset,
1581                                                     byte expected,
1582                                                     byte x) {
1583         return weakCompareAndSetByte(o, offset, expected, x);
1584     }
1585 
1586     @IntrinsicCandidate
1587     public final byte compareAndExchangeByteAcquire(Object o, long offset,
1588                                                     byte expected,
1589                                                     byte x) {
1590         return compareAndExchangeByte(o, offset, expected, x);
1591     }
1592 
1593     @IntrinsicCandidate
1594     public final byte compareAndExchangeByteRelease(Object o, long offset,
1595                                                     byte expected,
1596                                                     byte x) {
1597         return compareAndExchangeByte(o, offset, expected, x);
1598     }
1599 
1600     @IntrinsicCandidate
1601     public final short compareAndExchangeShort(Object o, long offset,
1602                                                short expected,
1603                                                short x) {
1604         if ((offset & 3) == 3) {
1605             throw new IllegalArgumentException("Update spans the word, not supported");
1606         }
1607         long wordOffset = offset & ~3;
1608         int shift = (int) (offset & 3) << 3;
1609         if (BIG_ENDIAN) {
1610             shift = 16 - shift;
1611         }
1612         int mask           = 0xFFFF << shift;
1613         int maskedExpected = (expected & 0xFFFF) << shift;
1614         int maskedX        = (x & 0xFFFF) << shift;
1615         int fullWord;
1616         do {
1617             fullWord = getIntVolatile(o, wordOffset);
1618             if ((fullWord & mask) != maskedExpected) {
1619                 return (short) ((fullWord & mask) >> shift);
1620             }
1621         } while (!weakCompareAndSetInt(o, wordOffset,
1622                                                 fullWord, (fullWord & ~mask) | maskedX));
1623         return expected;
1624     }
1625 
1626     @IntrinsicCandidate
1627     public final boolean compareAndSetShort(Object o, long offset,
1628                                             short expected,
1629                                             short x) {
1630         return compareAndExchangeShort(o, offset, expected, x) == expected;
1631     }
1632 
1633     @IntrinsicCandidate
1634     public final boolean weakCompareAndSetShort(Object o, long offset,
1635                                                 short expected,
1636                                                 short x) {
1637         return compareAndSetShort(o, offset, expected, x);
1638     }
1639 
1640     @IntrinsicCandidate
1641     public final boolean weakCompareAndSetShortAcquire(Object o, long offset,
1642                                                        short expected,
1643                                                        short x) {
1644         return weakCompareAndSetShort(o, offset, expected, x);
1645     }
1646 
1647     @IntrinsicCandidate
1648     public final boolean weakCompareAndSetShortRelease(Object o, long offset,
1649                                                        short expected,
1650                                                        short x) {
1651         return weakCompareAndSetShort(o, offset, expected, x);
1652     }
1653 
1654     @IntrinsicCandidate
1655     public final boolean weakCompareAndSetShortPlain(Object o, long offset,
1656                                                      short expected,
1657                                                      short x) {
1658         return weakCompareAndSetShort(o, offset, expected, x);
1659     }
1660 
1661 
1662     @IntrinsicCandidate
1663     public final short compareAndExchangeShortAcquire(Object o, long offset,
1664                                                      short expected,
1665                                                      short x) {
1666         return compareAndExchangeShort(o, offset, expected, x);
1667     }
1668 
1669     @IntrinsicCandidate
1670     public final short compareAndExchangeShortRelease(Object o, long offset,
1671                                                     short expected,
1672                                                     short x) {
1673         return compareAndExchangeShort(o, offset, expected, x);
1674     }
1675 
1676     @ForceInline
1677     private char s2c(short s) {
1678         return (char) s;
1679     }
1680 
1681     @ForceInline
1682     private short c2s(char s) {
1683         return (short) s;
1684     }
1685 
1686     @ForceInline
1687     public final boolean compareAndSetChar(Object o, long offset,
1688                                            char expected,
1689                                            char x) {
1690         return compareAndSetShort(o, offset, c2s(expected), c2s(x));
1691     }
1692 
1693     @ForceInline
1694     public final char compareAndExchangeChar(Object o, long offset,
1695                                              char expected,
1696                                              char x) {
1697         return s2c(compareAndExchangeShort(o, offset, c2s(expected), c2s(x)));
1698     }
1699 
1700     @ForceInline
1701     public final char compareAndExchangeCharAcquire(Object o, long offset,
1702                                             char expected,
1703                                             char x) {
1704         return s2c(compareAndExchangeShortAcquire(o, offset, c2s(expected), c2s(x)));
1705     }
1706 
1707     @ForceInline
1708     public final char compareAndExchangeCharRelease(Object o, long offset,
1709                                             char expected,
1710                                             char x) {
1711         return s2c(compareAndExchangeShortRelease(o, offset, c2s(expected), c2s(x)));
1712     }
1713 
1714     @ForceInline
1715     public final boolean weakCompareAndSetChar(Object o, long offset,
1716                                                char expected,
1717                                                char x) {
1718         return weakCompareAndSetShort(o, offset, c2s(expected), c2s(x));
1719     }
1720 
1721     @ForceInline
1722     public final boolean weakCompareAndSetCharAcquire(Object o, long offset,
1723                                                       char expected,
1724                                                       char x) {
1725         return weakCompareAndSetShortAcquire(o, offset, c2s(expected), c2s(x));
1726     }
1727 
1728     @ForceInline
1729     public final boolean weakCompareAndSetCharRelease(Object o, long offset,
1730                                                       char expected,
1731                                                       char x) {
1732         return weakCompareAndSetShortRelease(o, offset, c2s(expected), c2s(x));
1733     }
1734 
1735     @ForceInline
1736     public final boolean weakCompareAndSetCharPlain(Object o, long offset,
1737                                                     char expected,
1738                                                     char x) {
1739         return weakCompareAndSetShortPlain(o, offset, c2s(expected), c2s(x));
1740     }
1741 
1742     /**
1743      * The JVM converts integral values to boolean values using two
1744      * different conventions, byte testing against zero and truncation
1745      * to least-significant bit.
1746      *
1747      * <p>The JNI documents specify that, at least for returning
1748      * values from native methods, a Java boolean value is converted
1749      * to the value-set 0..1 by first truncating to a byte (0..255 or
1750      * maybe -128..127) and then testing against zero. Thus, Java
1751      * booleans in non-Java data structures are by convention
1752      * represented as 8-bit containers containing either zero (for
1753      * false) or any non-zero value (for true).
1754      *
1755      * <p>Java booleans in the heap are also stored in bytes, but are
1756      * strongly normalized to the value-set 0..1 (i.e., they are
1757      * truncated to the least-significant bit).
1758      *
1759      * <p>The main reason for having different conventions for
1760      * conversion is performance: Truncation to the least-significant
1761      * bit can be usually implemented with fewer (machine)
1762      * instructions than byte testing against zero.
1763      *
1764      * <p>A number of Unsafe methods load boolean values from the heap
1765      * as bytes. Unsafe converts those values according to the JNI
1766      * rules (i.e, using the "testing against zero" convention). The
1767      * method {@code byte2bool} implements that conversion.
1768      *
1769      * @param b the byte to be converted to boolean
1770      * @return the result of the conversion
1771      */
1772     @ForceInline
1773     private boolean byte2bool(byte b) {
1774         return b != 0;
1775     }
1776 
1777     /**
1778      * Convert a boolean value to a byte. The return value is strongly
1779      * normalized to the value-set 0..1 (i.e., the value is truncated
1780      * to the least-significant bit). See {@link #byte2bool(byte)} for
1781      * more details on conversion conventions.
1782      *
1783      * @param b the boolean to be converted to byte (and then normalized)
1784      * @return the result of the conversion
1785      */
1786     @ForceInline
1787     private byte bool2byte(boolean b) {
1788         return b ? (byte)1 : (byte)0;
1789     }
1790 
1791     @ForceInline
1792     public final boolean compareAndSetBoolean(Object o, long offset,
1793                                               boolean expected,
1794                                               boolean x) {
1795         return compareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
1796     }
1797 
1798     @ForceInline
1799     public final boolean compareAndExchangeBoolean(Object o, long offset,
1800                                                    boolean expected,
1801                                                    boolean x) {
1802         return byte2bool(compareAndExchangeByte(o, offset, bool2byte(expected), bool2byte(x)));
1803     }
1804 
1805     @ForceInline
1806     public final boolean compareAndExchangeBooleanAcquire(Object o, long offset,
1807                                                     boolean expected,
1808                                                     boolean x) {
1809         return byte2bool(compareAndExchangeByteAcquire(o, offset, bool2byte(expected), bool2byte(x)));
1810     }
1811 
1812     @ForceInline
1813     public final boolean compareAndExchangeBooleanRelease(Object o, long offset,
1814                                                        boolean expected,
1815                                                        boolean x) {
1816         return byte2bool(compareAndExchangeByteRelease(o, offset, bool2byte(expected), bool2byte(x)));
1817     }
1818 
1819     @ForceInline
1820     public final boolean weakCompareAndSetBoolean(Object o, long offset,
1821                                                   boolean expected,
1822                                                   boolean x) {
1823         return weakCompareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
1824     }
1825 
1826     @ForceInline
1827     public final boolean weakCompareAndSetBooleanAcquire(Object o, long offset,
1828                                                          boolean expected,
1829                                                          boolean x) {
1830         return weakCompareAndSetByteAcquire(o, offset, bool2byte(expected), bool2byte(x));
1831     }
1832 
1833     @ForceInline
1834     public final boolean weakCompareAndSetBooleanRelease(Object o, long offset,
1835                                                          boolean expected,
1836                                                          boolean x) {
1837         return weakCompareAndSetByteRelease(o, offset, bool2byte(expected), bool2byte(x));
1838     }
1839 
1840     @ForceInline
1841     public final boolean weakCompareAndSetBooleanPlain(Object o, long offset,
1842                                                        boolean expected,
1843                                                        boolean x) {
1844         return weakCompareAndSetBytePlain(o, offset, bool2byte(expected), bool2byte(x));
1845     }
1846 
1847     /**
1848      * Atomically updates Java variable to {@code x} if it is currently
1849      * holding {@code expected}.
1850      *
1851      * <p>This operation has memory semantics of a {@code volatile} read
1852      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1853      *
1854      * @return {@code true} if successful
1855      */
1856     @ForceInline
1857     public final boolean compareAndSetFloat(Object o, long offset,
1858                                             float expected,
1859                                             float x) {
1860         return compareAndSetInt(o, offset,
1861                                  Float.floatToRawIntBits(expected),
1862                                  Float.floatToRawIntBits(x));
1863     }
1864 
1865     @ForceInline
1866     public final float compareAndExchangeFloat(Object o, long offset,
1867                                                float expected,
1868                                                float x) {
1869         int w = compareAndExchangeInt(o, offset,
1870                                       Float.floatToRawIntBits(expected),
1871                                       Float.floatToRawIntBits(x));
1872         return Float.intBitsToFloat(w);
1873     }
1874 
1875     @ForceInline
1876     public final float compareAndExchangeFloatAcquire(Object o, long offset,
1877                                                   float expected,
1878                                                   float x) {
1879         int w = compareAndExchangeIntAcquire(o, offset,
1880                                              Float.floatToRawIntBits(expected),
1881                                              Float.floatToRawIntBits(x));
1882         return Float.intBitsToFloat(w);
1883     }
1884 
1885     @ForceInline
1886     public final float compareAndExchangeFloatRelease(Object o, long offset,
1887                                                   float expected,
1888                                                   float x) {
1889         int w = compareAndExchangeIntRelease(o, offset,
1890                                              Float.floatToRawIntBits(expected),
1891                                              Float.floatToRawIntBits(x));
1892         return Float.intBitsToFloat(w);
1893     }
1894 
1895     @ForceInline
1896     public final boolean weakCompareAndSetFloatPlain(Object o, long offset,
1897                                                      float expected,
1898                                                      float x) {
1899         return weakCompareAndSetIntPlain(o, offset,
1900                                      Float.floatToRawIntBits(expected),
1901                                      Float.floatToRawIntBits(x));
1902     }
1903 
1904     @ForceInline
1905     public final boolean weakCompareAndSetFloatAcquire(Object o, long offset,
1906                                                        float expected,
1907                                                        float x) {
1908         return weakCompareAndSetIntAcquire(o, offset,
1909                                             Float.floatToRawIntBits(expected),
1910                                             Float.floatToRawIntBits(x));
1911     }
1912 
1913     @ForceInline
1914     public final boolean weakCompareAndSetFloatRelease(Object o, long offset,
1915                                                        float expected,
1916                                                        float x) {
1917         return weakCompareAndSetIntRelease(o, offset,
1918                                             Float.floatToRawIntBits(expected),
1919                                             Float.floatToRawIntBits(x));
1920     }
1921 
1922     @ForceInline
1923     public final boolean weakCompareAndSetFloat(Object o, long offset,
1924                                                 float expected,
1925                                                 float x) {
1926         return weakCompareAndSetInt(o, offset,
1927                                              Float.floatToRawIntBits(expected),
1928                                              Float.floatToRawIntBits(x));
1929     }
1930 
1931     /**
1932      * Atomically updates Java variable to {@code x} if it is currently
1933      * holding {@code expected}.
1934      *
1935      * <p>This operation has memory semantics of a {@code volatile} read
1936      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1937      *
1938      * @return {@code true} if successful
1939      */
1940     @ForceInline
1941     public final boolean compareAndSetDouble(Object o, long offset,
1942                                              double expected,
1943                                              double x) {
1944         return compareAndSetLong(o, offset,
1945                                  Double.doubleToRawLongBits(expected),
1946                                  Double.doubleToRawLongBits(x));
1947     }
1948 
1949     @ForceInline
1950     public final double compareAndExchangeDouble(Object o, long offset,
1951                                                  double expected,
1952                                                  double x) {
1953         long w = compareAndExchangeLong(o, offset,
1954                                         Double.doubleToRawLongBits(expected),
1955                                         Double.doubleToRawLongBits(x));
1956         return Double.longBitsToDouble(w);
1957     }
1958 
1959     @ForceInline
1960     public final double compareAndExchangeDoubleAcquire(Object o, long offset,
1961                                                         double expected,
1962                                                         double x) {
1963         long w = compareAndExchangeLongAcquire(o, offset,
1964                                                Double.doubleToRawLongBits(expected),
1965                                                Double.doubleToRawLongBits(x));
1966         return Double.longBitsToDouble(w);
1967     }
1968 
1969     @ForceInline
1970     public final double compareAndExchangeDoubleRelease(Object o, long offset,
1971                                                         double expected,
1972                                                         double x) {
1973         long w = compareAndExchangeLongRelease(o, offset,
1974                                                Double.doubleToRawLongBits(expected),
1975                                                Double.doubleToRawLongBits(x));
1976         return Double.longBitsToDouble(w);
1977     }
1978 
1979     @ForceInline
1980     public final boolean weakCompareAndSetDoublePlain(Object o, long offset,
1981                                                       double expected,
1982                                                       double x) {
1983         return weakCompareAndSetLongPlain(o, offset,
1984                                      Double.doubleToRawLongBits(expected),
1985                                      Double.doubleToRawLongBits(x));
1986     }
1987 
1988     @ForceInline
1989     public final boolean weakCompareAndSetDoubleAcquire(Object o, long offset,
1990                                                         double expected,
1991                                                         double x) {
1992         return weakCompareAndSetLongAcquire(o, offset,
1993                                              Double.doubleToRawLongBits(expected),
1994                                              Double.doubleToRawLongBits(x));
1995     }
1996 
1997     @ForceInline
1998     public final boolean weakCompareAndSetDoubleRelease(Object o, long offset,
1999                                                         double expected,
2000                                                         double x) {
2001         return weakCompareAndSetLongRelease(o, offset,
2002                                              Double.doubleToRawLongBits(expected),
2003                                              Double.doubleToRawLongBits(x));
2004     }
2005 
2006     @ForceInline
2007     public final boolean weakCompareAndSetDouble(Object o, long offset,
2008                                                  double expected,
2009                                                  double x) {
2010         return weakCompareAndSetLong(o, offset,
2011                                               Double.doubleToRawLongBits(expected),
2012                                               Double.doubleToRawLongBits(x));
2013     }
2014 
2015     /**
2016      * Atomically updates Java variable to {@code x} if it is currently
2017      * holding {@code expected}.
2018      *
2019      * <p>This operation has memory semantics of a {@code volatile} read
2020      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2021      *
2022      * @return {@code true} if successful
2023      */
2024     @IntrinsicCandidate
2025     public final native boolean compareAndSetLong(Object o, long offset,
2026                                                   long expected,
2027                                                   long x);
2028 
2029     @IntrinsicCandidate
2030     public final native long compareAndExchangeLong(Object o, long offset,
2031                                                     long expected,
2032                                                     long x);
2033 
2034     @IntrinsicCandidate
2035     public final long compareAndExchangeLongAcquire(Object o, long offset,
2036                                                            long expected,
2037                                                            long x) {
2038         return compareAndExchangeLong(o, offset, expected, x);
2039     }
2040 
2041     @IntrinsicCandidate
2042     public final long compareAndExchangeLongRelease(Object o, long offset,
2043                                                            long expected,
2044                                                            long x) {
2045         return compareAndExchangeLong(o, offset, expected, x);
2046     }
2047 
2048     @IntrinsicCandidate
2049     public final boolean weakCompareAndSetLongPlain(Object o, long offset,
2050                                                     long expected,
2051                                                     long x) {
2052         return compareAndSetLong(o, offset, expected, x);
2053     }
2054 
2055     @IntrinsicCandidate
2056     public final boolean weakCompareAndSetLongAcquire(Object o, long offset,
2057                                                       long expected,
2058                                                       long x) {
2059         return compareAndSetLong(o, offset, expected, x);
2060     }
2061 
2062     @IntrinsicCandidate
2063     public final boolean weakCompareAndSetLongRelease(Object o, long offset,
2064                                                       long expected,
2065                                                       long x) {
2066         return compareAndSetLong(o, offset, expected, x);
2067     }
2068 
2069     @IntrinsicCandidate
2070     public final boolean weakCompareAndSetLong(Object o, long offset,
2071                                                long expected,
2072                                                long x) {
2073         return compareAndSetLong(o, offset, expected, x);
2074     }
2075 
2076     /**
2077      * Fetches a reference value from a given Java variable, with volatile
2078      * load semantics. Otherwise identical to {@link #getReference(Object, long)}
2079      */
2080     @IntrinsicCandidate
2081     public native Object getReferenceVolatile(Object o, long offset);
2082 
2083     /**
2084      * Stores a reference value into a given Java variable, with
2085      * volatile store semantics. Otherwise identical to {@link #putReference(Object, long, Object)}
2086      */
2087     @IntrinsicCandidate
2088     public native void putReferenceVolatile(Object o, long offset, Object x);
2089 
2090     /** Volatile version of {@link #getInt(Object, long)}  */
2091     @IntrinsicCandidate
2092     public native int     getIntVolatile(Object o, long offset);
2093 
2094     /** Volatile version of {@link #putInt(Object, long, int)}  */
2095     @IntrinsicCandidate
2096     public native void    putIntVolatile(Object o, long offset, int x);
2097 
2098     /** Volatile version of {@link #getBoolean(Object, long)}  */
2099     @IntrinsicCandidate
2100     public native boolean getBooleanVolatile(Object o, long offset);
2101 
2102     /** Volatile version of {@link #putBoolean(Object, long, boolean)}  */
2103     @IntrinsicCandidate
2104     public native void    putBooleanVolatile(Object o, long offset, boolean x);
2105 
2106     /** Volatile version of {@link #getByte(Object, long)}  */
2107     @IntrinsicCandidate
2108     public native byte    getByteVolatile(Object o, long offset);
2109 
2110     /** Volatile version of {@link #putByte(Object, long, byte)}  */
2111     @IntrinsicCandidate
2112     public native void    putByteVolatile(Object o, long offset, byte x);
2113 
2114     /** Volatile version of {@link #getShort(Object, long)}  */
2115     @IntrinsicCandidate
2116     public native short   getShortVolatile(Object o, long offset);
2117 
2118     /** Volatile version of {@link #putShort(Object, long, short)}  */
2119     @IntrinsicCandidate
2120     public native void    putShortVolatile(Object o, long offset, short x);
2121 
2122     /** Volatile version of {@link #getChar(Object, long)}  */
2123     @IntrinsicCandidate
2124     public native char    getCharVolatile(Object o, long offset);
2125 
2126     /** Volatile version of {@link #putChar(Object, long, char)}  */
2127     @IntrinsicCandidate
2128     public native void    putCharVolatile(Object o, long offset, char x);
2129 
2130     /** Volatile version of {@link #getLong(Object, long)}  */
2131     @IntrinsicCandidate
2132     public native long    getLongVolatile(Object o, long offset);
2133 
2134     /** Volatile version of {@link #putLong(Object, long, long)}  */
2135     @IntrinsicCandidate
2136     public native void    putLongVolatile(Object o, long offset, long x);
2137 
2138     /** Volatile version of {@link #getFloat(Object, long)}  */
2139     @IntrinsicCandidate
2140     public native float   getFloatVolatile(Object o, long offset);
2141 
2142     /** Volatile version of {@link #putFloat(Object, long, float)}  */
2143     @IntrinsicCandidate
2144     public native void    putFloatVolatile(Object o, long offset, float x);
2145 
2146     /** Volatile version of {@link #getDouble(Object, long)}  */
2147     @IntrinsicCandidate
2148     public native double  getDoubleVolatile(Object o, long offset);
2149 
2150     /** Volatile version of {@link #putDouble(Object, long, double)}  */
2151     @IntrinsicCandidate
2152     public native void    putDoubleVolatile(Object o, long offset, double x);
2153 
2154 
2155 
2156     /** Acquire version of {@link #getReferenceVolatile(Object, long)} */
2157     @IntrinsicCandidate
2158     public final Object getReferenceAcquire(Object o, long offset) {
2159         return getReferenceVolatile(o, offset);
2160     }
2161 
2162     /** Acquire version of {@link #getBooleanVolatile(Object, long)} */
2163     @IntrinsicCandidate
2164     public final boolean getBooleanAcquire(Object o, long offset) {
2165         return getBooleanVolatile(o, offset);
2166     }
2167 
2168     /** Acquire version of {@link #getByteVolatile(Object, long)} */
2169     @IntrinsicCandidate
2170     public final byte getByteAcquire(Object o, long offset) {
2171         return getByteVolatile(o, offset);
2172     }
2173 
2174     /** Acquire version of {@link #getShortVolatile(Object, long)} */
2175     @IntrinsicCandidate
2176     public final short getShortAcquire(Object o, long offset) {
2177         return getShortVolatile(o, offset);
2178     }
2179 
2180     /** Acquire version of {@link #getCharVolatile(Object, long)} */
2181     @IntrinsicCandidate
2182     public final char getCharAcquire(Object o, long offset) {
2183         return getCharVolatile(o, offset);
2184     }
2185 
2186     /** Acquire version of {@link #getIntVolatile(Object, long)} */
2187     @IntrinsicCandidate
2188     public final int getIntAcquire(Object o, long offset) {
2189         return getIntVolatile(o, offset);
2190     }
2191 
2192     /** Acquire version of {@link #getFloatVolatile(Object, long)} */
2193     @IntrinsicCandidate
2194     public final float getFloatAcquire(Object o, long offset) {
2195         return getFloatVolatile(o, offset);
2196     }
2197 
2198     /** Acquire version of {@link #getLongVolatile(Object, long)} */
2199     @IntrinsicCandidate
2200     public final long getLongAcquire(Object o, long offset) {
2201         return getLongVolatile(o, offset);
2202     }
2203 
2204     /** Acquire version of {@link #getDoubleVolatile(Object, long)} */
2205     @IntrinsicCandidate
2206     public final double getDoubleAcquire(Object o, long offset) {
2207         return getDoubleVolatile(o, offset);
2208     }
2209 
2210     /*
2211      * Versions of {@link #putReferenceVolatile(Object, long, Object)}
2212      * that do not guarantee immediate visibility of the store to
2213      * other threads. This method is generally only useful if the
2214      * underlying field is a Java volatile (or if an array cell, one
2215      * that is otherwise only accessed using volatile accesses).
2216      *
2217      * Corresponds to C11 atomic_store_explicit(..., memory_order_release).
2218      */
2219 
2220     /** Release version of {@link #putReferenceVolatile(Object, long, Object)} */
2221     @IntrinsicCandidate
2222     public final void putReferenceRelease(Object o, long offset, Object x) {
2223         putReferenceVolatile(o, offset, x);
2224     }
2225 
2226     /** Release version of {@link #putBooleanVolatile(Object, long, boolean)} */
2227     @IntrinsicCandidate
2228     public final void putBooleanRelease(Object o, long offset, boolean x) {
2229         putBooleanVolatile(o, offset, x);
2230     }
2231 
2232     /** Release version of {@link #putByteVolatile(Object, long, byte)} */
2233     @IntrinsicCandidate
2234     public final void putByteRelease(Object o, long offset, byte x) {
2235         putByteVolatile(o, offset, x);
2236     }
2237 
2238     /** Release version of {@link #putShortVolatile(Object, long, short)} */
2239     @IntrinsicCandidate
2240     public final void putShortRelease(Object o, long offset, short x) {
2241         putShortVolatile(o, offset, x);
2242     }
2243 
2244     /** Release version of {@link #putCharVolatile(Object, long, char)} */
2245     @IntrinsicCandidate
2246     public final void putCharRelease(Object o, long offset, char x) {
2247         putCharVolatile(o, offset, x);
2248     }
2249 
2250     /** Release version of {@link #putIntVolatile(Object, long, int)} */
2251     @IntrinsicCandidate
2252     public final void putIntRelease(Object o, long offset, int x) {
2253         putIntVolatile(o, offset, x);
2254     }
2255 
2256     /** Release version of {@link #putFloatVolatile(Object, long, float)} */
2257     @IntrinsicCandidate
2258     public final void putFloatRelease(Object o, long offset, float x) {
2259         putFloatVolatile(o, offset, x);
2260     }
2261 
2262     /** Release version of {@link #putLongVolatile(Object, long, long)} */
2263     @IntrinsicCandidate
2264     public final void putLongRelease(Object o, long offset, long x) {
2265         putLongVolatile(o, offset, x);
2266     }
2267 
2268     /** Release version of {@link #putDoubleVolatile(Object, long, double)} */
2269     @IntrinsicCandidate
2270     public final void putDoubleRelease(Object o, long offset, double x) {
2271         putDoubleVolatile(o, offset, x);
2272     }
2273 
2274     // ------------------------------ Opaque --------------------------------------
2275 
2276     /** Opaque version of {@link #getReferenceVolatile(Object, long)} */
2277     @IntrinsicCandidate
2278     public final Object getReferenceOpaque(Object o, long offset) {
2279         return getReferenceVolatile(o, offset);
2280     }
2281 
2282     /** Opaque version of {@link #getBooleanVolatile(Object, long)} */
2283     @IntrinsicCandidate
2284     public final boolean getBooleanOpaque(Object o, long offset) {
2285         return getBooleanVolatile(o, offset);
2286     }
2287 
2288     /** Opaque version of {@link #getByteVolatile(Object, long)} */
2289     @IntrinsicCandidate
2290     public final byte getByteOpaque(Object o, long offset) {
2291         return getByteVolatile(o, offset);
2292     }
2293 
2294     /** Opaque version of {@link #getShortVolatile(Object, long)} */
2295     @IntrinsicCandidate
2296     public final short getShortOpaque(Object o, long offset) {
2297         return getShortVolatile(o, offset);
2298     }
2299 
2300     /** Opaque version of {@link #getCharVolatile(Object, long)} */
2301     @IntrinsicCandidate
2302     public final char getCharOpaque(Object o, long offset) {
2303         return getCharVolatile(o, offset);
2304     }
2305 
2306     /** Opaque version of {@link #getIntVolatile(Object, long)} */
2307     @IntrinsicCandidate
2308     public final int getIntOpaque(Object o, long offset) {
2309         return getIntVolatile(o, offset);
2310     }
2311 
2312     /** Opaque version of {@link #getFloatVolatile(Object, long)} */
2313     @IntrinsicCandidate
2314     public final float getFloatOpaque(Object o, long offset) {
2315         return getFloatVolatile(o, offset);
2316     }
2317 
2318     /** Opaque version of {@link #getLongVolatile(Object, long)} */
2319     @IntrinsicCandidate
2320     public final long getLongOpaque(Object o, long offset) {
2321         return getLongVolatile(o, offset);
2322     }
2323 
2324     /** Opaque version of {@link #getDoubleVolatile(Object, long)} */
2325     @IntrinsicCandidate
2326     public final double getDoubleOpaque(Object o, long offset) {
2327         return getDoubleVolatile(o, offset);
2328     }
2329 
2330     /** Opaque version of {@link #putReferenceVolatile(Object, long, Object)} */
2331     @IntrinsicCandidate
2332     public final void putReferenceOpaque(Object o, long offset, Object x) {
2333         putReferenceVolatile(o, offset, x);
2334     }
2335 
2336     /** Opaque version of {@link #putBooleanVolatile(Object, long, boolean)} */
2337     @IntrinsicCandidate
2338     public final void putBooleanOpaque(Object o, long offset, boolean x) {
2339         putBooleanVolatile(o, offset, x);
2340     }
2341 
2342     /** Opaque version of {@link #putByteVolatile(Object, long, byte)} */
2343     @IntrinsicCandidate
2344     public final void putByteOpaque(Object o, long offset, byte x) {
2345         putByteVolatile(o, offset, x);
2346     }
2347 
2348     /** Opaque version of {@link #putShortVolatile(Object, long, short)} */
2349     @IntrinsicCandidate
2350     public final void putShortOpaque(Object o, long offset, short x) {
2351         putShortVolatile(o, offset, x);
2352     }
2353 
2354     /** Opaque version of {@link #putCharVolatile(Object, long, char)} */
2355     @IntrinsicCandidate
2356     public final void putCharOpaque(Object o, long offset, char x) {
2357         putCharVolatile(o, offset, x);
2358     }
2359 
2360     /** Opaque version of {@link #putIntVolatile(Object, long, int)} */
2361     @IntrinsicCandidate
2362     public final void putIntOpaque(Object o, long offset, int x) {
2363         putIntVolatile(o, offset, x);
2364     }
2365 
2366     /** Opaque version of {@link #putFloatVolatile(Object, long, float)} */
2367     @IntrinsicCandidate
2368     public final void putFloatOpaque(Object o, long offset, float x) {
2369         putFloatVolatile(o, offset, x);
2370     }
2371 
2372     /** Opaque version of {@link #putLongVolatile(Object, long, long)} */
2373     @IntrinsicCandidate
2374     public final void putLongOpaque(Object o, long offset, long x) {
2375         putLongVolatile(o, offset, x);
2376     }
2377 
2378     /** Opaque version of {@link #putDoubleVolatile(Object, long, double)} */
2379     @IntrinsicCandidate
2380     public final void putDoubleOpaque(Object o, long offset, double x) {
2381         putDoubleVolatile(o, offset, x);
2382     }
2383 
2384     /**
2385      * Unblocks the given thread blocked on {@code park}, or, if it is
2386      * not blocked, causes the subsequent call to {@code park} not to
2387      * block.  Note: this operation is "unsafe" solely because the
2388      * caller must somehow ensure that the thread has not been
2389      * destroyed. Nothing special is usually required to ensure this
2390      * when called from Java (in which there will ordinarily be a live
2391      * reference to the thread) but this is not nearly-automatically
2392      * so when calling from native code.
2393      *
2394      * @param thread the thread to unpark.
2395      */
2396     @IntrinsicCandidate
2397     public native void unpark(Object thread);
2398 
2399     /**
2400      * Blocks current thread, returning when a balancing
2401      * {@code unpark} occurs, or a balancing {@code unpark} has
2402      * already occurred, or the thread is interrupted, or, if not
2403      * absolute and time is not zero, the given time nanoseconds have
2404      * elapsed, or if absolute, the given deadline in milliseconds
2405      * since Epoch has passed, or spuriously (i.e., returning for no
2406      * "reason"). Note: This operation is in the Unsafe class only
2407      * because {@code unpark} is, so it would be strange to place it
2408      * elsewhere.
2409      */
2410     @IntrinsicCandidate
2411     public native void park(boolean isAbsolute, long time);
2412 
2413     /**
2414      * Gets the load average in the system run queue assigned
2415      * to the available processors averaged over various periods of time.
2416      * This method retrieves the given {@code nelem} samples and
2417      * assigns to the elements of the given {@code loadavg} array.
2418      * The system imposes a maximum of 3 samples, representing
2419      * averages over the last 1,  5,  and  15 minutes, respectively.
2420      *
2421      * @param loadavg an array of double of size nelems
2422      * @param nelems the number of samples to be retrieved and
2423      *        must be 1 to 3.
2424      *
2425      * @return the number of samples actually retrieved; or -1
2426      *         if the load average is unobtainable.
2427      */
2428     public int getLoadAverage(double[] loadavg, int nelems) {
2429         if (nelems < 0 || nelems > 3 || nelems > loadavg.length) {
2430             throw new ArrayIndexOutOfBoundsException();
2431         }
2432 
2433         return getLoadAverage0(loadavg, nelems);
2434     }
2435 
2436     // The following contain CAS-based Java implementations used on
2437     // platforms not supporting native instructions
2438 
2439     /**
2440      * Atomically adds the given value to the current value of a field
2441      * or array element within the given object {@code o}
2442      * at the given {@code offset}.
2443      *
2444      * @param o object/array to update the field/element in
2445      * @param offset field/element offset
2446      * @param delta the value to add
2447      * @return the previous value
2448      * @since 1.8
2449      */
2450     @IntrinsicCandidate
2451     public final int getAndAddInt(Object o, long offset, int delta) {
2452         int v;
2453         do {
2454             v = getIntVolatile(o, offset);
2455         } while (!weakCompareAndSetInt(o, offset, v, v + delta));
2456         return v;
2457     }
2458 
2459     @ForceInline
2460     public final int getAndAddIntRelease(Object o, long offset, int delta) {
2461         int v;
2462         do {
2463             v = getInt(o, offset);
2464         } while (!weakCompareAndSetIntRelease(o, offset, v, v + delta));
2465         return v;
2466     }
2467 
2468     @ForceInline
2469     public final int getAndAddIntAcquire(Object o, long offset, int delta) {
2470         int v;
2471         do {
2472             v = getIntAcquire(o, offset);
2473         } while (!weakCompareAndSetIntAcquire(o, offset, v, v + delta));
2474         return v;
2475     }
2476 
2477     /**
2478      * Atomically adds the given value to the current value of a field
2479      * or array element within the given object {@code o}
2480      * at the given {@code offset}.
2481      *
2482      * @param o object/array to update the field/element in
2483      * @param offset field/element offset
2484      * @param delta the value to add
2485      * @return the previous value
2486      * @since 1.8
2487      */
2488     @IntrinsicCandidate
2489     public final long getAndAddLong(Object o, long offset, long delta) {
2490         long v;
2491         do {
2492             v = getLongVolatile(o, offset);
2493         } while (!weakCompareAndSetLong(o, offset, v, v + delta));
2494         return v;
2495     }
2496 
2497     @ForceInline
2498     public final long getAndAddLongRelease(Object o, long offset, long delta) {
2499         long v;
2500         do {
2501             v = getLong(o, offset);
2502         } while (!weakCompareAndSetLongRelease(o, offset, v, v + delta));
2503         return v;
2504     }
2505 
2506     @ForceInline
2507     public final long getAndAddLongAcquire(Object o, long offset, long delta) {
2508         long v;
2509         do {
2510             v = getLongAcquire(o, offset);
2511         } while (!weakCompareAndSetLongAcquire(o, offset, v, v + delta));
2512         return v;
2513     }
2514 
2515     @IntrinsicCandidate
2516     public final byte getAndAddByte(Object o, long offset, byte delta) {
2517         byte v;
2518         do {
2519             v = getByteVolatile(o, offset);
2520         } while (!weakCompareAndSetByte(o, offset, v, (byte) (v + delta)));
2521         return v;
2522     }
2523 
2524     @ForceInline
2525     public final byte getAndAddByteRelease(Object o, long offset, byte delta) {
2526         byte v;
2527         do {
2528             v = getByte(o, offset);
2529         } while (!weakCompareAndSetByteRelease(o, offset, v, (byte) (v + delta)));
2530         return v;
2531     }
2532 
2533     @ForceInline
2534     public final byte getAndAddByteAcquire(Object o, long offset, byte delta) {
2535         byte v;
2536         do {
2537             v = getByteAcquire(o, offset);
2538         } while (!weakCompareAndSetByteAcquire(o, offset, v, (byte) (v + delta)));
2539         return v;
2540     }
2541 
2542     @IntrinsicCandidate
2543     public final short getAndAddShort(Object o, long offset, short delta) {
2544         short v;
2545         do {
2546             v = getShortVolatile(o, offset);
2547         } while (!weakCompareAndSetShort(o, offset, v, (short) (v + delta)));
2548         return v;
2549     }
2550 
2551     @ForceInline
2552     public final short getAndAddShortRelease(Object o, long offset, short delta) {
2553         short v;
2554         do {
2555             v = getShort(o, offset);
2556         } while (!weakCompareAndSetShortRelease(o, offset, v, (short) (v + delta)));
2557         return v;
2558     }
2559 
2560     @ForceInline
2561     public final short getAndAddShortAcquire(Object o, long offset, short delta) {
2562         short v;
2563         do {
2564             v = getShortAcquire(o, offset);
2565         } while (!weakCompareAndSetShortAcquire(o, offset, v, (short) (v + delta)));
2566         return v;
2567     }
2568 
2569     @ForceInline
2570     public final char getAndAddChar(Object o, long offset, char delta) {
2571         return (char) getAndAddShort(o, offset, (short) delta);
2572     }
2573 
2574     @ForceInline
2575     public final char getAndAddCharRelease(Object o, long offset, char delta) {
2576         return (char) getAndAddShortRelease(o, offset, (short) delta);
2577     }
2578 
2579     @ForceInline
2580     public final char getAndAddCharAcquire(Object o, long offset, char delta) {
2581         return (char) getAndAddShortAcquire(o, offset, (short) delta);
2582     }
2583 
2584     @ForceInline
2585     public final float getAndAddFloat(Object o, long offset, float delta) {
2586         int expectedBits;
2587         float v;
2588         do {
2589             // Load and CAS with the raw bits to avoid issues with NaNs and
2590             // possible bit conversion from signaling NaNs to quiet NaNs that
2591             // may result in the loop not terminating.
2592             expectedBits = getIntVolatile(o, offset);
2593             v = Float.intBitsToFloat(expectedBits);
2594         } while (!weakCompareAndSetInt(o, offset,
2595                                                 expectedBits, Float.floatToRawIntBits(v + delta)));
2596         return v;
2597     }
2598 
2599     @ForceInline
2600     public final float getAndAddFloatRelease(Object o, long offset, float delta) {
2601         int expectedBits;
2602         float v;
2603         do {
2604             // Load and CAS with the raw bits to avoid issues with NaNs and
2605             // possible bit conversion from signaling NaNs to quiet NaNs that
2606             // may result in the loop not terminating.
2607             expectedBits = getInt(o, offset);
2608             v = Float.intBitsToFloat(expectedBits);
2609         } while (!weakCompareAndSetIntRelease(o, offset,
2610                                                expectedBits, Float.floatToRawIntBits(v + delta)));
2611         return v;
2612     }
2613 
2614     @ForceInline
2615     public final float getAndAddFloatAcquire(Object o, long offset, float delta) {
2616         int expectedBits;
2617         float v;
2618         do {
2619             // Load and CAS with the raw bits to avoid issues with NaNs and
2620             // possible bit conversion from signaling NaNs to quiet NaNs that
2621             // may result in the loop not terminating.
2622             expectedBits = getIntAcquire(o, offset);
2623             v = Float.intBitsToFloat(expectedBits);
2624         } while (!weakCompareAndSetIntAcquire(o, offset,
2625                                                expectedBits, Float.floatToRawIntBits(v + delta)));
2626         return v;
2627     }
2628 
2629     @ForceInline
2630     public final double getAndAddDouble(Object o, long offset, double delta) {
2631         long expectedBits;
2632         double v;
2633         do {
2634             // Load and CAS with the raw bits to avoid issues with NaNs and
2635             // possible bit conversion from signaling NaNs to quiet NaNs that
2636             // may result in the loop not terminating.
2637             expectedBits = getLongVolatile(o, offset);
2638             v = Double.longBitsToDouble(expectedBits);
2639         } while (!weakCompareAndSetLong(o, offset,
2640                                                  expectedBits, Double.doubleToRawLongBits(v + delta)));
2641         return v;
2642     }
2643 
2644     @ForceInline
2645     public final double getAndAddDoubleRelease(Object o, long offset, double delta) {
2646         long expectedBits;
2647         double v;
2648         do {
2649             // Load and CAS with the raw bits to avoid issues with NaNs and
2650             // possible bit conversion from signaling NaNs to quiet NaNs that
2651             // may result in the loop not terminating.
2652             expectedBits = getLong(o, offset);
2653             v = Double.longBitsToDouble(expectedBits);
2654         } while (!weakCompareAndSetLongRelease(o, offset,
2655                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
2656         return v;
2657     }
2658 
2659     @ForceInline
2660     public final double getAndAddDoubleAcquire(Object o, long offset, double delta) {
2661         long expectedBits;
2662         double v;
2663         do {
2664             // Load and CAS with the raw bits to avoid issues with NaNs and
2665             // possible bit conversion from signaling NaNs to quiet NaNs that
2666             // may result in the loop not terminating.
2667             expectedBits = getLongAcquire(o, offset);
2668             v = Double.longBitsToDouble(expectedBits);
2669         } while (!weakCompareAndSetLongAcquire(o, offset,
2670                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
2671         return v;
2672     }
2673 
2674     /**
2675      * Atomically exchanges the given value with the current value of
2676      * a field or array element within the given object {@code o}
2677      * at the given {@code offset}.
2678      *
2679      * @param o object/array to update the field/element in
2680      * @param offset field/element offset
2681      * @param newValue new value
2682      * @return the previous value
2683      * @since 1.8
2684      */
2685     @IntrinsicCandidate
2686     public final int getAndSetInt(Object o, long offset, int newValue) {
2687         int v;
2688         do {
2689             v = getIntVolatile(o, offset);
2690         } while (!weakCompareAndSetInt(o, offset, v, newValue));
2691         return v;
2692     }
2693 
2694     @ForceInline
2695     public final int getAndSetIntRelease(Object o, long offset, int newValue) {
2696         int v;
2697         do {
2698             v = getInt(o, offset);
2699         } while (!weakCompareAndSetIntRelease(o, offset, v, newValue));
2700         return v;
2701     }
2702 
2703     @ForceInline
2704     public final int getAndSetIntAcquire(Object o, long offset, int newValue) {
2705         int v;
2706         do {
2707             v = getIntAcquire(o, offset);
2708         } while (!weakCompareAndSetIntAcquire(o, offset, v, newValue));
2709         return v;
2710     }
2711 
2712     /**
2713      * Atomically exchanges the given value with the current value of
2714      * a field or array element within the given object {@code o}
2715      * at the given {@code offset}.
2716      *
2717      * @param o object/array to update the field/element in
2718      * @param offset field/element offset
2719      * @param newValue new value
2720      * @return the previous value
2721      * @since 1.8
2722      */
2723     @IntrinsicCandidate
2724     public final long getAndSetLong(Object o, long offset, long newValue) {
2725         long v;
2726         do {
2727             v = getLongVolatile(o, offset);
2728         } while (!weakCompareAndSetLong(o, offset, v, newValue));
2729         return v;
2730     }
2731 
2732     @ForceInline
2733     public final long getAndSetLongRelease(Object o, long offset, long newValue) {
2734         long v;
2735         do {
2736             v = getLong(o, offset);
2737         } while (!weakCompareAndSetLongRelease(o, offset, v, newValue));
2738         return v;
2739     }
2740 
2741     @ForceInline
2742     public final long getAndSetLongAcquire(Object o, long offset, long newValue) {
2743         long v;
2744         do {
2745             v = getLongAcquire(o, offset);
2746         } while (!weakCompareAndSetLongAcquire(o, offset, v, newValue));
2747         return v;
2748     }
2749 
2750     /**
2751      * Atomically exchanges the given reference value with the current
2752      * reference value of a field or array element within the given
2753      * object {@code o} at the given {@code offset}.
2754      *
2755      * @param o object/array to update the field/element in
2756      * @param offset field/element offset
2757      * @param newValue new value
2758      * @return the previous value
2759      * @since 1.8
2760      */
2761     @IntrinsicCandidate
2762     public final Object getAndSetReference(Object o, long offset, Object newValue) {
2763         Object v;
2764         do {
2765             v = getReferenceVolatile(o, offset);
2766         } while (!weakCompareAndSetReference(o, offset, v, newValue));
2767         return v;
2768     }
2769 
2770     @ForceInline
2771     public final Object getAndSetReferenceRelease(Object o, long offset, Object newValue) {
2772         Object v;
2773         do {
2774             v = getReference(o, offset);
2775         } while (!weakCompareAndSetReferenceRelease(o, offset, v, newValue));
2776         return v;
2777     }
2778 
2779     @ForceInline
2780     public final Object getAndSetReferenceAcquire(Object o, long offset, Object newValue) {
2781         Object v;
2782         do {
2783             v = getReferenceAcquire(o, offset);
2784         } while (!weakCompareAndSetReferenceAcquire(o, offset, v, newValue));
2785         return v;
2786     }
2787 
2788     @IntrinsicCandidate
2789     public final byte getAndSetByte(Object o, long offset, byte newValue) {
2790         byte v;
2791         do {
2792             v = getByteVolatile(o, offset);
2793         } while (!weakCompareAndSetByte(o, offset, v, newValue));
2794         return v;
2795     }
2796 
2797     @ForceInline
2798     public final byte getAndSetByteRelease(Object o, long offset, byte newValue) {
2799         byte v;
2800         do {
2801             v = getByte(o, offset);
2802         } while (!weakCompareAndSetByteRelease(o, offset, v, newValue));
2803         return v;
2804     }
2805 
2806     @ForceInline
2807     public final byte getAndSetByteAcquire(Object o, long offset, byte newValue) {
2808         byte v;
2809         do {
2810             v = getByteAcquire(o, offset);
2811         } while (!weakCompareAndSetByteAcquire(o, offset, v, newValue));
2812         return v;
2813     }
2814 
2815     @ForceInline
2816     public final boolean getAndSetBoolean(Object o, long offset, boolean newValue) {
2817         return byte2bool(getAndSetByte(o, offset, bool2byte(newValue)));
2818     }
2819 
2820     @ForceInline
2821     public final boolean getAndSetBooleanRelease(Object o, long offset, boolean newValue) {
2822         return byte2bool(getAndSetByteRelease(o, offset, bool2byte(newValue)));
2823     }
2824 
2825     @ForceInline
2826     public final boolean getAndSetBooleanAcquire(Object o, long offset, boolean newValue) {
2827         return byte2bool(getAndSetByteAcquire(o, offset, bool2byte(newValue)));
2828     }
2829 
2830     @IntrinsicCandidate
2831     public final short getAndSetShort(Object o, long offset, short newValue) {
2832         short v;
2833         do {
2834             v = getShortVolatile(o, offset);
2835         } while (!weakCompareAndSetShort(o, offset, v, newValue));
2836         return v;
2837     }
2838 
2839     @ForceInline
2840     public final short getAndSetShortRelease(Object o, long offset, short newValue) {
2841         short v;
2842         do {
2843             v = getShort(o, offset);
2844         } while (!weakCompareAndSetShortRelease(o, offset, v, newValue));
2845         return v;
2846     }
2847 
2848     @ForceInline
2849     public final short getAndSetShortAcquire(Object o, long offset, short newValue) {
2850         short v;
2851         do {
2852             v = getShortAcquire(o, offset);
2853         } while (!weakCompareAndSetShortAcquire(o, offset, v, newValue));
2854         return v;
2855     }
2856 
2857     @ForceInline
2858     public final char getAndSetChar(Object o, long offset, char newValue) {
2859         return s2c(getAndSetShort(o, offset, c2s(newValue)));
2860     }
2861 
2862     @ForceInline
2863     public final char getAndSetCharRelease(Object o, long offset, char newValue) {
2864         return s2c(getAndSetShortRelease(o, offset, c2s(newValue)));
2865     }
2866 
2867     @ForceInline
2868     public final char getAndSetCharAcquire(Object o, long offset, char newValue) {
2869         return s2c(getAndSetShortAcquire(o, offset, c2s(newValue)));
2870     }
2871 
2872     @ForceInline
2873     public final float getAndSetFloat(Object o, long offset, float newValue) {
2874         int v = getAndSetInt(o, offset, Float.floatToRawIntBits(newValue));
2875         return Float.intBitsToFloat(v);
2876     }
2877 
2878     @ForceInline
2879     public final float getAndSetFloatRelease(Object o, long offset, float newValue) {
2880         int v = getAndSetIntRelease(o, offset, Float.floatToRawIntBits(newValue));
2881         return Float.intBitsToFloat(v);
2882     }
2883 
2884     @ForceInline
2885     public final float getAndSetFloatAcquire(Object o, long offset, float newValue) {
2886         int v = getAndSetIntAcquire(o, offset, Float.floatToRawIntBits(newValue));
2887         return Float.intBitsToFloat(v);
2888     }
2889 
2890     @ForceInline
2891     public final double getAndSetDouble(Object o, long offset, double newValue) {
2892         long v = getAndSetLong(o, offset, Double.doubleToRawLongBits(newValue));
2893         return Double.longBitsToDouble(v);
2894     }
2895 
2896     @ForceInline
2897     public final double getAndSetDoubleRelease(Object o, long offset, double newValue) {
2898         long v = getAndSetLongRelease(o, offset, Double.doubleToRawLongBits(newValue));
2899         return Double.longBitsToDouble(v);
2900     }
2901 
2902     @ForceInline
2903     public final double getAndSetDoubleAcquire(Object o, long offset, double newValue) {
2904         long v = getAndSetLongAcquire(o, offset, Double.doubleToRawLongBits(newValue));
2905         return Double.longBitsToDouble(v);
2906     }
2907 
2908 
2909     // The following contain CAS-based Java implementations used on
2910     // platforms not supporting native instructions
2911 
2912     @ForceInline
2913     public final boolean getAndBitwiseOrBoolean(Object o, long offset, boolean mask) {
2914         return byte2bool(getAndBitwiseOrByte(o, offset, bool2byte(mask)));
2915     }
2916 
2917     @ForceInline
2918     public final boolean getAndBitwiseOrBooleanRelease(Object o, long offset, boolean mask) {
2919         return byte2bool(getAndBitwiseOrByteRelease(o, offset, bool2byte(mask)));
2920     }
2921 
2922     @ForceInline
2923     public final boolean getAndBitwiseOrBooleanAcquire(Object o, long offset, boolean mask) {
2924         return byte2bool(getAndBitwiseOrByteAcquire(o, offset, bool2byte(mask)));
2925     }
2926 
2927     @ForceInline
2928     public final boolean getAndBitwiseAndBoolean(Object o, long offset, boolean mask) {
2929         return byte2bool(getAndBitwiseAndByte(o, offset, bool2byte(mask)));
2930     }
2931 
2932     @ForceInline
2933     public final boolean getAndBitwiseAndBooleanRelease(Object o, long offset, boolean mask) {
2934         return byte2bool(getAndBitwiseAndByteRelease(o, offset, bool2byte(mask)));
2935     }
2936 
2937     @ForceInline
2938     public final boolean getAndBitwiseAndBooleanAcquire(Object o, long offset, boolean mask) {
2939         return byte2bool(getAndBitwiseAndByteAcquire(o, offset, bool2byte(mask)));
2940     }
2941 
2942     @ForceInline
2943     public final boolean getAndBitwiseXorBoolean(Object o, long offset, boolean mask) {
2944         return byte2bool(getAndBitwiseXorByte(o, offset, bool2byte(mask)));
2945     }
2946 
2947     @ForceInline
2948     public final boolean getAndBitwiseXorBooleanRelease(Object o, long offset, boolean mask) {
2949         return byte2bool(getAndBitwiseXorByteRelease(o, offset, bool2byte(mask)));
2950     }
2951 
2952     @ForceInline
2953     public final boolean getAndBitwiseXorBooleanAcquire(Object o, long offset, boolean mask) {
2954         return byte2bool(getAndBitwiseXorByteAcquire(o, offset, bool2byte(mask)));
2955     }
2956 
2957 
2958     @ForceInline
2959     public final byte getAndBitwiseOrByte(Object o, long offset, byte mask) {
2960         byte current;
2961         do {
2962             current = getByteVolatile(o, offset);
2963         } while (!weakCompareAndSetByte(o, offset,
2964                                                   current, (byte) (current | mask)));
2965         return current;
2966     }
2967 
2968     @ForceInline
2969     public final byte getAndBitwiseOrByteRelease(Object o, long offset, byte mask) {
2970         byte current;
2971         do {
2972             current = getByte(o, offset);
2973         } while (!weakCompareAndSetByteRelease(o, offset,
2974                                                  current, (byte) (current | mask)));
2975         return current;
2976     }
2977 
2978     @ForceInline
2979     public final byte getAndBitwiseOrByteAcquire(Object o, long offset, byte mask) {
2980         byte current;
2981         do {
2982             // Plain read, the value is a hint, the acquire CAS does the work
2983             current = getByte(o, offset);
2984         } while (!weakCompareAndSetByteAcquire(o, offset,
2985                                                  current, (byte) (current | mask)));
2986         return current;
2987     }
2988 
2989     @ForceInline
2990     public final byte getAndBitwiseAndByte(Object o, long offset, byte mask) {
2991         byte current;
2992         do {
2993             current = getByteVolatile(o, offset);
2994         } while (!weakCompareAndSetByte(o, offset,
2995                                                   current, (byte) (current & mask)));
2996         return current;
2997     }
2998 
2999     @ForceInline
3000     public final byte getAndBitwiseAndByteRelease(Object o, long offset, byte mask) {
3001         byte current;
3002         do {
3003             current = getByte(o, offset);
3004         } while (!weakCompareAndSetByteRelease(o, offset,
3005                                                  current, (byte) (current & mask)));
3006         return current;
3007     }
3008 
3009     @ForceInline
3010     public final byte getAndBitwiseAndByteAcquire(Object o, long offset, byte mask) {
3011         byte current;
3012         do {
3013             // Plain read, the value is a hint, the acquire CAS does the work
3014             current = getByte(o, offset);
3015         } while (!weakCompareAndSetByteAcquire(o, offset,
3016                                                  current, (byte) (current & mask)));
3017         return current;
3018     }
3019 
3020     @ForceInline
3021     public final byte getAndBitwiseXorByte(Object o, long offset, byte mask) {
3022         byte current;
3023         do {
3024             current = getByteVolatile(o, offset);
3025         } while (!weakCompareAndSetByte(o, offset,
3026                                                   current, (byte) (current ^ mask)));
3027         return current;
3028     }
3029 
3030     @ForceInline
3031     public final byte getAndBitwiseXorByteRelease(Object o, long offset, byte mask) {
3032         byte current;
3033         do {
3034             current = getByte(o, offset);
3035         } while (!weakCompareAndSetByteRelease(o, offset,
3036                                                  current, (byte) (current ^ mask)));
3037         return current;
3038     }
3039 
3040     @ForceInline
3041     public final byte getAndBitwiseXorByteAcquire(Object o, long offset, byte mask) {
3042         byte current;
3043         do {
3044             // Plain read, the value is a hint, the acquire CAS does the work
3045             current = getByte(o, offset);
3046         } while (!weakCompareAndSetByteAcquire(o, offset,
3047                                                  current, (byte) (current ^ mask)));
3048         return current;
3049     }
3050 
3051 
3052     @ForceInline
3053     public final char getAndBitwiseOrChar(Object o, long offset, char mask) {
3054         return s2c(getAndBitwiseOrShort(o, offset, c2s(mask)));
3055     }
3056 
3057     @ForceInline
3058     public final char getAndBitwiseOrCharRelease(Object o, long offset, char mask) {
3059         return s2c(getAndBitwiseOrShortRelease(o, offset, c2s(mask)));
3060     }
3061 
3062     @ForceInline
3063     public final char getAndBitwiseOrCharAcquire(Object o, long offset, char mask) {
3064         return s2c(getAndBitwiseOrShortAcquire(o, offset, c2s(mask)));
3065     }
3066 
3067     @ForceInline
3068     public final char getAndBitwiseAndChar(Object o, long offset, char mask) {
3069         return s2c(getAndBitwiseAndShort(o, offset, c2s(mask)));
3070     }
3071 
3072     @ForceInline
3073     public final char getAndBitwiseAndCharRelease(Object o, long offset, char mask) {
3074         return s2c(getAndBitwiseAndShortRelease(o, offset, c2s(mask)));
3075     }
3076 
3077     @ForceInline
3078     public final char getAndBitwiseAndCharAcquire(Object o, long offset, char mask) {
3079         return s2c(getAndBitwiseAndShortAcquire(o, offset, c2s(mask)));
3080     }
3081 
3082     @ForceInline
3083     public final char getAndBitwiseXorChar(Object o, long offset, char mask) {
3084         return s2c(getAndBitwiseXorShort(o, offset, c2s(mask)));
3085     }
3086 
3087     @ForceInline
3088     public final char getAndBitwiseXorCharRelease(Object o, long offset, char mask) {
3089         return s2c(getAndBitwiseXorShortRelease(o, offset, c2s(mask)));
3090     }
3091 
3092     @ForceInline
3093     public final char getAndBitwiseXorCharAcquire(Object o, long offset, char mask) {
3094         return s2c(getAndBitwiseXorShortAcquire(o, offset, c2s(mask)));
3095     }
3096 
3097 
3098     @ForceInline
3099     public final short getAndBitwiseOrShort(Object o, long offset, short mask) {
3100         short current;
3101         do {
3102             current = getShortVolatile(o, offset);
3103         } while (!weakCompareAndSetShort(o, offset,
3104                                                 current, (short) (current | mask)));
3105         return current;
3106     }
3107 
3108     @ForceInline
3109     public final short getAndBitwiseOrShortRelease(Object o, long offset, short mask) {
3110         short current;
3111         do {
3112             current = getShort(o, offset);
3113         } while (!weakCompareAndSetShortRelease(o, offset,
3114                                                current, (short) (current | mask)));
3115         return current;
3116     }
3117 
3118     @ForceInline
3119     public final short getAndBitwiseOrShortAcquire(Object o, long offset, short mask) {
3120         short current;
3121         do {
3122             // Plain read, the value is a hint, the acquire CAS does the work
3123             current = getShort(o, offset);
3124         } while (!weakCompareAndSetShortAcquire(o, offset,
3125                                                current, (short) (current | mask)));
3126         return current;
3127     }
3128 
3129     @ForceInline
3130     public final short getAndBitwiseAndShort(Object o, long offset, short mask) {
3131         short current;
3132         do {
3133             current = getShortVolatile(o, offset);
3134         } while (!weakCompareAndSetShort(o, offset,
3135                                                 current, (short) (current & mask)));
3136         return current;
3137     }
3138 
3139     @ForceInline
3140     public final short getAndBitwiseAndShortRelease(Object o, long offset, short mask) {
3141         short current;
3142         do {
3143             current = getShort(o, offset);
3144         } while (!weakCompareAndSetShortRelease(o, offset,
3145                                                current, (short) (current & mask)));
3146         return current;
3147     }
3148 
3149     @ForceInline
3150     public final short getAndBitwiseAndShortAcquire(Object o, long offset, short mask) {
3151         short current;
3152         do {
3153             // Plain read, the value is a hint, the acquire CAS does the work
3154             current = getShort(o, offset);
3155         } while (!weakCompareAndSetShortAcquire(o, offset,
3156                                                current, (short) (current & mask)));
3157         return current;
3158     }
3159 
3160     @ForceInline
3161     public final short getAndBitwiseXorShort(Object o, long offset, short mask) {
3162         short current;
3163         do {
3164             current = getShortVolatile(o, offset);
3165         } while (!weakCompareAndSetShort(o, offset,
3166                                                 current, (short) (current ^ mask)));
3167         return current;
3168     }
3169 
3170     @ForceInline
3171     public final short getAndBitwiseXorShortRelease(Object o, long offset, short mask) {
3172         short current;
3173         do {
3174             current = getShort(o, offset);
3175         } while (!weakCompareAndSetShortRelease(o, offset,
3176                                                current, (short) (current ^ mask)));
3177         return current;
3178     }
3179 
3180     @ForceInline
3181     public final short getAndBitwiseXorShortAcquire(Object o, long offset, short mask) {
3182         short current;
3183         do {
3184             // Plain read, the value is a hint, the acquire CAS does the work
3185             current = getShort(o, offset);
3186         } while (!weakCompareAndSetShortAcquire(o, offset,
3187                                                current, (short) (current ^ mask)));
3188         return current;
3189     }
3190 
3191 
3192     @ForceInline
3193     public final int getAndBitwiseOrInt(Object o, long offset, int mask) {
3194         int current;
3195         do {
3196             current = getIntVolatile(o, offset);
3197         } while (!weakCompareAndSetInt(o, offset,
3198                                                 current, current | mask));
3199         return current;
3200     }
3201 
3202     @ForceInline
3203     public final int getAndBitwiseOrIntRelease(Object o, long offset, int mask) {
3204         int current;
3205         do {
3206             current = getInt(o, offset);
3207         } while (!weakCompareAndSetIntRelease(o, offset,
3208                                                current, current | mask));
3209         return current;
3210     }
3211 
3212     @ForceInline
3213     public final int getAndBitwiseOrIntAcquire(Object o, long offset, int mask) {
3214         int current;
3215         do {
3216             // Plain read, the value is a hint, the acquire CAS does the work
3217             current = getInt(o, offset);
3218         } while (!weakCompareAndSetIntAcquire(o, offset,
3219                                                current, current | mask));
3220         return current;
3221     }
3222 
3223     /**
3224      * Atomically replaces the current value of a field or array element within
3225      * the given object with the result of bitwise AND between the current value
3226      * and mask.
3227      *
3228      * @param o object/array to update the field/element in
3229      * @param offset field/element offset
3230      * @param mask the mask value
3231      * @return the previous value
3232      * @since 9
3233      */
3234     @ForceInline
3235     public final int getAndBitwiseAndInt(Object o, long offset, int mask) {
3236         int current;
3237         do {
3238             current = getIntVolatile(o, offset);
3239         } while (!weakCompareAndSetInt(o, offset,
3240                                                 current, current & mask));
3241         return current;
3242     }
3243 
3244     @ForceInline
3245     public final int getAndBitwiseAndIntRelease(Object o, long offset, int mask) {
3246         int current;
3247         do {
3248             current = getInt(o, offset);
3249         } while (!weakCompareAndSetIntRelease(o, offset,
3250                                                current, current & mask));
3251         return current;
3252     }
3253 
3254     @ForceInline
3255     public final int getAndBitwiseAndIntAcquire(Object o, long offset, int mask) {
3256         int current;
3257         do {
3258             // Plain read, the value is a hint, the acquire CAS does the work
3259             current = getInt(o, offset);
3260         } while (!weakCompareAndSetIntAcquire(o, offset,
3261                                                current, current & mask));
3262         return current;
3263     }
3264 
3265     @ForceInline
3266     public final int getAndBitwiseXorInt(Object o, long offset, int mask) {
3267         int current;
3268         do {
3269             current = getIntVolatile(o, offset);
3270         } while (!weakCompareAndSetInt(o, offset,
3271                                                 current, current ^ mask));
3272         return current;
3273     }
3274 
3275     @ForceInline
3276     public final int getAndBitwiseXorIntRelease(Object o, long offset, int mask) {
3277         int current;
3278         do {
3279             current = getInt(o, offset);
3280         } while (!weakCompareAndSetIntRelease(o, offset,
3281                                                current, current ^ mask));
3282         return current;
3283     }
3284 
3285     @ForceInline
3286     public final int getAndBitwiseXorIntAcquire(Object o, long offset, int mask) {
3287         int current;
3288         do {
3289             // Plain read, the value is a hint, the acquire CAS does the work
3290             current = getInt(o, offset);
3291         } while (!weakCompareAndSetIntAcquire(o, offset,
3292                                                current, current ^ mask));
3293         return current;
3294     }
3295 
3296 
3297     @ForceInline
3298     public final long getAndBitwiseOrLong(Object o, long offset, long mask) {
3299         long current;
3300         do {
3301             current = getLongVolatile(o, offset);
3302         } while (!weakCompareAndSetLong(o, offset,
3303                                                 current, current | mask));
3304         return current;
3305     }
3306 
3307     @ForceInline
3308     public final long getAndBitwiseOrLongRelease(Object o, long offset, long mask) {
3309         long current;
3310         do {
3311             current = getLong(o, offset);
3312         } while (!weakCompareAndSetLongRelease(o, offset,
3313                                                current, current | mask));
3314         return current;
3315     }
3316 
3317     @ForceInline
3318     public final long getAndBitwiseOrLongAcquire(Object o, long offset, long mask) {
3319         long current;
3320         do {
3321             // Plain read, the value is a hint, the acquire CAS does the work
3322             current = getLong(o, offset);
3323         } while (!weakCompareAndSetLongAcquire(o, offset,
3324                                                current, current | mask));
3325         return current;
3326     }
3327 
3328     @ForceInline
3329     public final long getAndBitwiseAndLong(Object o, long offset, long mask) {
3330         long current;
3331         do {
3332             current = getLongVolatile(o, offset);
3333         } while (!weakCompareAndSetLong(o, offset,
3334                                                 current, current & mask));
3335         return current;
3336     }
3337 
3338     @ForceInline
3339     public final long getAndBitwiseAndLongRelease(Object o, long offset, long mask) {
3340         long current;
3341         do {
3342             current = getLong(o, offset);
3343         } while (!weakCompareAndSetLongRelease(o, offset,
3344                                                current, current & mask));
3345         return current;
3346     }
3347 
3348     @ForceInline
3349     public final long getAndBitwiseAndLongAcquire(Object o, long offset, long mask) {
3350         long current;
3351         do {
3352             // Plain read, the value is a hint, the acquire CAS does the work
3353             current = getLong(o, offset);
3354         } while (!weakCompareAndSetLongAcquire(o, offset,
3355                                                current, current & mask));
3356         return current;
3357     }
3358 
3359     @ForceInline
3360     public final long getAndBitwiseXorLong(Object o, long offset, long mask) {
3361         long current;
3362         do {
3363             current = getLongVolatile(o, offset);
3364         } while (!weakCompareAndSetLong(o, offset,
3365                                                 current, current ^ mask));
3366         return current;
3367     }
3368 
3369     @ForceInline
3370     public final long getAndBitwiseXorLongRelease(Object o, long offset, long mask) {
3371         long current;
3372         do {
3373             current = getLong(o, offset);
3374         } while (!weakCompareAndSetLongRelease(o, offset,
3375                                                current, current ^ mask));
3376         return current;
3377     }
3378 
3379     @ForceInline
3380     public final long getAndBitwiseXorLongAcquire(Object o, long offset, long mask) {
3381         long current;
3382         do {
3383             // Plain read, the value is a hint, the acquire CAS does the work
3384             current = getLong(o, offset);
3385         } while (!weakCompareAndSetLongAcquire(o, offset,
3386                                                current, current ^ mask));
3387         return current;
3388     }
3389 
3390 
3391 
3392     /**
3393      * Ensures that loads before the fence will not be reordered with loads and
3394      * stores after the fence; a "LoadLoad plus LoadStore barrier".
3395      *
3396      * Corresponds to C11 atomic_thread_fence(memory_order_acquire)
3397      * (an "acquire fence").
3398      *
3399      * Provides a LoadLoad barrier followed by a LoadStore barrier.
3400      *
3401      * @since 1.8
3402      */
3403     @IntrinsicCandidate
3404     public final void loadFence() {
3405         // If loadFence intrinsic is not available, fall back to full fence.
3406         fullFence();
3407     }
3408 
3409     /**
3410      * Ensures that loads and stores before the fence will not be reordered with
3411      * stores after the fence; a "StoreStore plus LoadStore barrier".
3412      *
3413      * Corresponds to C11 atomic_thread_fence(memory_order_release)
3414      * (a "release fence").
3415      *
3416      * Provides a StoreStore barrier followed by a LoadStore barrier.
3417      *
3418      * @since 1.8
3419      */
3420     @IntrinsicCandidate
3421     public final void storeFence() {
3422         // If storeFence intrinsic is not available, fall back to full fence.
3423         fullFence();
3424     }
3425 
3426     /**
3427      * Ensures that loads and stores before the fence will not be reordered
3428      * with loads and stores after the fence.  Implies the effects of both
3429      * loadFence() and storeFence(), and in addition, the effect of a StoreLoad
3430      * barrier.
3431      *
3432      * Corresponds to C11 atomic_thread_fence(memory_order_seq_cst).
3433      * @since 1.8
3434      */
3435     @IntrinsicCandidate
3436     public native void fullFence();
3437 
3438     /**
3439      * Ensures that loads before the fence will not be reordered with
3440      * loads after the fence.
3441      *
3442      * @implNote
3443      * This method is operationally equivalent to {@link #loadFence()}.
3444      *
3445      * @since 9
3446      */
3447     public final void loadLoadFence() {
3448         loadFence();
3449     }
3450 
3451     /**
3452      * Ensures that stores before the fence will not be reordered with
3453      * stores after the fence.
3454      *
3455      * @since 9
3456      */
3457     @IntrinsicCandidate
3458     public final void storeStoreFence() {
3459         // If storeStoreFence intrinsic is not available, fall back to storeFence.
3460         storeFence();
3461     }
3462 
3463     /**
3464      * Throws IllegalAccessError; for use by the VM for access control
3465      * error support.
3466      * @since 1.8
3467      */
3468     private static void throwIllegalAccessError() {
3469         throw new IllegalAccessError();
3470     }
3471 
3472     /**
3473      * Throws NoSuchMethodError; for use by the VM for redefinition support.
3474      * @since 13
3475      */
3476     private static void throwNoSuchMethodError() {
3477         throw new NoSuchMethodError();
3478     }
3479 
3480     /**
3481      * @return Returns true if the native byte ordering of this
3482      * platform is big-endian, false if it is little-endian.
3483      */
3484     public final boolean isBigEndian() { return BIG_ENDIAN; }
3485 
3486     /**
3487      * @return Returns true if this platform is capable of performing
3488      * accesses at addresses which are not aligned for the type of the
3489      * primitive type being accessed, false otherwise.
3490      */
3491     public final boolean unalignedAccess() { return UNALIGNED_ACCESS; }
3492 
3493     /**
3494      * Fetches a value at some byte offset into a given Java object.
3495      * More specifically, fetches a value within the given object
3496      * <code>o</code> at the given offset, or (if <code>o</code> is
3497      * null) from the memory address whose numerical value is the
3498      * given offset.  <p>
3499      *
3500      * The specification of this method is the same as {@link
3501      * #getLong(Object, long)} except that the offset does not need to
3502      * have been obtained from {@link #objectFieldOffset} on the
3503      * {@link java.lang.reflect.Field} of some Java field.  The value
3504      * in memory is raw data, and need not correspond to any Java
3505      * variable.  Unless <code>o</code> is null, the value accessed
3506      * must be entirely within the allocated object.  The endianness
3507      * of the value in memory is the endianness of the native platform.
3508      *
3509      * <p> The read will be atomic with respect to the largest power
3510      * of two that divides the GCD of the offset and the storage size.
3511      * For example, getLongUnaligned will make atomic reads of 2-, 4-,
3512      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
3513      * respectively.  There are no other guarantees of atomicity.
3514      * <p>
3515      * 8-byte atomicity is only guaranteed on platforms on which
3516      * support atomic accesses to longs.
3517      *
3518      * @param o Java heap object in which the value resides, if any, else
3519      *        null
3520      * @param offset The offset in bytes from the start of the object
3521      * @return the value fetched from the indicated object
3522      * @throws RuntimeException No defined exceptions are thrown, not even
3523      *         {@link NullPointerException}
3524      * @since 9
3525      */
3526     @IntrinsicCandidate
3527     public final long getLongUnaligned(Object o, long offset) {
3528         if ((offset & 7) == 0) {
3529             return getLong(o, offset);
3530         } else if ((offset & 3) == 0) {
3531             return makeLong(getInt(o, offset),
3532                             getInt(o, offset + 4));
3533         } else if ((offset & 1) == 0) {
3534             return makeLong(getShort(o, offset),
3535                             getShort(o, offset + 2),
3536                             getShort(o, offset + 4),
3537                             getShort(o, offset + 6));
3538         } else {
3539             return makeLong(getByte(o, offset),
3540                             getByte(o, offset + 1),
3541                             getByte(o, offset + 2),
3542                             getByte(o, offset + 3),
3543                             getByte(o, offset + 4),
3544                             getByte(o, offset + 5),
3545                             getByte(o, offset + 6),
3546                             getByte(o, offset + 7));
3547         }
3548     }
3549     /**
3550      * As {@link #getLongUnaligned(Object, long)} but with an
3551      * additional argument which specifies the endianness of the value
3552      * as stored in memory.
3553      *
3554      * @param o Java heap object in which the variable resides
3555      * @param offset The offset in bytes from the start of the object
3556      * @param bigEndian The endianness of the value
3557      * @return the value fetched from the indicated object
3558      * @since 9
3559      */
3560     public final long getLongUnaligned(Object o, long offset, boolean bigEndian) {
3561         return convEndian(bigEndian, getLongUnaligned(o, offset));
3562     }
3563 
3564     /** @see #getLongUnaligned(Object, long) */
3565     @IntrinsicCandidate
3566     public final int getIntUnaligned(Object o, long offset) {
3567         if ((offset & 3) == 0) {
3568             return getInt(o, offset);
3569         } else if ((offset & 1) == 0) {
3570             return makeInt(getShort(o, offset),
3571                            getShort(o, offset + 2));
3572         } else {
3573             return makeInt(getByte(o, offset),
3574                            getByte(o, offset + 1),
3575                            getByte(o, offset + 2),
3576                            getByte(o, offset + 3));
3577         }
3578     }
3579     /** @see #getLongUnaligned(Object, long, boolean) */
3580     public final int getIntUnaligned(Object o, long offset, boolean bigEndian) {
3581         return convEndian(bigEndian, getIntUnaligned(o, offset));
3582     }
3583 
3584     /** @see #getLongUnaligned(Object, long) */
3585     @IntrinsicCandidate
3586     public final short getShortUnaligned(Object o, long offset) {
3587         if ((offset & 1) == 0) {
3588             return getShort(o, offset);
3589         } else {
3590             return makeShort(getByte(o, offset),
3591                              getByte(o, offset + 1));
3592         }
3593     }
3594     /** @see #getLongUnaligned(Object, long, boolean) */
3595     public final short getShortUnaligned(Object o, long offset, boolean bigEndian) {
3596         return convEndian(bigEndian, getShortUnaligned(o, offset));
3597     }
3598 
3599     /** @see #getLongUnaligned(Object, long) */
3600     @IntrinsicCandidate
3601     public final char getCharUnaligned(Object o, long offset) {
3602         if ((offset & 1) == 0) {
3603             return getChar(o, offset);
3604         } else {
3605             return (char)makeShort(getByte(o, offset),
3606                                    getByte(o, offset + 1));
3607         }
3608     }
3609 
3610     /** @see #getLongUnaligned(Object, long, boolean) */
3611     public final char getCharUnaligned(Object o, long offset, boolean bigEndian) {
3612         return convEndian(bigEndian, getCharUnaligned(o, offset));
3613     }
3614 
3615     /**
3616      * Stores a value at some byte offset into a given Java object.
3617      * <p>
3618      * The specification of this method is the same as {@link
3619      * #getLong(Object, long)} except that the offset does not need to
3620      * have been obtained from {@link #objectFieldOffset} on the
3621      * {@link java.lang.reflect.Field} of some Java field.  The value
3622      * in memory is raw data, and need not correspond to any Java
3623      * variable.  The endianness of the value in memory is the
3624      * endianness of the native platform.
3625      * <p>
3626      * The write will be atomic with respect to the largest power of
3627      * two that divides the GCD of the offset and the storage size.
3628      * For example, putLongUnaligned will make atomic writes of 2-, 4-,
3629      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
3630      * respectively.  There are no other guarantees of atomicity.
3631      * <p>
3632      * 8-byte atomicity is only guaranteed on platforms on which
3633      * support atomic accesses to longs.
3634      *
3635      * @param o Java heap object in which the value resides, if any, else
3636      *        null
3637      * @param offset The offset in bytes from the start of the object
3638      * @param x the value to store
3639      * @throws RuntimeException No defined exceptions are thrown, not even
3640      *         {@link NullPointerException}
3641      * @since 9
3642      */
3643     @IntrinsicCandidate
3644     public final void putLongUnaligned(Object o, long offset, long x) {
3645         if ((offset & 7) == 0) {
3646             putLong(o, offset, x);
3647         } else if ((offset & 3) == 0) {
3648             putLongParts(o, offset,
3649                          (int)(x >> 0),
3650                          (int)(x >>> 32));
3651         } else if ((offset & 1) == 0) {
3652             putLongParts(o, offset,
3653                          (short)(x >>> 0),
3654                          (short)(x >>> 16),
3655                          (short)(x >>> 32),
3656                          (short)(x >>> 48));
3657         } else {
3658             putLongParts(o, offset,
3659                          (byte)(x >>> 0),
3660                          (byte)(x >>> 8),
3661                          (byte)(x >>> 16),
3662                          (byte)(x >>> 24),
3663                          (byte)(x >>> 32),
3664                          (byte)(x >>> 40),
3665                          (byte)(x >>> 48),
3666                          (byte)(x >>> 56));
3667         }
3668     }
3669 
3670     /**
3671      * As {@link #putLongUnaligned(Object, long, long)} but with an additional
3672      * argument which specifies the endianness of the value as stored in memory.
3673      * @param o Java heap object in which the value resides
3674      * @param offset The offset in bytes from the start of the object
3675      * @param x the value to store
3676      * @param bigEndian The endianness of the value
3677      * @throws RuntimeException No defined exceptions are thrown, not even
3678      *         {@link NullPointerException}
3679      * @since 9
3680      */
3681     public final void putLongUnaligned(Object o, long offset, long x, boolean bigEndian) {
3682         putLongUnaligned(o, offset, convEndian(bigEndian, x));
3683     }
3684 
3685     /** @see #putLongUnaligned(Object, long, long) */
3686     @IntrinsicCandidate
3687     public final void putIntUnaligned(Object o, long offset, int x) {
3688         if ((offset & 3) == 0) {
3689             putInt(o, offset, x);
3690         } else if ((offset & 1) == 0) {
3691             putIntParts(o, offset,
3692                         (short)(x >> 0),
3693                         (short)(x >>> 16));
3694         } else {
3695             putIntParts(o, offset,
3696                         (byte)(x >>> 0),
3697                         (byte)(x >>> 8),
3698                         (byte)(x >>> 16),
3699                         (byte)(x >>> 24));
3700         }
3701     }
3702     /** @see #putLongUnaligned(Object, long, long, boolean) */
3703     public final void putIntUnaligned(Object o, long offset, int x, boolean bigEndian) {
3704         putIntUnaligned(o, offset, convEndian(bigEndian, x));
3705     }
3706 
3707     /** @see #putLongUnaligned(Object, long, long) */
3708     @IntrinsicCandidate
3709     public final void putShortUnaligned(Object o, long offset, short x) {
3710         if ((offset & 1) == 0) {
3711             putShort(o, offset, x);
3712         } else {
3713             putShortParts(o, offset,
3714                           (byte)(x >>> 0),
3715                           (byte)(x >>> 8));
3716         }
3717     }
3718     /** @see #putLongUnaligned(Object, long, long, boolean) */
3719     public final void putShortUnaligned(Object o, long offset, short x, boolean bigEndian) {
3720         putShortUnaligned(o, offset, convEndian(bigEndian, x));
3721     }
3722 
3723     /** @see #putLongUnaligned(Object, long, long) */
3724     @IntrinsicCandidate
3725     public final void putCharUnaligned(Object o, long offset, char x) {
3726         putShortUnaligned(o, offset, (short)x);
3727     }
3728     /** @see #putLongUnaligned(Object, long, long, boolean) */
3729     public final void putCharUnaligned(Object o, long offset, char x, boolean bigEndian) {
3730         putCharUnaligned(o, offset, convEndian(bigEndian, x));
3731     }
3732 
3733     private static int pickPos(int top, int pos) { return BIG_ENDIAN ? top - pos : pos; }
3734 
3735     // These methods construct integers from bytes.  The byte ordering
3736     // is the native endianness of this platform.
3737     private static long makeLong(byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
3738         return ((toUnsignedLong(i0) << pickPos(56, 0))
3739               | (toUnsignedLong(i1) << pickPos(56, 8))
3740               | (toUnsignedLong(i2) << pickPos(56, 16))
3741               | (toUnsignedLong(i3) << pickPos(56, 24))
3742               | (toUnsignedLong(i4) << pickPos(56, 32))
3743               | (toUnsignedLong(i5) << pickPos(56, 40))
3744               | (toUnsignedLong(i6) << pickPos(56, 48))
3745               | (toUnsignedLong(i7) << pickPos(56, 56)));
3746     }
3747     private static long makeLong(short i0, short i1, short i2, short i3) {
3748         return ((toUnsignedLong(i0) << pickPos(48, 0))
3749               | (toUnsignedLong(i1) << pickPos(48, 16))
3750               | (toUnsignedLong(i2) << pickPos(48, 32))
3751               | (toUnsignedLong(i3) << pickPos(48, 48)));
3752     }
3753     private static long makeLong(int i0, int i1) {
3754         return (toUnsignedLong(i0) << pickPos(32, 0))
3755              | (toUnsignedLong(i1) << pickPos(32, 32));
3756     }
3757     private static int makeInt(short i0, short i1) {
3758         return (toUnsignedInt(i0) << pickPos(16, 0))
3759              | (toUnsignedInt(i1) << pickPos(16, 16));
3760     }
3761     private static int makeInt(byte i0, byte i1, byte i2, byte i3) {
3762         return ((toUnsignedInt(i0) << pickPos(24, 0))
3763               | (toUnsignedInt(i1) << pickPos(24, 8))
3764               | (toUnsignedInt(i2) << pickPos(24, 16))
3765               | (toUnsignedInt(i3) << pickPos(24, 24)));
3766     }
3767     private static short makeShort(byte i0, byte i1) {
3768         return (short)((toUnsignedInt(i0) << pickPos(8, 0))
3769                      | (toUnsignedInt(i1) << pickPos(8, 8)));
3770     }
3771 
3772     private static byte  pick(byte  le, byte  be) { return BIG_ENDIAN ? be : le; }
3773     private static short pick(short le, short be) { return BIG_ENDIAN ? be : le; }
3774     private static int   pick(int   le, int   be) { return BIG_ENDIAN ? be : le; }
3775 
3776     // These methods write integers to memory from smaller parts
3777     // provided by their caller.  The ordering in which these parts
3778     // are written is the native endianness of this platform.
3779     private void putLongParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
3780         putByte(o, offset + 0, pick(i0, i7));
3781         putByte(o, offset + 1, pick(i1, i6));
3782         putByte(o, offset + 2, pick(i2, i5));
3783         putByte(o, offset + 3, pick(i3, i4));
3784         putByte(o, offset + 4, pick(i4, i3));
3785         putByte(o, offset + 5, pick(i5, i2));
3786         putByte(o, offset + 6, pick(i6, i1));
3787         putByte(o, offset + 7, pick(i7, i0));
3788     }
3789     private void putLongParts(Object o, long offset, short i0, short i1, short i2, short i3) {
3790         putShort(o, offset + 0, pick(i0, i3));
3791         putShort(o, offset + 2, pick(i1, i2));
3792         putShort(o, offset + 4, pick(i2, i1));
3793         putShort(o, offset + 6, pick(i3, i0));
3794     }
3795     private void putLongParts(Object o, long offset, int i0, int i1) {
3796         putInt(o, offset + 0, pick(i0, i1));
3797         putInt(o, offset + 4, pick(i1, i0));
3798     }
3799     private void putIntParts(Object o, long offset, short i0, short i1) {
3800         putShort(o, offset + 0, pick(i0, i1));
3801         putShort(o, offset + 2, pick(i1, i0));
3802     }
3803     private void putIntParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3) {
3804         putByte(o, offset + 0, pick(i0, i3));
3805         putByte(o, offset + 1, pick(i1, i2));
3806         putByte(o, offset + 2, pick(i2, i1));
3807         putByte(o, offset + 3, pick(i3, i0));
3808     }
3809     private void putShortParts(Object o, long offset, byte i0, byte i1) {
3810         putByte(o, offset + 0, pick(i0, i1));
3811         putByte(o, offset + 1, pick(i1, i0));
3812     }
3813 
3814     // Zero-extend an integer
3815     private static int toUnsignedInt(byte n)    { return n & 0xff; }
3816     private static int toUnsignedInt(short n)   { return n & 0xffff; }
3817     private static long toUnsignedLong(byte n)  { return n & 0xffl; }
3818     private static long toUnsignedLong(short n) { return n & 0xffffl; }
3819     private static long toUnsignedLong(int n)   { return n & 0xffffffffl; }
3820 
3821     // Maybe byte-reverse an integer
3822     private static char convEndian(boolean big, char n)   { return big == BIG_ENDIAN ? n : Character.reverseBytes(n); }
3823     private static short convEndian(boolean big, short n) { return big == BIG_ENDIAN ? n : Short.reverseBytes(n)    ; }
3824     private static int convEndian(boolean big, int n)     { return big == BIG_ENDIAN ? n : Integer.reverseBytes(n)  ; }
3825     private static long convEndian(boolean big, long n)   { return big == BIG_ENDIAN ? n : Long.reverseBytes(n)     ; }
3826 
3827 
3828 
3829     private native long allocateMemory0(long bytes);
3830     private native long reallocateMemory0(long address, long bytes);
3831     private native void freeMemory0(long address);
3832     @IntrinsicCandidate
3833     private native void setMemory0(Object o, long offset, long bytes, byte value);
3834     @IntrinsicCandidate
3835     private native void copyMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
3836     private native void copySwapMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes, long elemSize);
3837     private native long objectFieldOffset0(Field f);
3838     private native long objectFieldOffset1(Class<?> c, String name);
3839     private native long staticFieldOffset0(Field f);
3840     private native Object staticFieldBase0(Field f);
3841     private native boolean shouldBeInitialized0(Class<?> c);
3842     private native void ensureClassInitialized0(Class<?> c);
3843     private native int arrayBaseOffset0(Class<?> arrayClass);
3844     private native int arrayIndexScale0(Class<?> arrayClass);
3845     private native int getLoadAverage0(double[] loadavg, int nelems);
3846 
3847 
3848     /**
3849      * Invokes the given direct byte buffer's cleaner, if any.
3850      *
3851      * @param directBuffer a direct byte buffer
3852      * @throws NullPointerException     if {@code directBuffer} is null
3853      * @throws IllegalArgumentException if {@code directBuffer} is non-direct,
3854      *                                  or is a {@link java.nio.Buffer#slice slice}, or is a
3855      *                                  {@link java.nio.Buffer#duplicate duplicate}
3856      */
3857     public void invokeCleaner(java.nio.ByteBuffer directBuffer) {
3858         if (!directBuffer.isDirect())
3859             throw new IllegalArgumentException("buffer is non-direct");
3860 
3861         DirectBuffer db = (DirectBuffer) directBuffer;
3862         if (db.attachment() != null)
3863             throw new IllegalArgumentException("duplicate or slice");
3864 
3865         Cleaner cleaner = db.cleaner();
3866         if (cleaner != null) {
3867             cleaner.clean();
3868         }
3869     }
3870 }