1 /*
   2  * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.internal.misc;
  27 
  28 import jdk.internal.value.ValueClass;
  29 import jdk.internal.vm.annotation.AOTRuntimeSetup;
  30 import jdk.internal.vm.annotation.AOTSafeClassInitializer;
  31 import jdk.internal.vm.annotation.ForceInline;
  32 import jdk.internal.vm.annotation.IntrinsicCandidate;
  33 import sun.nio.Cleaner;
  34 import sun.nio.ch.DirectBuffer;
  35 
  36 import java.lang.reflect.Field;
  37 import java.security.ProtectionDomain;
  38 
  39 import static jdk.internal.misc.UnsafeConstants.*;
  40 
  41 /**
  42  * A collection of methods for performing low-level, unsafe operations.
  43  * Although the class and all methods are public, use of this class is
  44  * limited because only trusted code can obtain instances of it.
  45  *
  46  * <em>Note:</em> It is the responsibility of the caller to make sure
  47  * arguments are checked before methods of this class are
  48  * called. While some rudimentary checks are performed on the input,
  49  * the checks are best effort and when performance is an overriding
  50  * priority, as when methods of this class are optimized by the
  51  * runtime compiler, some or all checks (if any) may be elided. Hence,
  52  * the caller must not rely on the checks and corresponding
  53  * exceptions!
  54  *
  55  * @author John R. Rose
  56  * @see #getUnsafe
  57  */
  58 @AOTSafeClassInitializer
  59 public final class Unsafe {
  60 
  61     private static native void registerNatives();
  62     static {
  63         runtimeSetup();
  64     }
  65 
  66     /// BASE_OFFSET, INDEX_SCALE, and ADDRESS_SIZE fields are equivalent if the
  67     /// AOT initialized heap is reused, so just register natives
  68     @AOTRuntimeSetup
  69     private static void runtimeSetup() {
  70         registerNatives();
  71     }
  72 
  73     private Unsafe() {}
  74 
  75     private static final Unsafe theUnsafe = new Unsafe();
  76 
  77     /**
  78      * Provides the caller with the capability of performing unsafe
  79      * operations.
  80      *
  81      * <p>The returned {@code Unsafe} object should be carefully guarded
  82      * by the caller, since it can be used to read and write data at arbitrary
  83      * memory addresses.  It must never be passed to untrusted code.
  84      *
  85      * <p>Most methods in this class are very low-level, and correspond to a
  86      * small number of hardware instructions (on typical machines).  Compilers
  87      * are encouraged to optimize these methods accordingly.
  88      *
  89      * <p>Here is a suggested idiom for using unsafe operations:
  90      *
  91      * <pre> {@code
  92      * class MyTrustedClass {
  93      *   private static final Unsafe unsafe = Unsafe.getUnsafe();
  94      *   ...
  95      *   private long myCountAddress = ...;
  96      *   public int getCount() { return unsafe.getByte(myCountAddress); }
  97      * }}</pre>
  98      *
  99      * (It may assist compilers to make the local variable {@code final}.)
 100      */
 101     public static Unsafe getUnsafe() {
 102         return theUnsafe;
 103     }
 104 
 105     //--- peek and poke operations
 106     // (compilers should optimize these to memory ops)
 107 
 108     // These work on object fields in the Java heap.
 109     // They will not work on elements of packed arrays.
 110 
 111     /**
 112      * Fetches a value from a given Java variable.
 113      * More specifically, fetches a field or array element within the given
 114      * object {@code o} at the given offset, or (if {@code o} is null)
 115      * from the memory address whose numerical value is the given offset.
 116      * <p>
 117      * The results are undefined unless one of the following cases is true:
 118      * <ul>
 119      * <li>The offset was obtained from {@link #objectFieldOffset} on
 120      * the {@link java.lang.reflect.Field} of some Java field and the object
 121      * referred to by {@code o} is of a class compatible with that
 122      * field's class.
 123      *
 124      * <li>The offset and object reference {@code o} (either null or
 125      * non-null) were both obtained via {@link #staticFieldOffset}
 126      * and {@link #staticFieldBase} (respectively) from the
 127      * reflective {@link Field} representation of some Java field.
 128      *
 129      * <li>The object referred to by {@code o} is an array, and the offset
 130      * is an integer of the form {@code B+N*S}, where {@code N} is
 131      * a valid index into the array, and {@code B} and {@code S} are
 132      * the values obtained by {@link #arrayBaseOffset} and {@link
 133      * #arrayIndexScale} (respectively) from the array's class.  The value
 134      * referred to is the {@code N}<em>th</em> element of the array.
 135      *
 136      * </ul>
 137      * <p>
 138      * If one of the above cases is true, the call references a specific Java
 139      * variable (field or array element).  However, the results are undefined
 140      * if that variable is not in fact of the type returned by this method.
 141      * <p>
 142      * This method refers to a variable by means of two parameters, and so
 143      * it provides (in effect) a <em>double-register</em> addressing mode
 144      * for Java variables.  When the object reference is null, this method
 145      * uses its offset as an absolute address.  This is similar in operation
 146      * to methods such as {@link #getInt(long)}, which provide (in effect) a
 147      * <em>single-register</em> addressing mode for non-Java variables.
 148      * However, because Java variables may have a different layout in memory
 149      * from non-Java variables, programmers should not assume that these
 150      * two addressing modes are ever equivalent.  Also, programmers should
 151      * remember that offsets from the double-register addressing mode cannot
 152      * be portably confused with longs used in the single-register addressing
 153      * mode.
 154      *
 155      * @param o Java heap object in which the variable resides, if any, else
 156      *        null
 157      * @param offset indication of where the variable resides in a Java heap
 158      *        object, if any, else a memory address locating the variable
 159      *        statically
 160      * @return the value fetched from the indicated Java variable
 161      * @throws RuntimeException No defined exceptions are thrown, not even
 162      *         {@link NullPointerException}
 163      */
 164     @IntrinsicCandidate
 165     public native int getInt(Object o, long offset);
 166 
 167     /**
 168      * Stores a value into a given Java variable.
 169      * <p>
 170      * The first two parameters are interpreted exactly as with
 171      * {@link #getInt(Object, long)} to refer to a specific
 172      * Java variable (field or array element).  The given value
 173      * is stored into that variable.
 174      * <p>
 175      * The variable must be of the same type as the method
 176      * parameter {@code x}.
 177      *
 178      * @param o Java heap object in which the variable resides, if any, else
 179      *        null
 180      * @param offset indication of where the variable resides in a Java heap
 181      *        object, if any, else a memory address locating the variable
 182      *        statically
 183      * @param x the value to store into the indicated Java variable
 184      * @throws RuntimeException No defined exceptions are thrown, not even
 185      *         {@link NullPointerException}
 186      */
 187     @IntrinsicCandidate
 188     public native void putInt(Object o, long offset, int x);
 189 
 190 
 191     /**
 192      * Returns true if the given field is flattened.
 193      */
 194     public boolean isFlatField(Field f) {
 195         if (f == null) {
 196             throw new NullPointerException();
 197         }
 198         return isFlatField0(f);
 199     }
 200 
 201     private native boolean isFlatField0(Object o);
 202 
 203     /* Returns true if the given field has a null marker
 204      * <p>
 205      * Nullable flat fields are stored in a flattened representation
 206      * and have an associated null marker to indicate if the the field value is
 207      * null or the one stored with the flat representation
 208      */
 209 
 210     public boolean hasNullMarker(Field f) {
 211         if (f == null) {
 212             throw new NullPointerException();
 213         }
 214         return hasNullMarker0(f);
 215     }
 216 
 217     private native boolean hasNullMarker0(Object o);
 218 
 219     /* Returns the offset of the null marker of the field,
 220     * or -1 if the field doesn't have a null marker
 221     */
 222 
 223     public int nullMarkerOffset(Field f) {
 224         if (f == null) {
 225             throw new NullPointerException();
 226         }
 227         return nullMarkerOffset0(f);
 228     }
 229 
 230     private native int nullMarkerOffset0(Object o);
 231 
 232     public static final int NON_FLAT_LAYOUT = 0;
 233 
 234     /* Reports the kind of layout used for an element in the storage
 235      * allocation of the given array. Do not expect to perform any logic
 236      * or layout control with this value, it is just an opaque token
 237      * used for performance reasons.
 238      *
 239      * A layout of 0 indicates this array is not flat.
 240      */
 241     public int arrayLayout(Object[] array) {
 242         if (array == null) {
 243             throw new NullPointerException();
 244         }
 245         return arrayLayout0(array);
 246     }
 247 
 248     private native int arrayLayout0(Object[] array);
 249 
 250 
 251     /* Reports the kind of layout used for a given field in the storage
 252      * allocation of its class.  Do not expect to perform any logic
 253      * or layout control with this value, it is just an opaque token
 254      * used for performance reasons.
 255      *
 256      * A layout of 0 indicates this field is not flat.
 257      */
 258     public int fieldLayout(Field f) {
 259         if (f == null) {
 260             throw new NullPointerException();
 261         }
 262         return fieldLayout0(f);
 263     }
 264 
 265     private native int fieldLayout0(Object o);
 266 
 267     public native Object[] newSpecialArray(Class<?> componentType,
 268                                                   int length, int layoutKind);
 269 
 270     /**
 271      * Fetches a reference value from a given Java variable.
 272      * This method can return a reference to either an object or value
 273      * or a null reference.
 274      *
 275      * @see #getInt(Object, long)
 276      */
 277     @IntrinsicCandidate
 278     public native Object getReference(Object o, long offset);
 279 
 280     /**
 281      * Stores a reference value into a given Java variable.
 282      * This method can store a reference to either an object or value
 283      * or a null reference.
 284      * <p>
 285      * Unless the reference {@code x} being stored is either null
 286      * or matches the field type, the results are undefined.
 287      * If the reference {@code o} is non-null, card marks or
 288      * other store barriers for that object (if the VM requires them)
 289      * are updated.
 290      * @see #putInt(Object, long, int)
 291      */
 292     @IntrinsicCandidate
 293     public native void putReference(Object o, long offset, Object x);
 294 
 295     /**
 296      * Fetches a value of type {@code <V>} from a given Java variable.
 297      * More specifically, fetches a field or array element within the given
 298      * {@code o} object at the given offset, or (if {@code o} is null)
 299      * from the memory address whose numerical value is the given offset.
 300      *
 301      * @param o Java heap object in which the variable resides, if any, else
 302      *        null
 303      * @param offset indication of where the variable resides in a Java heap
 304      *        object, if any, else a memory address locating the variable
 305      *        statically
 306      * @param valueType value type
 307      * @param <V> the type of a value
 308      * @return the value fetched from the indicated Java variable
 309      * @throws RuntimeException No defined exceptions are thrown, not even
 310      *         {@link NullPointerException}
 311      */
 312     @IntrinsicCandidate
 313     public native <V> V getValue(Object o, long offset, Class<?> valueType);
 314 
 315     /**
 316      * Fetches a value of type {@code <V>} from a given Java variable.
 317      * More specifically, fetches a field or array element within the given
 318      * {@code o} object at the given offset, or (if {@code o} is null)
 319      * from the memory address whose numerical value is the given offset.
 320      *
 321      * @param o Java heap object in which the variable resides, if any, else
 322      *        null
 323      * @param offset indication of where the variable resides in a Java heap
 324      *        object, if any, else a memory address locating the variable
 325      *        statically
 326      * @param layoutKind opaque value used by the VM to know the layout
 327      *        the field or array element. This value must be retrieved with
 328      *        {@link #fieldLayout} or {@link #arrayLayout}.
 329      * @param valueType value type
 330      * @param <V> the type of a value
 331      * @return the value fetched from the indicated Java variable
 332      * @throws RuntimeException No defined exceptions are thrown, not even
 333      *         {@link NullPointerException}
 334      */
 335     @IntrinsicCandidate
 336     public native <V> V getFlatValue(Object o, long offset, int layoutKind, Class<?> valueType);
 337 
 338 
 339     /**
 340      * Stores the given value into a given Java variable.
 341      *
 342      * Unless the reference {@code o} being stored is either null
 343      * or matches the field type, the results are undefined.
 344      *
 345      * @param o Java heap object in which the variable resides, if any, else
 346      *        null
 347      * @param offset indication of where the variable resides in a Java heap
 348      *        object, if any, else a memory address locating the variable
 349      *        statically
 350      * @param valueType value type
 351      * @param v the value to store into the indicated Java variable
 352      * @param <V> the type of a value
 353      * @throws RuntimeException No defined exceptions are thrown, not even
 354      *         {@link NullPointerException}
 355      */
 356     @IntrinsicCandidate
 357     public native <V> void putValue(Object o, long offset, Class<?> valueType, V v);
 358 
 359     /**
 360      * Stores the given value into a given Java variable.
 361      *
 362      * Unless the reference {@code o} being stored is either null
 363      * or matches the field type, the results are undefined.
 364      *
 365      * @param o Java heap object in which the variable resides, if any, else
 366      *        null
 367      * @param offset indication of where the variable resides in a Java heap
 368      *        object, if any, else a memory address locating the variable
 369      *        statically
 370      * @param layoutKind opaque value used by the VM to know the layout
 371      *        the field or array element. This value must be retrieved with
 372      *        {@link #fieldLayout} or {@link #arrayLayout}.
 373      * @param valueType value type
 374      * @param v the value to store into the indicated Java variable
 375      * @param <V> the type of a value
 376      * @throws RuntimeException No defined exceptions are thrown, not even
 377      *         {@link NullPointerException}
 378      */
 379     @IntrinsicCandidate
 380     public native <V> void putFlatValue(Object o, long offset, int layoutKind, Class<?> valueType, V v);
 381 
 382     /**
 383      * Returns an object instance with a private buffered value whose layout
 384      * and contents is exactly the given value instance.  The return object
 385      * is in the larval state that can be updated using the unsafe put operation.
 386      *
 387      * @param value a value instance
 388      * @param <V> the type of the given value instance
 389      */
 390     @IntrinsicCandidate
 391     public native <V> V makePrivateBuffer(V value);
 392 
 393     /**
 394      * Exits the larval state and returns a value instance.
 395      *
 396      * @param value a value instance
 397      * @param <V> the type of the given value instance
 398      */
 399     @IntrinsicCandidate
 400     public native <V> V finishPrivateBuffer(V value);
 401 
 402     /**
 403      * Returns the header size of the given value type.
 404      *
 405      * @param valueType value type
 406      * @return the header size of the value type
 407      */
 408     public native <V> long valueHeaderSize(Class<V> valueType);
 409 
 410     /** @see #getInt(Object, long) */
 411     @IntrinsicCandidate
 412     public native boolean getBoolean(Object o, long offset);
 413 
 414     /** @see #putInt(Object, long, int) */
 415     @IntrinsicCandidate
 416     public native void    putBoolean(Object o, long offset, boolean x);
 417 
 418     /** @see #getInt(Object, long) */
 419     @IntrinsicCandidate
 420     public native byte    getByte(Object o, long offset);
 421 
 422     /** @see #putInt(Object, long, int) */
 423     @IntrinsicCandidate
 424     public native void    putByte(Object o, long offset, byte x);
 425 
 426     /** @see #getInt(Object, long) */
 427     @IntrinsicCandidate
 428     public native short   getShort(Object o, long offset);
 429 
 430     /** @see #putInt(Object, long, int) */
 431     @IntrinsicCandidate
 432     public native void    putShort(Object o, long offset, short x);
 433 
 434     /** @see #getInt(Object, long) */
 435     @IntrinsicCandidate
 436     public native char    getChar(Object o, long offset);
 437 
 438     /** @see #putInt(Object, long, int) */
 439     @IntrinsicCandidate
 440     public native void    putChar(Object o, long offset, char x);
 441 
 442     /** @see #getInt(Object, long) */
 443     @IntrinsicCandidate
 444     public native long    getLong(Object o, long offset);
 445 
 446     /** @see #putInt(Object, long, int) */
 447     @IntrinsicCandidate
 448     public native void    putLong(Object o, long offset, long x);
 449 
 450     /** @see #getInt(Object, long) */
 451     @IntrinsicCandidate
 452     public native float   getFloat(Object o, long offset);
 453 
 454     /** @see #putInt(Object, long, int) */
 455     @IntrinsicCandidate
 456     public native void    putFloat(Object o, long offset, float x);
 457 
 458     /** @see #getInt(Object, long) */
 459     @IntrinsicCandidate
 460     public native double  getDouble(Object o, long offset);
 461 
 462     /** @see #putInt(Object, long, int) */
 463     @IntrinsicCandidate
 464     public native void    putDouble(Object o, long offset, double x);
 465 
 466     /**
 467      * Fetches a native pointer from a given memory address.  If the address is
 468      * zero, or does not point into a block obtained from {@link
 469      * #allocateMemory}, the results are undefined.
 470      *
 471      * <p>If the native pointer is less than 64 bits wide, it is extended as
 472      * an unsigned number to a Java long.  The pointer may be indexed by any
 473      * given byte offset, simply by adding that offset (as a simple integer) to
 474      * the long representing the pointer.  The number of bytes actually read
 475      * from the target address may be determined by consulting {@link
 476      * #addressSize}.
 477      *
 478      * @see #allocateMemory
 479      * @see #getInt(Object, long)
 480      */
 481     @ForceInline
 482     public long getAddress(Object o, long offset) {
 483         if (ADDRESS_SIZE == 4) {
 484             return Integer.toUnsignedLong(getInt(o, offset));
 485         } else {
 486             return getLong(o, offset);
 487         }
 488     }
 489 
 490     /**
 491      * Stores a native pointer into a given memory address.  If the address is
 492      * zero, or does not point into a block obtained from {@link
 493      * #allocateMemory}, the results are undefined.
 494      *
 495      * <p>The number of bytes actually written at the target address may be
 496      * determined by consulting {@link #addressSize}.
 497      *
 498      * @see #allocateMemory
 499      * @see #putInt(Object, long, int)
 500      */
 501     @ForceInline
 502     public void putAddress(Object o, long offset, long x) {
 503         if (ADDRESS_SIZE == 4) {
 504             putInt(o, offset, (int)x);
 505         } else {
 506             putLong(o, offset, x);
 507         }
 508     }
 509 
 510     // These read VM internal data.
 511 
 512     /**
 513      * Fetches an uncompressed reference value from a given native variable
 514      * ignoring the VM's compressed references mode.
 515      *
 516      * @param address a memory address locating the variable
 517      * @return the value fetched from the indicated native variable
 518      */
 519     public native Object getUncompressedObject(long address);
 520 
 521     // These work on values in the C heap.
 522 
 523     /**
 524      * Fetches a value from a given memory address.  If the address is zero, or
 525      * does not point into a block obtained from {@link #allocateMemory}, the
 526      * results are undefined.
 527      *
 528      * @see #allocateMemory
 529      */
 530     @ForceInline
 531     public byte getByte(long address) {
 532         return getByte(null, address);
 533     }
 534 
 535     /**
 536      * Stores a value into a given memory address.  If the address is zero, or
 537      * does not point into a block obtained from {@link #allocateMemory}, the
 538      * results are undefined.
 539      *
 540      * @see #getByte(long)
 541      */
 542     @ForceInline
 543     public void putByte(long address, byte x) {
 544         putByte(null, address, x);
 545     }
 546 
 547     /** @see #getByte(long) */
 548     @ForceInline
 549     public short getShort(long address) {
 550         return getShort(null, address);
 551     }
 552 
 553     /** @see #putByte(long, byte) */
 554     @ForceInline
 555     public void putShort(long address, short x) {
 556         putShort(null, address, x);
 557     }
 558 
 559     /** @see #getByte(long) */
 560     @ForceInline
 561     public char getChar(long address) {
 562         return getChar(null, address);
 563     }
 564 
 565     /** @see #putByte(long, byte) */
 566     @ForceInline
 567     public void putChar(long address, char x) {
 568         putChar(null, address, x);
 569     }
 570 
 571     /** @see #getByte(long) */
 572     @ForceInline
 573     public int getInt(long address) {
 574         return getInt(null, address);
 575     }
 576 
 577     /** @see #putByte(long, byte) */
 578     @ForceInline
 579     public void putInt(long address, int x) {
 580         putInt(null, address, x);
 581     }
 582 
 583     /** @see #getByte(long) */
 584     @ForceInline
 585     public long getLong(long address) {
 586         return getLong(null, address);
 587     }
 588 
 589     /** @see #putByte(long, byte) */
 590     @ForceInline
 591     public void putLong(long address, long x) {
 592         putLong(null, address, x);
 593     }
 594 
 595     /** @see #getByte(long) */
 596     @ForceInline
 597     public float getFloat(long address) {
 598         return getFloat(null, address);
 599     }
 600 
 601     /** @see #putByte(long, byte) */
 602     @ForceInline
 603     public void putFloat(long address, float x) {
 604         putFloat(null, address, x);
 605     }
 606 
 607     /** @see #getByte(long) */
 608     @ForceInline
 609     public double getDouble(long address) {
 610         return getDouble(null, address);
 611     }
 612 
 613     /** @see #putByte(long, byte) */
 614     @ForceInline
 615     public void putDouble(long address, double x) {
 616         putDouble(null, address, x);
 617     }
 618 
 619     /** @see #getAddress(Object, long) */
 620     @ForceInline
 621     public long getAddress(long address) {
 622         return getAddress(null, address);
 623     }
 624 
 625     /** @see #putAddress(Object, long, long) */
 626     @ForceInline
 627     public void putAddress(long address, long x) {
 628         putAddress(null, address, x);
 629     }
 630 
 631 
 632 
 633     //--- helper methods for validating various types of objects/values
 634 
 635     /**
 636      * Create an exception reflecting that some of the input was invalid
 637      *
 638      * <em>Note:</em> It is the responsibility of the caller to make
 639      * sure arguments are checked before the methods are called. While
 640      * some rudimentary checks are performed on the input, the checks
 641      * are best effort and when performance is an overriding priority,
 642      * as when methods of this class are optimized by the runtime
 643      * compiler, some or all checks (if any) may be elided. Hence, the
 644      * caller must not rely on the checks and corresponding
 645      * exceptions!
 646      *
 647      * @return an exception object
 648      */
 649     private RuntimeException invalidInput() {
 650         return new IllegalArgumentException();
 651     }
 652 
 653     /**
 654      * Check if a value is 32-bit clean (32 MSB are all zero)
 655      *
 656      * @param value the 64-bit value to check
 657      *
 658      * @return true if the value is 32-bit clean
 659      */
 660     private boolean is32BitClean(long value) {
 661         return value >>> 32 == 0;
 662     }
 663 
 664     /**
 665      * Check the validity of a size (the equivalent of a size_t)
 666      *
 667      * @throws RuntimeException if the size is invalid
 668      *         (<em>Note:</em> after optimization, invalid inputs may
 669      *         go undetected, which will lead to unpredictable
 670      *         behavior)
 671      */
 672     private void checkSize(long size) {
 673         if (ADDRESS_SIZE == 4) {
 674             // Note: this will also check for negative sizes
 675             if (!is32BitClean(size)) {
 676                 throw invalidInput();
 677             }
 678         } else if (size < 0) {
 679             throw invalidInput();
 680         }
 681     }
 682 
 683     /**
 684      * Check the validity of a native address (the equivalent of void*)
 685      *
 686      * @throws RuntimeException if the address is invalid
 687      *         (<em>Note:</em> after optimization, invalid inputs may
 688      *         go undetected, which will lead to unpredictable
 689      *         behavior)
 690      */
 691     private void checkNativeAddress(long address) {
 692         if (ADDRESS_SIZE == 4) {
 693             // Accept both zero and sign extended pointers. A valid
 694             // pointer will, after the +1 below, either have produced
 695             // the value 0x0 or 0x1. Masking off the low bit allows
 696             // for testing against 0.
 697             if ((((address >> 32) + 1) & ~1) != 0) {
 698                 throw invalidInput();
 699             }
 700         }
 701     }
 702 
 703     /**
 704      * Check the validity of an offset, relative to a base object
 705      *
 706      * @param o the base object
 707      * @param offset the offset to check
 708      *
 709      * @throws RuntimeException if the size is invalid
 710      *         (<em>Note:</em> after optimization, invalid inputs may
 711      *         go undetected, which will lead to unpredictable
 712      *         behavior)
 713      */
 714     private void checkOffset(Object o, long offset) {
 715         if (ADDRESS_SIZE == 4) {
 716             // Note: this will also check for negative offsets
 717             if (!is32BitClean(offset)) {
 718                 throw invalidInput();
 719             }
 720         } else if (offset < 0) {
 721             throw invalidInput();
 722         }
 723     }
 724 
 725     /**
 726      * Check the validity of a double-register pointer
 727      *
 728      * Note: This code deliberately does *not* check for NPE for (at
 729      * least) three reasons:
 730      *
 731      * 1) NPE is not just NULL/0 - there is a range of values all
 732      * resulting in an NPE, which is not trivial to check for
 733      *
 734      * 2) It is the responsibility of the callers of Unsafe methods
 735      * to verify the input, so throwing an exception here is not really
 736      * useful - passing in a NULL pointer is a critical error and the
 737      * must not expect an exception to be thrown anyway.
 738      *
 739      * 3) the actual operations will detect NULL pointers anyway by
 740      * means of traps and signals (like SIGSEGV).
 741      *
 742      * @param o Java heap object, or null
 743      * @param offset indication of where the variable resides in a Java heap
 744      *        object, if any, else a memory address locating the variable
 745      *        statically
 746      *
 747      * @throws RuntimeException if the pointer is invalid
 748      *         (<em>Note:</em> after optimization, invalid inputs may
 749      *         go undetected, which will lead to unpredictable
 750      *         behavior)
 751      */
 752     private void checkPointer(Object o, long offset) {
 753         if (o == null) {
 754             checkNativeAddress(offset);
 755         } else {
 756             checkOffset(o, offset);
 757         }
 758     }
 759 
 760     /**
 761      * Check if a type is a primitive array type
 762      *
 763      * @param c the type to check
 764      *
 765      * @return true if the type is a primitive array type
 766      */
 767     private void checkPrimitiveArray(Class<?> c) {
 768         Class<?> componentType = c.getComponentType();
 769         if (componentType == null || !componentType.isPrimitive()) {
 770             throw invalidInput();
 771         }
 772     }
 773 
 774     /**
 775      * Check that a pointer is a valid primitive array type pointer
 776      *
 777      * Note: pointers off-heap are considered to be primitive arrays
 778      *
 779      * @throws RuntimeException if the pointer is invalid
 780      *         (<em>Note:</em> after optimization, invalid inputs may
 781      *         go undetected, which will lead to unpredictable
 782      *         behavior)
 783      */
 784     private void checkPrimitivePointer(Object o, long offset) {
 785         checkPointer(o, offset);
 786 
 787         if (o != null) {
 788             // If on heap, it must be a primitive array
 789             checkPrimitiveArray(o.getClass());
 790         }
 791     }
 792 
 793 
 794     //--- wrappers for malloc, realloc, free:
 795 
 796     /**
 797      * Round up allocation size to a multiple of HeapWordSize.
 798      */
 799     private long alignToHeapWordSize(long bytes) {
 800         if (bytes >= 0) {
 801             return (bytes + ADDRESS_SIZE - 1) & ~(ADDRESS_SIZE - 1);
 802         } else {
 803             throw invalidInput();
 804         }
 805     }
 806 
 807     /**
 808      * Allocates a new block of native memory, of the given size in bytes.  The
 809      * contents of the memory are uninitialized; they will generally be
 810      * garbage.  The resulting native pointer will be zero if and only if the
 811      * requested size is zero.  The resulting native pointer will be aligned for
 812      * all value types.   Dispose of this memory by calling {@link #freeMemory}
 813      * or resize it with {@link #reallocateMemory}.
 814      *
 815      * <em>Note:</em> It is the responsibility of the caller to make
 816      * sure arguments are checked before the methods are called. While
 817      * some rudimentary checks are performed on the input, the checks
 818      * are best effort and when performance is an overriding priority,
 819      * as when methods of this class are optimized by the runtime
 820      * compiler, some or all checks (if any) may be elided. Hence, the
 821      * caller must not rely on the checks and corresponding
 822      * exceptions!
 823      *
 824      * @throws RuntimeException if the size is negative or too large
 825      *         for the native size_t type
 826      *
 827      * @throws OutOfMemoryError if the allocation is refused by the system
 828      *
 829      * @see #getByte(long)
 830      * @see #putByte(long, byte)
 831      */
 832     public long allocateMemory(long bytes) {
 833         bytes = alignToHeapWordSize(bytes);
 834 
 835         allocateMemoryChecks(bytes);
 836 
 837         if (bytes == 0) {
 838             return 0;
 839         }
 840 
 841         long p = allocateMemory0(bytes);
 842         if (p == 0) {
 843             throw new OutOfMemoryError("Unable to allocate " + bytes + " bytes");
 844         }
 845 
 846         return p;
 847     }
 848 
 849     /**
 850      * Validate the arguments to allocateMemory
 851      *
 852      * @throws RuntimeException if the arguments are invalid
 853      *         (<em>Note:</em> after optimization, invalid inputs may
 854      *         go undetected, which will lead to unpredictable
 855      *         behavior)
 856      */
 857     private void allocateMemoryChecks(long bytes) {
 858         checkSize(bytes);
 859     }
 860 
 861     /**
 862      * Resizes a new block of native memory, to the given size in bytes.  The
 863      * contents of the new block past the size of the old block are
 864      * uninitialized; they will generally be garbage.  The resulting native
 865      * pointer will be zero if and only if the requested size is zero.  The
 866      * resulting native pointer will be aligned for all value types.  Dispose
 867      * of this memory by calling {@link #freeMemory}, or resize it with {@link
 868      * #reallocateMemory}.  The address passed to this method may be null, in
 869      * which case an allocation will be performed.
 870      *
 871      * <em>Note:</em> It is the responsibility of the caller to make
 872      * sure arguments are checked before the methods are called. While
 873      * some rudimentary checks are performed on the input, the checks
 874      * are best effort and when performance is an overriding priority,
 875      * as when methods of this class are optimized by the runtime
 876      * compiler, some or all checks (if any) may be elided. Hence, the
 877      * caller must not rely on the checks and corresponding
 878      * exceptions!
 879      *
 880      * @throws RuntimeException if the size is negative or too large
 881      *         for the native size_t type
 882      *
 883      * @throws OutOfMemoryError if the allocation is refused by the system
 884      *
 885      * @see #allocateMemory
 886      */
 887     public long reallocateMemory(long address, long bytes) {
 888         bytes = alignToHeapWordSize(bytes);
 889 
 890         reallocateMemoryChecks(address, bytes);
 891 
 892         if (bytes == 0) {
 893             freeMemory(address);
 894             return 0;
 895         }
 896 
 897         long p = (address == 0) ? allocateMemory0(bytes) : reallocateMemory0(address, bytes);
 898         if (p == 0) {
 899             throw new OutOfMemoryError("Unable to allocate " + bytes + " bytes");
 900         }
 901 
 902         return p;
 903     }
 904 
 905     /**
 906      * Validate the arguments to reallocateMemory
 907      *
 908      * @throws RuntimeException if the arguments are invalid
 909      *         (<em>Note:</em> after optimization, invalid inputs may
 910      *         go undetected, which will lead to unpredictable
 911      *         behavior)
 912      */
 913     private void reallocateMemoryChecks(long address, long bytes) {
 914         checkPointer(null, address);
 915         checkSize(bytes);
 916     }
 917 
 918     /**
 919      * Sets all bytes in a given block of memory to a fixed value
 920      * (usually zero).
 921      *
 922      * <p>This method determines a block's base address by means of two parameters,
 923      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 924      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 925      * the offset supplies an absolute base address.
 926      *
 927      * <p>The stores are in coherent (atomic) units of a size determined
 928      * by the address and length parameters.  If the effective address and
 929      * length are all even modulo 8, the stores take place in 'long' units.
 930      * If the effective address and length are (resp.) even modulo 4 or 2,
 931      * the stores take place in units of 'int' or 'short'.
 932      *
 933      * <em>Note:</em> It is the responsibility of the caller to make
 934      * sure arguments are checked before the methods are called. While
 935      * some rudimentary checks are performed on the input, the checks
 936      * are best effort and when performance is an overriding priority,
 937      * as when methods of this class are optimized by the runtime
 938      * compiler, some or all checks (if any) may be elided. Hence, the
 939      * caller must not rely on the checks and corresponding
 940      * exceptions!
 941      *
 942      * @throws RuntimeException if any of the arguments is invalid
 943      *
 944      * @since 1.7
 945      */
 946     public void setMemory(Object o, long offset, long bytes, byte value) {
 947         setMemoryChecks(o, offset, bytes, value);
 948 
 949         if (bytes == 0) {
 950             return;
 951         }
 952 
 953         setMemory0(o, offset, bytes, value);
 954     }
 955 
 956     /**
 957      * Sets all bytes in a given block of memory to a fixed value
 958      * (usually zero).  This provides a <em>single-register</em> addressing mode,
 959      * as discussed in {@link #getInt(Object,long)}.
 960      *
 961      * <p>Equivalent to {@code setMemory(null, address, bytes, value)}.
 962      */
 963     public void setMemory(long address, long bytes, byte value) {
 964         setMemory(null, address, bytes, value);
 965     }
 966 
 967     /**
 968      * Validate the arguments to setMemory
 969      *
 970      * @throws RuntimeException if the arguments are invalid
 971      *         (<em>Note:</em> after optimization, invalid inputs may
 972      *         go undetected, which will lead to unpredictable
 973      *         behavior)
 974      */
 975     private void setMemoryChecks(Object o, long offset, long bytes, byte value) {
 976         checkPrimitivePointer(o, offset);
 977         checkSize(bytes);
 978     }
 979 
 980     /**
 981      * Sets all bytes in a given block of memory to a copy of another
 982      * block.
 983      *
 984      * <p>This method determines each block's base address by means of two parameters,
 985      * and so it provides (in effect) a <em>double-register</em> addressing mode,
 986      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
 987      * the offset supplies an absolute base address.
 988      *
 989      * <p>The transfers are in coherent (atomic) units of a size determined
 990      * by the address and length parameters.  If the effective addresses and
 991      * length are all even modulo 8, the transfer takes place in 'long' units.
 992      * If the effective addresses and length are (resp.) even modulo 4 or 2,
 993      * the transfer takes place in units of 'int' or 'short'.
 994      *
 995      * <em>Note:</em> It is the responsibility of the caller to make
 996      * sure arguments are checked before the methods are called. While
 997      * some rudimentary checks are performed on the input, the checks
 998      * are best effort and when performance is an overriding priority,
 999      * as when methods of this class are optimized by the runtime
1000      * compiler, some or all checks (if any) may be elided. Hence, the
1001      * caller must not rely on the checks and corresponding
1002      * exceptions!
1003      *
1004      * @throws RuntimeException if any of the arguments is invalid
1005      *
1006      * @since 1.7
1007      */
1008     public void copyMemory(Object srcBase, long srcOffset,
1009                            Object destBase, long destOffset,
1010                            long bytes) {
1011         copyMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes);
1012 
1013         if (bytes == 0) {
1014             return;
1015         }
1016 
1017         copyMemory0(srcBase, srcOffset, destBase, destOffset, bytes);
1018     }
1019 
1020     /**
1021      * Sets all bytes in a given block of memory to a copy of another
1022      * block.  This provides a <em>single-register</em> addressing mode,
1023      * as discussed in {@link #getInt(Object,long)}.
1024      *
1025      * Equivalent to {@code copyMemory(null, srcAddress, null, destAddress, bytes)}.
1026      */
1027     public void copyMemory(long srcAddress, long destAddress, long bytes) {
1028         copyMemory(null, srcAddress, null, destAddress, bytes);
1029     }
1030 
1031     /**
1032      * Validate the arguments to copyMemory
1033      *
1034      * @throws RuntimeException if any of the arguments is invalid
1035      *         (<em>Note:</em> after optimization, invalid inputs may
1036      *         go undetected, which will lead to unpredictable
1037      *         behavior)
1038      */
1039     private void copyMemoryChecks(Object srcBase, long srcOffset,
1040                                   Object destBase, long destOffset,
1041                                   long bytes) {
1042         checkSize(bytes);
1043         checkPrimitivePointer(srcBase, srcOffset);
1044         checkPrimitivePointer(destBase, destOffset);
1045     }
1046 
1047     /**
1048      * Copies all elements from one block of memory to another block,
1049      * *unconditionally* byte swapping the elements on the fly.
1050      *
1051      * <p>This method determines each block's base address by means of two parameters,
1052      * and so it provides (in effect) a <em>double-register</em> addressing mode,
1053      * as discussed in {@link #getInt(Object,long)}.  When the object reference is null,
1054      * the offset supplies an absolute base address.
1055      *
1056      * <em>Note:</em> It is the responsibility of the caller to make
1057      * sure arguments are checked before the methods are called. While
1058      * some rudimentary checks are performed on the input, the checks
1059      * are best effort and when performance is an overriding priority,
1060      * as when methods of this class are optimized by the runtime
1061      * compiler, some or all checks (if any) may be elided. Hence, the
1062      * caller must not rely on the checks and corresponding
1063      * exceptions!
1064      *
1065      * @throws RuntimeException if any of the arguments is invalid
1066      *
1067      * @since 9
1068      */
1069     public void copySwapMemory(Object srcBase, long srcOffset,
1070                                Object destBase, long destOffset,
1071                                long bytes, long elemSize) {
1072         copySwapMemoryChecks(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
1073 
1074         if (bytes == 0) {
1075             return;
1076         }
1077 
1078         copySwapMemory0(srcBase, srcOffset, destBase, destOffset, bytes, elemSize);
1079     }
1080 
1081     private void copySwapMemoryChecks(Object srcBase, long srcOffset,
1082                                       Object destBase, long destOffset,
1083                                       long bytes, long elemSize) {
1084         checkSize(bytes);
1085 
1086         if (elemSize != 2 && elemSize != 4 && elemSize != 8) {
1087             throw invalidInput();
1088         }
1089         if (bytes % elemSize != 0) {
1090             throw invalidInput();
1091         }
1092 
1093         checkPrimitivePointer(srcBase, srcOffset);
1094         checkPrimitivePointer(destBase, destOffset);
1095     }
1096 
1097     /**
1098      * Copies all elements from one block of memory to another block, byte swapping the
1099      * elements on the fly.
1100      *
1101      * This provides a <em>single-register</em> addressing mode, as
1102      * discussed in {@link #getInt(Object,long)}.
1103      *
1104      * Equivalent to {@code copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize)}.
1105      */
1106     public void copySwapMemory(long srcAddress, long destAddress, long bytes, long elemSize) {
1107         copySwapMemory(null, srcAddress, null, destAddress, bytes, elemSize);
1108     }
1109 
1110     /**
1111      * Disposes of a block of native memory, as obtained from {@link
1112      * #allocateMemory} or {@link #reallocateMemory}.  The address passed to
1113      * this method may be null, in which case no action is taken.
1114      *
1115      * <em>Note:</em> It is the responsibility of the caller to make
1116      * sure arguments are checked before the methods are called. While
1117      * some rudimentary checks are performed on the input, the checks
1118      * are best effort and when performance is an overriding priority,
1119      * as when methods of this class are optimized by the runtime
1120      * compiler, some or all checks (if any) may be elided. Hence, the
1121      * caller must not rely on the checks and corresponding
1122      * exceptions!
1123      *
1124      * @throws RuntimeException if any of the arguments is invalid
1125      *
1126      * @see #allocateMemory
1127      */
1128     public void freeMemory(long address) {
1129         freeMemoryChecks(address);
1130 
1131         if (address == 0) {
1132             return;
1133         }
1134 
1135         freeMemory0(address);
1136     }
1137 
1138     /**
1139      * Validate the arguments to freeMemory
1140      *
1141      * @throws RuntimeException if the arguments are invalid
1142      *         (<em>Note:</em> after optimization, invalid inputs may
1143      *         go undetected, which will lead to unpredictable
1144      *         behavior)
1145      */
1146     private void freeMemoryChecks(long address) {
1147         checkPointer(null, address);
1148     }
1149 
1150     /**
1151      * Ensure writeback of a specified virtual memory address range
1152      * from cache to physical memory. All bytes in the address range
1153      * are guaranteed to have been written back to physical memory on
1154      * return from this call i.e. subsequently executed store
1155      * instructions are guaranteed not to be visible before the
1156      * writeback is completed.
1157      *
1158      * @param address
1159      *        the lowest byte address that must be guaranteed written
1160      *        back to memory. bytes at lower addresses may also be
1161      *        written back.
1162      *
1163      * @param length
1164      *        the length in bytes of the region starting at address
1165      *        that must be guaranteed written back to memory.
1166      *
1167      * @throws RuntimeException if memory writeback is not supported
1168      *         on the current hardware of if the arguments are invalid.
1169      *         (<em>Note:</em> after optimization, invalid inputs may
1170      *         go undetected, which will lead to unpredictable
1171      *         behavior)
1172      *
1173      * @since 14
1174      */
1175 
1176     public void writebackMemory(long address, long length) {
1177         checkWritebackEnabled();
1178         checkWritebackMemory(address, length);
1179 
1180         // perform any required pre-writeback barrier
1181         writebackPreSync0();
1182 
1183         // write back one cache line at a time
1184         long line = dataCacheLineAlignDown(address);
1185         long end = address + length;
1186         while (line < end) {
1187             writeback0(line);
1188             line += dataCacheLineFlushSize();
1189         }
1190 
1191         // perform any required post-writeback barrier
1192         writebackPostSync0();
1193     }
1194 
1195     /**
1196      * Validate the arguments to writebackMemory
1197      *
1198      * @throws RuntimeException if the arguments are invalid
1199      *         (<em>Note:</em> after optimization, invalid inputs may
1200      *         go undetected, which will lead to unpredictable
1201      *         behavior)
1202      */
1203     private void checkWritebackMemory(long address, long length) {
1204         checkNativeAddress(address);
1205         checkSize(length);
1206     }
1207 
1208     /**
1209      * Validate that the current hardware supports memory writeback.
1210      * (<em>Note:</em> this is a belt and braces check.  Clients are
1211      * expected to test whether writeback is enabled by calling
1212      * ({@link isWritebackEnabled #isWritebackEnabled} and avoid
1213      * calling method {@link writeback #writeback} if it is disabled).
1214      *
1215      *
1216      * @throws RuntimeException if memory writeback is not supported
1217      */
1218     private void checkWritebackEnabled() {
1219         if (!isWritebackEnabled()) {
1220             throw new RuntimeException("writebackMemory not enabled!");
1221         }
1222     }
1223 
1224     /**
1225      * force writeback of an individual cache line.
1226      *
1227      * @param address
1228      *        the start address of the cache line to be written back
1229      */
1230     @IntrinsicCandidate
1231     private native void writeback0(long address);
1232 
1233      /**
1234       * Serialize writeback operations relative to preceding memory writes.
1235       */
1236     @IntrinsicCandidate
1237     private native void writebackPreSync0();
1238 
1239      /**
1240       * Serialize writeback operations relative to following memory writes.
1241       */
1242     @IntrinsicCandidate
1243     private native void writebackPostSync0();
1244 
1245     //--- random queries
1246 
1247     /**
1248      * This constant differs from all results that will ever be returned from
1249      * {@link #staticFieldOffset}, {@link #objectFieldOffset},
1250      * or {@link #arrayBaseOffset}.
1251      * <p>
1252      * The static type is @code long} to emphasize that long arithmetic should
1253      * always be used for offset calculations to avoid overflows.
1254      */
1255     public static final long INVALID_FIELD_OFFSET = -1;
1256 
1257     /**
1258      * Reports the location of a given field in the storage allocation of its
1259      * class.  Do not expect to perform any sort of arithmetic on this offset;
1260      * it is just a cookie which is passed to the unsafe heap memory accessors.
1261      *
1262      * <p>Any given field will always have the same offset and base, and no
1263      * two distinct fields of the same class will ever have the same offset
1264      * and base.
1265      *
1266      * <p>As of 1.4.1, offsets for fields are represented as long values,
1267      * although the Sun JVM does not use the most significant 32 bits.
1268      * However, JVM implementations which store static fields at absolute
1269      * addresses can use long offsets and null base pointers to express
1270      * the field locations in a form usable by {@link #getInt(Object,long)}.
1271      * Therefore, code which will be ported to such JVMs on 64-bit platforms
1272      * must preserve all bits of static field offsets.
1273      *
1274      * @throws NullPointerException if the field is {@code null}
1275      * @throws IllegalArgumentException if the field is static
1276      * @see #getInt(Object, long)
1277      */
1278     public long objectFieldOffset(Field f) {
1279         if (f == null) {
1280             throw new NullPointerException();
1281         }
1282 
1283         return objectFieldOffset0(f);
1284     }
1285 
1286     /**
1287      * (For compile-time known instance fields in JDK code only) Reports the
1288      * location of the field with a given name in the storage allocation of its
1289      * class.
1290      * <p>
1291      * This API is used to avoid creating reflective Objects in Java code at
1292      * startup.  This should not be used to find fields in non-trusted code.
1293      * Use the {@link #objectFieldOffset(Field) Field}-accepting version for
1294      * arbitrary fields instead.
1295      *
1296      * @throws NullPointerException if any parameter is {@code null}.
1297      * @throws InternalError if the presumably known field couldn't be found
1298      *
1299      * @see #objectFieldOffset(Field)
1300      */
1301     public long objectFieldOffset(Class<?> c, String name) {
1302         if (c == null || name == null) {
1303             throw new NullPointerException();
1304         }
1305 
1306         long result = knownObjectFieldOffset0(c, name);
1307         if (result < 0) {
1308             String type = switch ((int) result) {
1309                 case -2 -> "a static field";
1310                 case -1 -> "not found";
1311                 default -> "unknown";
1312             };
1313             throw new InternalError("Field %s.%s %s".formatted(c.getTypeName(), name, type));
1314         }
1315         return result;
1316     }
1317 
1318     /**
1319      * Reports the location of a given static field, in conjunction with {@link
1320      * #staticFieldBase}.
1321      * <p>Do not expect to perform any sort of arithmetic on this offset;
1322      * it is just a cookie which is passed to the unsafe heap memory accessors.
1323      *
1324      * <p>Any given field will always have the same offset, and no two distinct
1325      * fields of the same class will ever have the same offset.
1326      *
1327      * <p>As of 1.4.1, offsets for fields are represented as long values,
1328      * although the Sun JVM does not use the most significant 32 bits.
1329      * It is hard to imagine a JVM technology which needs more than
1330      * a few bits to encode an offset within a non-array object,
1331      * However, for consistency with other methods in this class,
1332      * this method reports its result as a long value.
1333      *
1334      * @throws NullPointerException if the field is {@code null}
1335      * @throws IllegalArgumentException if the field is not static
1336      * @see #getInt(Object, long)
1337      */
1338     public long staticFieldOffset(Field f) {
1339         if (f == null) {
1340             throw new NullPointerException();
1341         }
1342 
1343         return staticFieldOffset0(f);
1344     }
1345 
1346     /**
1347      * Reports the location of a given static field, in conjunction with {@link
1348      * #staticFieldOffset}.
1349      * <p>Fetch the base "Object", if any, with which static fields of the
1350      * given class can be accessed via methods like {@link #getInt(Object,
1351      * long)}.  This value may be null.  This value may refer to an object
1352      * which is a "cookie", not guaranteed to be a real Object, and it should
1353      * not be used in any way except as argument to the get and put routines in
1354      * this class.
1355      *
1356      * @throws NullPointerException if the field is {@code null}
1357      * @throws IllegalArgumentException if the field is not static
1358      */
1359     public Object staticFieldBase(Field f) {
1360         if (f == null) {
1361             throw new NullPointerException();
1362         }
1363 
1364         return staticFieldBase0(f);
1365     }
1366 
1367     /**
1368      * Detects if the given class may need to be initialized. This is often
1369      * needed in conjunction with obtaining the static field base of a
1370      * class.
1371      * @return false only if a call to {@code ensureClassInitialized} would have no effect
1372      */
1373     public boolean shouldBeInitialized(Class<?> c) {
1374         if (c == null) {
1375             throw new NullPointerException();
1376         }
1377 
1378         return shouldBeInitialized0(c);
1379     }
1380 
1381     /**
1382      * Ensures the given class has been initialized (see JVMS-5.5 for details).
1383      * This is often needed in conjunction with obtaining the static field base
1384      * of a class.
1385      *
1386      * The call returns when either class {@code c} is fully initialized or
1387      * class {@code c} is being initialized and the call is performed from
1388      * the initializing thread. In the latter case a subsequent call to
1389      * {@link #shouldBeInitialized} will return {@code true}.
1390      */
1391     public void ensureClassInitialized(Class<?> c) {
1392         if (c == null) {
1393             throw new NullPointerException();
1394         }
1395 
1396         ensureClassInitialized0(c);
1397     }
1398 
1399     /**
1400      * The reading or writing of strict static fields may require
1401      * special processing.  Notify the VM that such an event is about
1402      * to happen.  The VM may respond by throwing an exception, in the
1403      * case of a read of an uninitialized field.  If the VM allows the
1404      * method to return normally, no further calls are needed, with
1405      * the same arguments.
1406      */
1407     public void notifyStrictStaticAccess(Class<?> c, long staticFieldOffset, boolean writing) {
1408         if (c == null) {
1409             throw new NullPointerException();
1410         }
1411         notifyStrictStaticAccess0(c, staticFieldOffset, writing);
1412     }
1413 
1414     /**
1415      * Reports the offset of the first element in the storage allocation of a
1416      * given array class.  If {@link #arrayIndexScale} returns a non-zero value
1417      * for the same class, you may use that scale factor, together with this
1418      * base offset, to form new offsets to access elements of arrays of the
1419      * given class.
1420      * <p>
1421      * The return value is in the range of a {@code int}.  The return type is
1422      * {@code long} to emphasize that long arithmetic should always be used
1423      * for offset calculations to avoid overflows.
1424      * <p>
1425      * This method doesn't support arrays with an element type that is
1426      * a value class, because this type of array can have multiple layouts.
1427      * For these arrays, {@code arrayInstanceBaseOffset(Object[] array)}
1428      * must be used instead.
1429      *
1430      * @see #getInt(Object, long)
1431      * @see #putInt(Object, long, int)
1432      */
1433     public long arrayBaseOffset(Class<?> arrayClass) {
1434         if (arrayClass == null) {
1435             throw new NullPointerException();
1436         }
1437 
1438         return arrayBaseOffset0(arrayClass);
1439     }
1440 
1441     public long arrayInstanceBaseOffset(Object[] array) {
1442         if (array == null) {
1443             throw new NullPointerException();
1444         }
1445 
1446         return arrayInstanceBaseOffset0(array);
1447     }
1448 
1449     /** The value of {@code arrayBaseOffset(boolean[].class)} */
1450     public static final long ARRAY_BOOLEAN_BASE_OFFSET
1451             = theUnsafe.arrayBaseOffset(boolean[].class);
1452 
1453     /** The value of {@code arrayBaseOffset(byte[].class)} */
1454     public static final long ARRAY_BYTE_BASE_OFFSET
1455             = theUnsafe.arrayBaseOffset(byte[].class);
1456 
1457     /** The value of {@code arrayBaseOffset(short[].class)} */
1458     public static final long ARRAY_SHORT_BASE_OFFSET
1459             = theUnsafe.arrayBaseOffset(short[].class);
1460 
1461     /** The value of {@code arrayBaseOffset(char[].class)} */
1462     public static final long ARRAY_CHAR_BASE_OFFSET
1463             = theUnsafe.arrayBaseOffset(char[].class);
1464 
1465     /** The value of {@code arrayBaseOffset(int[].class)} */
1466     public static final long ARRAY_INT_BASE_OFFSET
1467             = theUnsafe.arrayBaseOffset(int[].class);
1468 
1469     /** The value of {@code arrayBaseOffset(long[].class)} */
1470     public static final long ARRAY_LONG_BASE_OFFSET
1471             = theUnsafe.arrayBaseOffset(long[].class);
1472 
1473     /** The value of {@code arrayBaseOffset(float[].class)} */
1474     public static final long ARRAY_FLOAT_BASE_OFFSET
1475             = theUnsafe.arrayBaseOffset(float[].class);
1476 
1477     /** The value of {@code arrayBaseOffset(double[].class)} */
1478     public static final long ARRAY_DOUBLE_BASE_OFFSET
1479             = theUnsafe.arrayBaseOffset(double[].class);
1480 
1481     /** The value of {@code arrayBaseOffset(Object[].class)} */
1482     public static final long ARRAY_OBJECT_BASE_OFFSET
1483             = theUnsafe.arrayBaseOffset(Object[].class);
1484 
1485     /**
1486      * Reports the scale factor for addressing elements in the storage
1487      * allocation of a given array class.  However, arrays of "narrow" types
1488      * will generally not work properly with accessors like {@link
1489      * #getByte(Object, long)}, so the scale factor for such classes is reported
1490      * as zero.
1491      * <p>
1492      * The computation of the actual memory offset should always use {@code
1493      * long} arithmetic to avoid overflows.
1494      * <p>
1495      * This method doesn't support arrays with an element type that is
1496      * a value class, because this type of array can have multiple layouts.
1497      * For these arrays, {@code arrayInstanceIndexScale(Object[] array)}
1498      * must be used instead.
1499      *
1500      * @see #arrayBaseOffset
1501      * @see #getInt(Object, long)
1502      * @see #putInt(Object, long, int)
1503      */
1504     public int arrayIndexScale(Class<?> arrayClass) {
1505         if (arrayClass == null) {
1506             throw new NullPointerException();
1507         }
1508 
1509         return arrayIndexScale0(arrayClass);
1510     }
1511 
1512     public int arrayInstanceIndexScale(Object[] array) {
1513         if (array == null) {
1514             throw new NullPointerException();
1515         }
1516 
1517         return arrayInstanceIndexScale0(array);
1518     }
1519 
1520     /**
1521      * Return the size of the object in the heap.
1522      * @param o an object
1523      * @return the objects's size
1524      * @since Valhalla
1525      */
1526     public long getObjectSize(Object o) {
1527         if (o == null)
1528             throw new NullPointerException();
1529         return getObjectSize0(o);
1530     }
1531 
1532     /** The value of {@code arrayIndexScale(boolean[].class)} */
1533     public static final int ARRAY_BOOLEAN_INDEX_SCALE
1534             = theUnsafe.arrayIndexScale(boolean[].class);
1535 
1536     /** The value of {@code arrayIndexScale(byte[].class)} */
1537     public static final int ARRAY_BYTE_INDEX_SCALE
1538             = theUnsafe.arrayIndexScale(byte[].class);
1539 
1540     /** The value of {@code arrayIndexScale(short[].class)} */
1541     public static final int ARRAY_SHORT_INDEX_SCALE
1542             = theUnsafe.arrayIndexScale(short[].class);
1543 
1544     /** The value of {@code arrayIndexScale(char[].class)} */
1545     public static final int ARRAY_CHAR_INDEX_SCALE
1546             = theUnsafe.arrayIndexScale(char[].class);
1547 
1548     /** The value of {@code arrayIndexScale(int[].class)} */
1549     public static final int ARRAY_INT_INDEX_SCALE
1550             = theUnsafe.arrayIndexScale(int[].class);
1551 
1552     /** The value of {@code arrayIndexScale(long[].class)} */
1553     public static final int ARRAY_LONG_INDEX_SCALE
1554             = theUnsafe.arrayIndexScale(long[].class);
1555 
1556     /** The value of {@code arrayIndexScale(float[].class)} */
1557     public static final int ARRAY_FLOAT_INDEX_SCALE
1558             = theUnsafe.arrayIndexScale(float[].class);
1559 
1560     /** The value of {@code arrayIndexScale(double[].class)} */
1561     public static final int ARRAY_DOUBLE_INDEX_SCALE
1562             = theUnsafe.arrayIndexScale(double[].class);
1563 
1564     /** The value of {@code arrayIndexScale(Object[].class)} */
1565     public static final int ARRAY_OBJECT_INDEX_SCALE
1566             = theUnsafe.arrayIndexScale(Object[].class);
1567 
1568     /**
1569      * Reports the size in bytes of a native pointer, as stored via {@link
1570      * #putAddress}.  This value will be either 4 or 8.  Note that the sizes of
1571      * other primitive types (as stored in native memory blocks) is determined
1572      * fully by their information content.
1573      */
1574     public int addressSize() {
1575         return ADDRESS_SIZE;
1576     }
1577 
1578     /** The value of {@code addressSize()} */
1579     public static final int ADDRESS_SIZE = ADDRESS_SIZE0;
1580 
1581     /**
1582      * Reports the size in bytes of a native memory page (whatever that is).
1583      * This value will always be a power of two.
1584      */
1585     public int pageSize() { return PAGE_SIZE; }
1586 
1587     /**
1588      * Reports the size in bytes of a data cache line written back by
1589      * the hardware cache line flush operation available to the JVM or
1590      * 0 if data cache line flushing is not enabled.
1591      */
1592     public int dataCacheLineFlushSize() { return DATA_CACHE_LINE_FLUSH_SIZE; }
1593 
1594     /**
1595      * Rounds down address to a data cache line boundary as
1596      * determined by {@link #dataCacheLineFlushSize}
1597      * @return the rounded down address
1598      */
1599     public long dataCacheLineAlignDown(long address) {
1600         return (address & ~(DATA_CACHE_LINE_FLUSH_SIZE - 1));
1601     }
1602 
1603     /**
1604      * Returns true if data cache line writeback
1605      */
1606     public static boolean isWritebackEnabled() { return DATA_CACHE_LINE_FLUSH_SIZE != 0; }
1607 
1608     //--- random trusted operations from JNI:
1609 
1610     /**
1611      * Tells the VM to define a class, without security checks.  By default, the
1612      * class loader and protection domain come from the caller's class.
1613      */
1614     public Class<?> defineClass(String name, byte[] b, int off, int len,
1615                                 ClassLoader loader,
1616                                 ProtectionDomain protectionDomain) {
1617         if (b == null) {
1618             throw new NullPointerException();
1619         }
1620         if (len < 0) {
1621             throw new ArrayIndexOutOfBoundsException();
1622         }
1623 
1624         return defineClass0(name, b, off, len, loader, protectionDomain);
1625     }
1626 
1627     public native Class<?> defineClass0(String name, byte[] b, int off, int len,
1628                                         ClassLoader loader,
1629                                         ProtectionDomain protectionDomain);
1630 
1631     /**
1632      * Allocates an instance but does not run any constructor.
1633      * Initializes the class if it has not yet been.
1634      */
1635     @IntrinsicCandidate
1636     public native Object allocateInstance(Class<?> cls)
1637         throws InstantiationException;
1638 
1639     /**
1640      * Allocates an array of a given type, but does not do zeroing.
1641      * <p>
1642      * This method should only be used in the very rare cases where a high-performance code
1643      * overwrites the destination array completely, and compilers cannot assist in zeroing elimination.
1644      * In an overwhelming majority of cases, a normal Java allocation should be used instead.
1645      * <p>
1646      * Users of this method are <b>required</b> to overwrite the initial (garbage) array contents
1647      * before allowing untrusted code, or code in other threads, to observe the reference
1648      * to the newly allocated array. In addition, the publication of the array reference must be
1649      * safe according to the Java Memory Model requirements.
1650      * <p>
1651      * The safest approach to deal with an uninitialized array is to keep the reference to it in local
1652      * variable at least until the initialization is complete, and then publish it <b>once</b>, either
1653      * by writing it to a <em>volatile</em> field, or storing it into a <em>final</em> field in constructor,
1654      * or issuing a {@link #storeFence} before publishing the reference.
1655      * <p>
1656      * @implnote This method can only allocate primitive arrays, to avoid garbage reference
1657      * elements that could break heap integrity.
1658      *
1659      * @param componentType array component type to allocate
1660      * @param length array size to allocate
1661      * @throws IllegalArgumentException if component type is null, or not a primitive class;
1662      *                                  or the length is negative
1663      */
1664     public Object allocateUninitializedArray(Class<?> componentType, int length) {
1665        if (componentType == null) {
1666            throw new IllegalArgumentException("Component type is null");
1667        }
1668        if (!componentType.isPrimitive()) {
1669            throw new IllegalArgumentException("Component type is not primitive");
1670        }
1671        if (length < 0) {
1672            throw new IllegalArgumentException("Negative length");
1673        }
1674        return allocateUninitializedArray0(componentType, length);
1675     }
1676 
1677     @IntrinsicCandidate
1678     private Object allocateUninitializedArray0(Class<?> componentType, int length) {
1679        // These fallbacks provide zeroed arrays, but intrinsic is not required to
1680        // return the zeroed arrays.
1681        if (componentType == byte.class)    return new byte[length];
1682        if (componentType == boolean.class) return new boolean[length];
1683        if (componentType == short.class)   return new short[length];
1684        if (componentType == char.class)    return new char[length];
1685        if (componentType == int.class)     return new int[length];
1686        if (componentType == float.class)   return new float[length];
1687        if (componentType == long.class)    return new long[length];
1688        if (componentType == double.class)  return new double[length];
1689        return null;
1690     }
1691 
1692     /** Throws the exception without telling the verifier. */
1693     public native void throwException(Throwable ee);
1694 
1695     /**
1696      * Atomically updates Java variable to {@code x} if it is currently
1697      * holding {@code expected}.
1698      *
1699      * <p>This operation has memory semantics of a {@code volatile} read
1700      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1701      *
1702      * @return {@code true} if successful
1703      */
1704     @IntrinsicCandidate
1705     public final native boolean compareAndSetReference(Object o, long offset,
1706                                                        Object expected,
1707                                                        Object x);
1708 
1709     private final boolean isValueObject(Object o) {
1710         return o != null && o.getClass().isValue();
1711     }
1712 
1713     /*
1714      * For value type, CAS should do substitutability test as opposed
1715      * to two pointers comparison.
1716      */
1717     @ForceInline
1718     public final <V> boolean compareAndSetReference(Object o, long offset,
1719                                                     Class<?> type,
1720                                                     V expected,
1721                                                     V x) {
1722         if (type.isValue() || isValueObject(expected)) {
1723             while (true) {
1724                 Object witness = getReferenceVolatile(o, offset);
1725                 if (witness != expected) {
1726                     return false;
1727                 }
1728                 if (compareAndSetReference(o, offset, witness, x)) {
1729                     return true;
1730                 }
1731             }
1732         } else {
1733             return compareAndSetReference(o, offset, expected, x);
1734         }
1735     }
1736 
1737     @ForceInline
1738     public final <V> boolean compareAndSetFlatValue(Object o, long offset,
1739                                                 int layout,
1740                                                 Class<?> valueType,
1741                                                 V expected,
1742                                                 V x) {
1743         while (true) {
1744             Object witness = getFlatValueVolatile(o, offset, layout, valueType);
1745             if (witness != expected) {
1746                 return false;
1747             }
1748             if (compareAndSetFlatValueAsBytes(o, offset, layout, valueType, witness, x)) {
1749                 return true;
1750             }
1751         }
1752     }
1753 
1754     @IntrinsicCandidate
1755     public final native Object compareAndExchangeReference(Object o, long offset,
1756                                                            Object expected,
1757                                                            Object x);
1758 
1759     @ForceInline
1760     public final <V> Object compareAndExchangeReference(Object o, long offset,
1761                                                         Class<?> valueType,
1762                                                         V expected,
1763                                                         V x) {
1764         if (valueType.isValue() || isValueObject(expected)) {
1765             while (true) {
1766                 Object witness = getReferenceVolatile(o, offset);
1767                 if (witness != expected) {
1768                     return witness;
1769                 }
1770                 if (compareAndSetReference(o, offset, witness, x)) {
1771                     return witness;
1772                 }
1773             }
1774         } else {
1775             return compareAndExchangeReference(o, offset, expected, x);
1776         }
1777     }
1778 
1779     @ForceInline
1780     public final <V> Object compareAndExchangeFlatValue(Object o, long offset,
1781                                                     int layout,
1782                                                     Class<?> valueType,
1783                                                     V expected,
1784                                                     V x) {
1785         while (true) {
1786             Object witness = getFlatValueVolatile(o, offset, layout, valueType);
1787             if (witness != expected) {
1788                 return witness;
1789             }
1790             if (compareAndSetFlatValueAsBytes(o, offset, layout, valueType, witness, x)) {
1791                 return witness;
1792             }
1793         }
1794     }
1795 
1796     @IntrinsicCandidate
1797     public final Object compareAndExchangeReferenceAcquire(Object o, long offset,
1798                                                            Object expected,
1799                                                            Object x) {
1800         return compareAndExchangeReference(o, offset, expected, x);
1801     }
1802 
1803     public final <V> Object compareAndExchangeReferenceAcquire(Object o, long offset,
1804                                                                Class<?> valueType,
1805                                                                V expected,
1806                                                                V x) {
1807         return compareAndExchangeReference(o, offset, valueType, expected, x);
1808     }
1809 
1810     @ForceInline
1811     public final <V> Object compareAndExchangeFlatValueAcquire(Object o, long offset,
1812                                                            int layout,
1813                                                            Class<?> valueType,
1814                                                            V expected,
1815                                                            V x) {
1816         return compareAndExchangeFlatValue(o, offset, layout, valueType, expected, x);
1817     }
1818 
1819     @IntrinsicCandidate
1820     public final Object compareAndExchangeReferenceRelease(Object o, long offset,
1821                                                            Object expected,
1822                                                            Object x) {
1823         return compareAndExchangeReference(o, offset, expected, x);
1824     }
1825 
1826     public final <V> Object compareAndExchangeReferenceRelease(Object o, long offset,
1827                                                                Class<?> valueType,
1828                                                                V expected,
1829                                                                V x) {
1830         return compareAndExchangeReference(o, offset, valueType, expected, x);
1831     }
1832 
1833     @ForceInline
1834     public final <V> Object compareAndExchangeFlatValueRelease(Object o, long offset,
1835                                                            int layout,
1836                                                            Class<?> valueType,
1837                                                            V expected,
1838                                                            V x) {
1839         return compareAndExchangeFlatValue(o, offset, layout, valueType, expected, x);
1840     }
1841 
1842     @IntrinsicCandidate
1843     public final boolean weakCompareAndSetReferencePlain(Object o, long offset,
1844                                                          Object expected,
1845                                                          Object x) {
1846         return compareAndSetReference(o, offset, expected, x);
1847     }
1848 
1849     public final <V> boolean weakCompareAndSetReferencePlain(Object o, long offset,
1850                                                              Class<?> valueType,
1851                                                              V expected,
1852                                                              V x) {
1853         if (valueType.isValue() || isValueObject(expected)) {
1854             return compareAndSetReference(o, offset, valueType, expected, x);
1855         } else {
1856             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1857         }
1858     }
1859 
1860     @ForceInline
1861     public final <V> boolean weakCompareAndSetFlatValuePlain(Object o, long offset,
1862                                                          int layout,
1863                                                          Class<?> valueType,
1864                                                          V expected,
1865                                                          V x) {
1866         return compareAndSetFlatValue(o, offset, layout, valueType, expected, x);
1867     }
1868 
1869     @IntrinsicCandidate
1870     public final boolean weakCompareAndSetReferenceAcquire(Object o, long offset,
1871                                                            Object expected,
1872                                                            Object x) {
1873         return compareAndSetReference(o, offset, expected, x);
1874     }
1875 
1876     public final <V> boolean weakCompareAndSetReferenceAcquire(Object o, long offset,
1877                                                                Class<?> valueType,
1878                                                                V expected,
1879                                                                V x) {
1880         if (valueType.isValue() || isValueObject(expected)) {
1881             return compareAndSetReference(o, offset, valueType, expected, x);
1882         } else {
1883             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1884         }
1885     }
1886 
1887     @ForceInline
1888     public final <V> boolean weakCompareAndSetFlatValueAcquire(Object o, long offset,
1889                                                            int layout,
1890                                                            Class<?> valueType,
1891                                                            V expected,
1892                                                            V x) {
1893         return compareAndSetFlatValue(o, offset, layout, valueType, expected, x);
1894     }
1895 
1896     @IntrinsicCandidate
1897     public final boolean weakCompareAndSetReferenceRelease(Object o, long offset,
1898                                                            Object expected,
1899                                                            Object x) {
1900         return compareAndSetReference(o, offset, expected, x);
1901     }
1902 
1903     public final <V> boolean weakCompareAndSetReferenceRelease(Object o, long offset,
1904                                                                Class<?> valueType,
1905                                                                V expected,
1906                                                                V x) {
1907         if (valueType.isValue() || isValueObject(expected)) {
1908             return compareAndSetReference(o, offset, valueType, expected, x);
1909         } else {
1910             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1911         }
1912     }
1913 
1914     @ForceInline
1915     public final <V> boolean weakCompareAndSetFlatValueRelease(Object o, long offset,
1916                                                            int layout,
1917                                                            Class<?> valueType,
1918                                                            V expected,
1919                                                            V x) {
1920         return compareAndSetFlatValue(o, offset, layout, valueType, expected, x);
1921     }
1922 
1923     @IntrinsicCandidate
1924     public final boolean weakCompareAndSetReference(Object o, long offset,
1925                                                     Object expected,
1926                                                     Object x) {
1927         return compareAndSetReference(o, offset, expected, x);
1928     }
1929 
1930     public final <V> boolean weakCompareAndSetReference(Object o, long offset,
1931                                                         Class<?> valueType,
1932                                                         V expected,
1933                                                         V x) {
1934         if (valueType.isValue() || isValueObject(expected)) {
1935             return compareAndSetReference(o, offset, valueType, expected, x);
1936         } else {
1937             return weakCompareAndSetReferencePlain(o, offset, expected, x);
1938         }
1939     }
1940 
1941     @ForceInline
1942     public final <V> boolean weakCompareAndSetFlatValue(Object o, long offset,
1943                                                     int layout,
1944                                                     Class<?> valueType,
1945                                                     V expected,
1946                                                     V x) {
1947         return compareAndSetFlatValue(o, offset, layout, valueType, expected, x);
1948     }
1949 
1950     /**
1951      * Atomically updates Java variable to {@code x} if it is currently
1952      * holding {@code expected}.
1953      *
1954      * <p>This operation has memory semantics of a {@code volatile} read
1955      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
1956      *
1957      * @return {@code true} if successful
1958      */
1959     @IntrinsicCandidate
1960     public final native boolean compareAndSetInt(Object o, long offset,
1961                                                  int expected,
1962                                                  int x);
1963 
1964     @IntrinsicCandidate
1965     public final native int compareAndExchangeInt(Object o, long offset,
1966                                                   int expected,
1967                                                   int x);
1968 
1969     @IntrinsicCandidate
1970     public final int compareAndExchangeIntAcquire(Object o, long offset,
1971                                                          int expected,
1972                                                          int x) {
1973         return compareAndExchangeInt(o, offset, expected, x);
1974     }
1975 
1976     @IntrinsicCandidate
1977     public final int compareAndExchangeIntRelease(Object o, long offset,
1978                                                          int expected,
1979                                                          int x) {
1980         return compareAndExchangeInt(o, offset, expected, x);
1981     }
1982 
1983     @IntrinsicCandidate
1984     public final boolean weakCompareAndSetIntPlain(Object o, long offset,
1985                                                    int expected,
1986                                                    int x) {
1987         return compareAndSetInt(o, offset, expected, x);
1988     }
1989 
1990     @IntrinsicCandidate
1991     public final boolean weakCompareAndSetIntAcquire(Object o, long offset,
1992                                                      int expected,
1993                                                      int x) {
1994         return compareAndSetInt(o, offset, expected, x);
1995     }
1996 
1997     @IntrinsicCandidate
1998     public final boolean weakCompareAndSetIntRelease(Object o, long offset,
1999                                                      int expected,
2000                                                      int x) {
2001         return compareAndSetInt(o, offset, expected, x);
2002     }
2003 
2004     @IntrinsicCandidate
2005     public final boolean weakCompareAndSetInt(Object o, long offset,
2006                                               int expected,
2007                                               int x) {
2008         return compareAndSetInt(o, offset, expected, x);
2009     }
2010 
2011     @IntrinsicCandidate
2012     public final byte compareAndExchangeByte(Object o, long offset,
2013                                              byte expected,
2014                                              byte x) {
2015         long wordOffset = offset & ~3;
2016         int shift = (int) (offset & 3) << 3;
2017         if (BIG_ENDIAN) {
2018             shift = 24 - shift;
2019         }
2020         int mask           = 0xFF << shift;
2021         int maskedExpected = (expected & 0xFF) << shift;
2022         int maskedX        = (x & 0xFF) << shift;
2023         int fullWord;
2024         do {
2025             fullWord = getIntVolatile(o, wordOffset);
2026             if ((fullWord & mask) != maskedExpected)
2027                 return (byte) ((fullWord & mask) >> shift);
2028         } while (!weakCompareAndSetInt(o, wordOffset,
2029                                                 fullWord, (fullWord & ~mask) | maskedX));
2030         return expected;
2031     }
2032 
2033     @IntrinsicCandidate
2034     public final boolean compareAndSetByte(Object o, long offset,
2035                                            byte expected,
2036                                            byte x) {
2037         return compareAndExchangeByte(o, offset, expected, x) == expected;
2038     }
2039 
2040     @IntrinsicCandidate
2041     public final boolean weakCompareAndSetByte(Object o, long offset,
2042                                                byte expected,
2043                                                byte x) {
2044         return compareAndSetByte(o, offset, expected, x);
2045     }
2046 
2047     @IntrinsicCandidate
2048     public final boolean weakCompareAndSetByteAcquire(Object o, long offset,
2049                                                       byte expected,
2050                                                       byte x) {
2051         return weakCompareAndSetByte(o, offset, expected, x);
2052     }
2053 
2054     @IntrinsicCandidate
2055     public final boolean weakCompareAndSetByteRelease(Object o, long offset,
2056                                                       byte expected,
2057                                                       byte x) {
2058         return weakCompareAndSetByte(o, offset, expected, x);
2059     }
2060 
2061     @IntrinsicCandidate
2062     public final boolean weakCompareAndSetBytePlain(Object o, long offset,
2063                                                     byte expected,
2064                                                     byte x) {
2065         return weakCompareAndSetByte(o, offset, expected, x);
2066     }
2067 
2068     @IntrinsicCandidate
2069     public final byte compareAndExchangeByteAcquire(Object o, long offset,
2070                                                     byte expected,
2071                                                     byte x) {
2072         return compareAndExchangeByte(o, offset, expected, x);
2073     }
2074 
2075     @IntrinsicCandidate
2076     public final byte compareAndExchangeByteRelease(Object o, long offset,
2077                                                     byte expected,
2078                                                     byte x) {
2079         return compareAndExchangeByte(o, offset, expected, x);
2080     }
2081 
2082     @IntrinsicCandidate
2083     public final short compareAndExchangeShort(Object o, long offset,
2084                                                short expected,
2085                                                short x) {
2086         if ((offset & 3) == 3) {
2087             throw new IllegalArgumentException("Update spans the word, not supported");
2088         }
2089         long wordOffset = offset & ~3;
2090         int shift = (int) (offset & 3) << 3;
2091         if (BIG_ENDIAN) {
2092             shift = 16 - shift;
2093         }
2094         int mask           = 0xFFFF << shift;
2095         int maskedExpected = (expected & 0xFFFF) << shift;
2096         int maskedX        = (x & 0xFFFF) << shift;
2097         int fullWord;
2098         do {
2099             fullWord = getIntVolatile(o, wordOffset);
2100             if ((fullWord & mask) != maskedExpected) {
2101                 return (short) ((fullWord & mask) >> shift);
2102             }
2103         } while (!weakCompareAndSetInt(o, wordOffset,
2104                                                 fullWord, (fullWord & ~mask) | maskedX));
2105         return expected;
2106     }
2107 
2108     @IntrinsicCandidate
2109     public final boolean compareAndSetShort(Object o, long offset,
2110                                             short expected,
2111                                             short x) {
2112         return compareAndExchangeShort(o, offset, expected, x) == expected;
2113     }
2114 
2115     @IntrinsicCandidate
2116     public final boolean weakCompareAndSetShort(Object o, long offset,
2117                                                 short expected,
2118                                                 short x) {
2119         return compareAndSetShort(o, offset, expected, x);
2120     }
2121 
2122     @IntrinsicCandidate
2123     public final boolean weakCompareAndSetShortAcquire(Object o, long offset,
2124                                                        short expected,
2125                                                        short x) {
2126         return weakCompareAndSetShort(o, offset, expected, x);
2127     }
2128 
2129     @IntrinsicCandidate
2130     public final boolean weakCompareAndSetShortRelease(Object o, long offset,
2131                                                        short expected,
2132                                                        short x) {
2133         return weakCompareAndSetShort(o, offset, expected, x);
2134     }
2135 
2136     @IntrinsicCandidate
2137     public final boolean weakCompareAndSetShortPlain(Object o, long offset,
2138                                                      short expected,
2139                                                      short x) {
2140         return weakCompareAndSetShort(o, offset, expected, x);
2141     }
2142 
2143 
2144     @IntrinsicCandidate
2145     public final short compareAndExchangeShortAcquire(Object o, long offset,
2146                                                      short expected,
2147                                                      short x) {
2148         return compareAndExchangeShort(o, offset, expected, x);
2149     }
2150 
2151     @IntrinsicCandidate
2152     public final short compareAndExchangeShortRelease(Object o, long offset,
2153                                                     short expected,
2154                                                     short x) {
2155         return compareAndExchangeShort(o, offset, expected, x);
2156     }
2157 
2158     @ForceInline
2159     private char s2c(short s) {
2160         return (char) s;
2161     }
2162 
2163     @ForceInline
2164     private short c2s(char s) {
2165         return (short) s;
2166     }
2167 
2168     @ForceInline
2169     public final boolean compareAndSetChar(Object o, long offset,
2170                                            char expected,
2171                                            char x) {
2172         return compareAndSetShort(o, offset, c2s(expected), c2s(x));
2173     }
2174 
2175     @ForceInline
2176     public final char compareAndExchangeChar(Object o, long offset,
2177                                              char expected,
2178                                              char x) {
2179         return s2c(compareAndExchangeShort(o, offset, c2s(expected), c2s(x)));
2180     }
2181 
2182     @ForceInline
2183     public final char compareAndExchangeCharAcquire(Object o, long offset,
2184                                             char expected,
2185                                             char x) {
2186         return s2c(compareAndExchangeShortAcquire(o, offset, c2s(expected), c2s(x)));
2187     }
2188 
2189     @ForceInline
2190     public final char compareAndExchangeCharRelease(Object o, long offset,
2191                                             char expected,
2192                                             char x) {
2193         return s2c(compareAndExchangeShortRelease(o, offset, c2s(expected), c2s(x)));
2194     }
2195 
2196     @ForceInline
2197     public final boolean weakCompareAndSetChar(Object o, long offset,
2198                                                char expected,
2199                                                char x) {
2200         return weakCompareAndSetShort(o, offset, c2s(expected), c2s(x));
2201     }
2202 
2203     @ForceInline
2204     public final boolean weakCompareAndSetCharAcquire(Object o, long offset,
2205                                                       char expected,
2206                                                       char x) {
2207         return weakCompareAndSetShortAcquire(o, offset, c2s(expected), c2s(x));
2208     }
2209 
2210     @ForceInline
2211     public final boolean weakCompareAndSetCharRelease(Object o, long offset,
2212                                                       char expected,
2213                                                       char x) {
2214         return weakCompareAndSetShortRelease(o, offset, c2s(expected), c2s(x));
2215     }
2216 
2217     @ForceInline
2218     public final boolean weakCompareAndSetCharPlain(Object o, long offset,
2219                                                     char expected,
2220                                                     char x) {
2221         return weakCompareAndSetShortPlain(o, offset, c2s(expected), c2s(x));
2222     }
2223 
2224     /**
2225      * The JVM converts integral values to boolean values using two
2226      * different conventions, byte testing against zero and truncation
2227      * to least-significant bit.
2228      *
2229      * <p>The JNI documents specify that, at least for returning
2230      * values from native methods, a Java boolean value is converted
2231      * to the value-set 0..1 by first truncating to a byte (0..255 or
2232      * maybe -128..127) and then testing against zero. Thus, Java
2233      * booleans in non-Java data structures are by convention
2234      * represented as 8-bit containers containing either zero (for
2235      * false) or any non-zero value (for true).
2236      *
2237      * <p>Java booleans in the heap are also stored in bytes, but are
2238      * strongly normalized to the value-set 0..1 (i.e., they are
2239      * truncated to the least-significant bit).
2240      *
2241      * <p>The main reason for having different conventions for
2242      * conversion is performance: Truncation to the least-significant
2243      * bit can be usually implemented with fewer (machine)
2244      * instructions than byte testing against zero.
2245      *
2246      * <p>A number of Unsafe methods load boolean values from the heap
2247      * as bytes. Unsafe converts those values according to the JNI
2248      * rules (i.e, using the "testing against zero" convention). The
2249      * method {@code byte2bool} implements that conversion.
2250      *
2251      * @param b the byte to be converted to boolean
2252      * @return the result of the conversion
2253      */
2254     @ForceInline
2255     private boolean byte2bool(byte b) {
2256         return b != 0;
2257     }
2258 
2259     /**
2260      * Convert a boolean value to a byte. The return value is strongly
2261      * normalized to the value-set 0..1 (i.e., the value is truncated
2262      * to the least-significant bit). See {@link #byte2bool(byte)} for
2263      * more details on conversion conventions.
2264      *
2265      * @param b the boolean to be converted to byte (and then normalized)
2266      * @return the result of the conversion
2267      */
2268     @ForceInline
2269     private byte bool2byte(boolean b) {
2270         return b ? (byte)1 : (byte)0;
2271     }
2272 
2273     @ForceInline
2274     public final boolean compareAndSetBoolean(Object o, long offset,
2275                                               boolean expected,
2276                                               boolean x) {
2277         return compareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
2278     }
2279 
2280     @ForceInline
2281     public final boolean compareAndExchangeBoolean(Object o, long offset,
2282                                                    boolean expected,
2283                                                    boolean x) {
2284         return byte2bool(compareAndExchangeByte(o, offset, bool2byte(expected), bool2byte(x)));
2285     }
2286 
2287     @ForceInline
2288     public final boolean compareAndExchangeBooleanAcquire(Object o, long offset,
2289                                                     boolean expected,
2290                                                     boolean x) {
2291         return byte2bool(compareAndExchangeByteAcquire(o, offset, bool2byte(expected), bool2byte(x)));
2292     }
2293 
2294     @ForceInline
2295     public final boolean compareAndExchangeBooleanRelease(Object o, long offset,
2296                                                        boolean expected,
2297                                                        boolean x) {
2298         return byte2bool(compareAndExchangeByteRelease(o, offset, bool2byte(expected), bool2byte(x)));
2299     }
2300 
2301     @ForceInline
2302     public final boolean weakCompareAndSetBoolean(Object o, long offset,
2303                                                   boolean expected,
2304                                                   boolean x) {
2305         return weakCompareAndSetByte(o, offset, bool2byte(expected), bool2byte(x));
2306     }
2307 
2308     @ForceInline
2309     public final boolean weakCompareAndSetBooleanAcquire(Object o, long offset,
2310                                                          boolean expected,
2311                                                          boolean x) {
2312         return weakCompareAndSetByteAcquire(o, offset, bool2byte(expected), bool2byte(x));
2313     }
2314 
2315     @ForceInline
2316     public final boolean weakCompareAndSetBooleanRelease(Object o, long offset,
2317                                                          boolean expected,
2318                                                          boolean x) {
2319         return weakCompareAndSetByteRelease(o, offset, bool2byte(expected), bool2byte(x));
2320     }
2321 
2322     @ForceInline
2323     public final boolean weakCompareAndSetBooleanPlain(Object o, long offset,
2324                                                        boolean expected,
2325                                                        boolean x) {
2326         return weakCompareAndSetBytePlain(o, offset, bool2byte(expected), bool2byte(x));
2327     }
2328 
2329     /**
2330      * Atomically updates Java variable to {@code x} if it is currently
2331      * holding {@code expected}.
2332      *
2333      * <p>This operation has memory semantics of a {@code volatile} read
2334      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2335      *
2336      * @return {@code true} if successful
2337      */
2338     @ForceInline
2339     public final boolean compareAndSetFloat(Object o, long offset,
2340                                             float expected,
2341                                             float x) {
2342         return compareAndSetInt(o, offset,
2343                                  Float.floatToRawIntBits(expected),
2344                                  Float.floatToRawIntBits(x));
2345     }
2346 
2347     @ForceInline
2348     public final float compareAndExchangeFloat(Object o, long offset,
2349                                                float expected,
2350                                                float x) {
2351         int w = compareAndExchangeInt(o, offset,
2352                                       Float.floatToRawIntBits(expected),
2353                                       Float.floatToRawIntBits(x));
2354         return Float.intBitsToFloat(w);
2355     }
2356 
2357     @ForceInline
2358     public final float compareAndExchangeFloatAcquire(Object o, long offset,
2359                                                   float expected,
2360                                                   float x) {
2361         int w = compareAndExchangeIntAcquire(o, offset,
2362                                              Float.floatToRawIntBits(expected),
2363                                              Float.floatToRawIntBits(x));
2364         return Float.intBitsToFloat(w);
2365     }
2366 
2367     @ForceInline
2368     public final float compareAndExchangeFloatRelease(Object o, long offset,
2369                                                   float expected,
2370                                                   float x) {
2371         int w = compareAndExchangeIntRelease(o, offset,
2372                                              Float.floatToRawIntBits(expected),
2373                                              Float.floatToRawIntBits(x));
2374         return Float.intBitsToFloat(w);
2375     }
2376 
2377     @ForceInline
2378     public final boolean weakCompareAndSetFloatPlain(Object o, long offset,
2379                                                      float expected,
2380                                                      float x) {
2381         return weakCompareAndSetIntPlain(o, offset,
2382                                      Float.floatToRawIntBits(expected),
2383                                      Float.floatToRawIntBits(x));
2384     }
2385 
2386     @ForceInline
2387     public final boolean weakCompareAndSetFloatAcquire(Object o, long offset,
2388                                                        float expected,
2389                                                        float x) {
2390         return weakCompareAndSetIntAcquire(o, offset,
2391                                             Float.floatToRawIntBits(expected),
2392                                             Float.floatToRawIntBits(x));
2393     }
2394 
2395     @ForceInline
2396     public final boolean weakCompareAndSetFloatRelease(Object o, long offset,
2397                                                        float expected,
2398                                                        float x) {
2399         return weakCompareAndSetIntRelease(o, offset,
2400                                             Float.floatToRawIntBits(expected),
2401                                             Float.floatToRawIntBits(x));
2402     }
2403 
2404     @ForceInline
2405     public final boolean weakCompareAndSetFloat(Object o, long offset,
2406                                                 float expected,
2407                                                 float x) {
2408         return weakCompareAndSetInt(o, offset,
2409                                              Float.floatToRawIntBits(expected),
2410                                              Float.floatToRawIntBits(x));
2411     }
2412 
2413     /**
2414      * Atomically updates Java variable to {@code x} if it is currently
2415      * holding {@code expected}.
2416      *
2417      * <p>This operation has memory semantics of a {@code volatile} read
2418      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2419      *
2420      * @return {@code true} if successful
2421      */
2422     @ForceInline
2423     public final boolean compareAndSetDouble(Object o, long offset,
2424                                              double expected,
2425                                              double x) {
2426         return compareAndSetLong(o, offset,
2427                                  Double.doubleToRawLongBits(expected),
2428                                  Double.doubleToRawLongBits(x));
2429     }
2430 
2431     @ForceInline
2432     public final double compareAndExchangeDouble(Object o, long offset,
2433                                                  double expected,
2434                                                  double x) {
2435         long w = compareAndExchangeLong(o, offset,
2436                                         Double.doubleToRawLongBits(expected),
2437                                         Double.doubleToRawLongBits(x));
2438         return Double.longBitsToDouble(w);
2439     }
2440 
2441     @ForceInline
2442     public final double compareAndExchangeDoubleAcquire(Object o, long offset,
2443                                                         double expected,
2444                                                         double x) {
2445         long w = compareAndExchangeLongAcquire(o, offset,
2446                                                Double.doubleToRawLongBits(expected),
2447                                                Double.doubleToRawLongBits(x));
2448         return Double.longBitsToDouble(w);
2449     }
2450 
2451     @ForceInline
2452     public final double compareAndExchangeDoubleRelease(Object o, long offset,
2453                                                         double expected,
2454                                                         double x) {
2455         long w = compareAndExchangeLongRelease(o, offset,
2456                                                Double.doubleToRawLongBits(expected),
2457                                                Double.doubleToRawLongBits(x));
2458         return Double.longBitsToDouble(w);
2459     }
2460 
2461     @ForceInline
2462     public final boolean weakCompareAndSetDoublePlain(Object o, long offset,
2463                                                       double expected,
2464                                                       double x) {
2465         return weakCompareAndSetLongPlain(o, offset,
2466                                      Double.doubleToRawLongBits(expected),
2467                                      Double.doubleToRawLongBits(x));
2468     }
2469 
2470     @ForceInline
2471     public final boolean weakCompareAndSetDoubleAcquire(Object o, long offset,
2472                                                         double expected,
2473                                                         double x) {
2474         return weakCompareAndSetLongAcquire(o, offset,
2475                                              Double.doubleToRawLongBits(expected),
2476                                              Double.doubleToRawLongBits(x));
2477     }
2478 
2479     @ForceInline
2480     public final boolean weakCompareAndSetDoubleRelease(Object o, long offset,
2481                                                         double expected,
2482                                                         double x) {
2483         return weakCompareAndSetLongRelease(o, offset,
2484                                              Double.doubleToRawLongBits(expected),
2485                                              Double.doubleToRawLongBits(x));
2486     }
2487 
2488     @ForceInline
2489     public final boolean weakCompareAndSetDouble(Object o, long offset,
2490                                                  double expected,
2491                                                  double x) {
2492         return weakCompareAndSetLong(o, offset,
2493                                               Double.doubleToRawLongBits(expected),
2494                                               Double.doubleToRawLongBits(x));
2495     }
2496 
2497     /**
2498      * Atomically updates Java variable to {@code x} if it is currently
2499      * holding {@code expected}.
2500      *
2501      * <p>This operation has memory semantics of a {@code volatile} read
2502      * and write.  Corresponds to C11 atomic_compare_exchange_strong.
2503      *
2504      * @return {@code true} if successful
2505      */
2506     @IntrinsicCandidate
2507     public final native boolean compareAndSetLong(Object o, long offset,
2508                                                   long expected,
2509                                                   long x);
2510 
2511     @IntrinsicCandidate
2512     public final native long compareAndExchangeLong(Object o, long offset,
2513                                                     long expected,
2514                                                     long x);
2515 
2516     @IntrinsicCandidate
2517     public final long compareAndExchangeLongAcquire(Object o, long offset,
2518                                                            long expected,
2519                                                            long x) {
2520         return compareAndExchangeLong(o, offset, expected, x);
2521     }
2522 
2523     @IntrinsicCandidate
2524     public final long compareAndExchangeLongRelease(Object o, long offset,
2525                                                            long expected,
2526                                                            long x) {
2527         return compareAndExchangeLong(o, offset, expected, x);
2528     }
2529 
2530     @IntrinsicCandidate
2531     public final boolean weakCompareAndSetLongPlain(Object o, long offset,
2532                                                     long expected,
2533                                                     long x) {
2534         return compareAndSetLong(o, offset, expected, x);
2535     }
2536 
2537     @IntrinsicCandidate
2538     public final boolean weakCompareAndSetLongAcquire(Object o, long offset,
2539                                                       long expected,
2540                                                       long x) {
2541         return compareAndSetLong(o, offset, expected, x);
2542     }
2543 
2544     @IntrinsicCandidate
2545     public final boolean weakCompareAndSetLongRelease(Object o, long offset,
2546                                                       long expected,
2547                                                       long x) {
2548         return compareAndSetLong(o, offset, expected, x);
2549     }
2550 
2551     @IntrinsicCandidate
2552     public final boolean weakCompareAndSetLong(Object o, long offset,
2553                                                long expected,
2554                                                long x) {
2555         return compareAndSetLong(o, offset, expected, x);
2556     }
2557 
2558     /**
2559      * Fetches a reference value from a given Java variable, with volatile
2560      * load semantics. Otherwise identical to {@link #getReference(Object, long)}
2561      */
2562     @IntrinsicCandidate
2563     public native Object getReferenceVolatile(Object o, long offset);
2564 
2565     @ForceInline
2566     public final <V> Object getFlatValueVolatile(Object o, long offset, int layout, Class<?> valueType) {
2567         // we translate using fences (see: https://gee.cs.oswego.edu/dl/html/j9mm.html)
2568         Object res = getFlatValue(o, offset, layout, valueType);
2569         fullFence();
2570         return res;
2571     }
2572 
2573     /**
2574      * Stores a reference value into a given Java variable, with
2575      * volatile store semantics. Otherwise identical to {@link #putReference(Object, long, Object)}
2576      */
2577     @IntrinsicCandidate
2578     public native void putReferenceVolatile(Object o, long offset, Object x);
2579 
2580     @ForceInline
2581     public final <V> void putFlatValueVolatile(Object o, long offset, int layout, Class<?> valueType, V x) {
2582         // we translate using fences (see: https://gee.cs.oswego.edu/dl/html/j9mm.html)
2583         putFlatValueRelease(o, offset, layout, valueType, x);
2584         fullFence();
2585     }
2586 
2587     /** Volatile version of {@link #getInt(Object, long)}  */
2588     @IntrinsicCandidate
2589     public native int     getIntVolatile(Object o, long offset);
2590 
2591     /** Volatile version of {@link #putInt(Object, long, int)}  */
2592     @IntrinsicCandidate
2593     public native void    putIntVolatile(Object o, long offset, int x);
2594 
2595     /** Volatile version of {@link #getBoolean(Object, long)}  */
2596     @IntrinsicCandidate
2597     public native boolean getBooleanVolatile(Object o, long offset);
2598 
2599     /** Volatile version of {@link #putBoolean(Object, long, boolean)}  */
2600     @IntrinsicCandidate
2601     public native void    putBooleanVolatile(Object o, long offset, boolean x);
2602 
2603     /** Volatile version of {@link #getByte(Object, long)}  */
2604     @IntrinsicCandidate
2605     public native byte    getByteVolatile(Object o, long offset);
2606 
2607     /** Volatile version of {@link #putByte(Object, long, byte)}  */
2608     @IntrinsicCandidate
2609     public native void    putByteVolatile(Object o, long offset, byte x);
2610 
2611     /** Volatile version of {@link #getShort(Object, long)}  */
2612     @IntrinsicCandidate
2613     public native short   getShortVolatile(Object o, long offset);
2614 
2615     /** Volatile version of {@link #putShort(Object, long, short)}  */
2616     @IntrinsicCandidate
2617     public native void    putShortVolatile(Object o, long offset, short x);
2618 
2619     /** Volatile version of {@link #getChar(Object, long)}  */
2620     @IntrinsicCandidate
2621     public native char    getCharVolatile(Object o, long offset);
2622 
2623     /** Volatile version of {@link #putChar(Object, long, char)}  */
2624     @IntrinsicCandidate
2625     public native void    putCharVolatile(Object o, long offset, char x);
2626 
2627     /** Volatile version of {@link #getLong(Object, long)}  */
2628     @IntrinsicCandidate
2629     public native long    getLongVolatile(Object o, long offset);
2630 
2631     /** Volatile version of {@link #putLong(Object, long, long)}  */
2632     @IntrinsicCandidate
2633     public native void    putLongVolatile(Object o, long offset, long x);
2634 
2635     /** Volatile version of {@link #getFloat(Object, long)}  */
2636     @IntrinsicCandidate
2637     public native float   getFloatVolatile(Object o, long offset);
2638 
2639     /** Volatile version of {@link #putFloat(Object, long, float)}  */
2640     @IntrinsicCandidate
2641     public native void    putFloatVolatile(Object o, long offset, float x);
2642 
2643     /** Volatile version of {@link #getDouble(Object, long)}  */
2644     @IntrinsicCandidate
2645     public native double  getDoubleVolatile(Object o, long offset);
2646 
2647     /** Volatile version of {@link #putDouble(Object, long, double)}  */
2648     @IntrinsicCandidate
2649     public native void    putDoubleVolatile(Object o, long offset, double x);
2650 
2651 
2652 
2653     /** Acquire version of {@link #getReferenceVolatile(Object, long)} */
2654     @IntrinsicCandidate
2655     public final Object getReferenceAcquire(Object o, long offset) {
2656         return getReferenceVolatile(o, offset);
2657     }
2658 
2659     @ForceInline
2660     public final <V> Object getFlatValueAcquire(Object o, long offset, int layout, Class<?> valueType) {
2661         // we translate using fences (see: https://gee.cs.oswego.edu/dl/html/j9mm.html)
2662         Object res = getFlatValue(o, offset, layout, valueType);
2663         loadFence();
2664         return res;
2665     }
2666 
2667     /** Acquire version of {@link #getBooleanVolatile(Object, long)} */
2668     @IntrinsicCandidate
2669     public final boolean getBooleanAcquire(Object o, long offset) {
2670         return getBooleanVolatile(o, offset);
2671     }
2672 
2673     /** Acquire version of {@link #getByteVolatile(Object, long)} */
2674     @IntrinsicCandidate
2675     public final byte getByteAcquire(Object o, long offset) {
2676         return getByteVolatile(o, offset);
2677     }
2678 
2679     /** Acquire version of {@link #getShortVolatile(Object, long)} */
2680     @IntrinsicCandidate
2681     public final short getShortAcquire(Object o, long offset) {
2682         return getShortVolatile(o, offset);
2683     }
2684 
2685     /** Acquire version of {@link #getCharVolatile(Object, long)} */
2686     @IntrinsicCandidate
2687     public final char getCharAcquire(Object o, long offset) {
2688         return getCharVolatile(o, offset);
2689     }
2690 
2691     /** Acquire version of {@link #getIntVolatile(Object, long)} */
2692     @IntrinsicCandidate
2693     public final int getIntAcquire(Object o, long offset) {
2694         return getIntVolatile(o, offset);
2695     }
2696 
2697     /** Acquire version of {@link #getFloatVolatile(Object, long)} */
2698     @IntrinsicCandidate
2699     public final float getFloatAcquire(Object o, long offset) {
2700         return getFloatVolatile(o, offset);
2701     }
2702 
2703     /** Acquire version of {@link #getLongVolatile(Object, long)} */
2704     @IntrinsicCandidate
2705     public final long getLongAcquire(Object o, long offset) {
2706         return getLongVolatile(o, offset);
2707     }
2708 
2709     /** Acquire version of {@link #getDoubleVolatile(Object, long)} */
2710     @IntrinsicCandidate
2711     public final double getDoubleAcquire(Object o, long offset) {
2712         return getDoubleVolatile(o, offset);
2713     }
2714 
2715     /*
2716      * Versions of {@link #putReferenceVolatile(Object, long, Object)}
2717      * that do not guarantee immediate visibility of the store to
2718      * other threads. This method is generally only useful if the
2719      * underlying field is a Java volatile (or if an array cell, one
2720      * that is otherwise only accessed using volatile accesses).
2721      *
2722      * Corresponds to C11 atomic_store_explicit(..., memory_order_release).
2723      */
2724 
2725     /** Release version of {@link #putReferenceVolatile(Object, long, Object)} */
2726     @IntrinsicCandidate
2727     public final void putReferenceRelease(Object o, long offset, Object x) {
2728         putReferenceVolatile(o, offset, x);
2729     }
2730 
2731     @ForceInline
2732     public final <V> void putFlatValueRelease(Object o, long offset, int layout, Class<?> valueType, V x) {
2733         // we translate using fences (see: https://gee.cs.oswego.edu/dl/html/j9mm.html)
2734         storeFence();
2735         putFlatValue(o, offset, layout, valueType, x);
2736     }
2737 
2738     /** Release version of {@link #putBooleanVolatile(Object, long, boolean)} */
2739     @IntrinsicCandidate
2740     public final void putBooleanRelease(Object o, long offset, boolean x) {
2741         putBooleanVolatile(o, offset, x);
2742     }
2743 
2744     /** Release version of {@link #putByteVolatile(Object, long, byte)} */
2745     @IntrinsicCandidate
2746     public final void putByteRelease(Object o, long offset, byte x) {
2747         putByteVolatile(o, offset, x);
2748     }
2749 
2750     /** Release version of {@link #putShortVolatile(Object, long, short)} */
2751     @IntrinsicCandidate
2752     public final void putShortRelease(Object o, long offset, short x) {
2753         putShortVolatile(o, offset, x);
2754     }
2755 
2756     /** Release version of {@link #putCharVolatile(Object, long, char)} */
2757     @IntrinsicCandidate
2758     public final void putCharRelease(Object o, long offset, char x) {
2759         putCharVolatile(o, offset, x);
2760     }
2761 
2762     /** Release version of {@link #putIntVolatile(Object, long, int)} */
2763     @IntrinsicCandidate
2764     public final void putIntRelease(Object o, long offset, int x) {
2765         putIntVolatile(o, offset, x);
2766     }
2767 
2768     /** Release version of {@link #putFloatVolatile(Object, long, float)} */
2769     @IntrinsicCandidate
2770     public final void putFloatRelease(Object o, long offset, float x) {
2771         putFloatVolatile(o, offset, x);
2772     }
2773 
2774     /** Release version of {@link #putLongVolatile(Object, long, long)} */
2775     @IntrinsicCandidate
2776     public final void putLongRelease(Object o, long offset, long x) {
2777         putLongVolatile(o, offset, x);
2778     }
2779 
2780     /** Release version of {@link #putDoubleVolatile(Object, long, double)} */
2781     @IntrinsicCandidate
2782     public final void putDoubleRelease(Object o, long offset, double x) {
2783         putDoubleVolatile(o, offset, x);
2784     }
2785 
2786     // ------------------------------ Opaque --------------------------------------
2787 
2788     /** Opaque version of {@link #getReferenceVolatile(Object, long)} */
2789     @IntrinsicCandidate
2790     public final Object getReferenceOpaque(Object o, long offset) {
2791         return getReferenceVolatile(o, offset);
2792     }
2793 
2794     @ForceInline
2795     public final <V> Object getFlatValueOpaque(Object o, long offset, int layout, Class<?> valueType) {
2796         // this is stronger than opaque semantics
2797         return getFlatValueAcquire(o, offset, layout, valueType);
2798     }
2799 
2800     /** Opaque version of {@link #getBooleanVolatile(Object, long)} */
2801     @IntrinsicCandidate
2802     public final boolean getBooleanOpaque(Object o, long offset) {
2803         return getBooleanVolatile(o, offset);
2804     }
2805 
2806     /** Opaque version of {@link #getByteVolatile(Object, long)} */
2807     @IntrinsicCandidate
2808     public final byte getByteOpaque(Object o, long offset) {
2809         return getByteVolatile(o, offset);
2810     }
2811 
2812     /** Opaque version of {@link #getShortVolatile(Object, long)} */
2813     @IntrinsicCandidate
2814     public final short getShortOpaque(Object o, long offset) {
2815         return getShortVolatile(o, offset);
2816     }
2817 
2818     /** Opaque version of {@link #getCharVolatile(Object, long)} */
2819     @IntrinsicCandidate
2820     public final char getCharOpaque(Object o, long offset) {
2821         return getCharVolatile(o, offset);
2822     }
2823 
2824     /** Opaque version of {@link #getIntVolatile(Object, long)} */
2825     @IntrinsicCandidate
2826     public final int getIntOpaque(Object o, long offset) {
2827         return getIntVolatile(o, offset);
2828     }
2829 
2830     /** Opaque version of {@link #getFloatVolatile(Object, long)} */
2831     @IntrinsicCandidate
2832     public final float getFloatOpaque(Object o, long offset) {
2833         return getFloatVolatile(o, offset);
2834     }
2835 
2836     /** Opaque version of {@link #getLongVolatile(Object, long)} */
2837     @IntrinsicCandidate
2838     public final long getLongOpaque(Object o, long offset) {
2839         return getLongVolatile(o, offset);
2840     }
2841 
2842     /** Opaque version of {@link #getDoubleVolatile(Object, long)} */
2843     @IntrinsicCandidate
2844     public final double getDoubleOpaque(Object o, long offset) {
2845         return getDoubleVolatile(o, offset);
2846     }
2847 
2848     /** Opaque version of {@link #putReferenceVolatile(Object, long, Object)} */
2849     @IntrinsicCandidate
2850     public final void putReferenceOpaque(Object o, long offset, Object x) {
2851         putReferenceVolatile(o, offset, x);
2852     }
2853 
2854     @ForceInline
2855     public final <V> void putFlatValueOpaque(Object o, long offset, int layout, Class<?> valueType, V x) {
2856         // this is stronger than opaque semantics
2857         putFlatValueRelease(o, offset, layout, valueType, x);
2858     }
2859 
2860     /** Opaque version of {@link #putBooleanVolatile(Object, long, boolean)} */
2861     @IntrinsicCandidate
2862     public final void putBooleanOpaque(Object o, long offset, boolean x) {
2863         putBooleanVolatile(o, offset, x);
2864     }
2865 
2866     /** Opaque version of {@link #putByteVolatile(Object, long, byte)} */
2867     @IntrinsicCandidate
2868     public final void putByteOpaque(Object o, long offset, byte x) {
2869         putByteVolatile(o, offset, x);
2870     }
2871 
2872     /** Opaque version of {@link #putShortVolatile(Object, long, short)} */
2873     @IntrinsicCandidate
2874     public final void putShortOpaque(Object o, long offset, short x) {
2875         putShortVolatile(o, offset, x);
2876     }
2877 
2878     /** Opaque version of {@link #putCharVolatile(Object, long, char)} */
2879     @IntrinsicCandidate
2880     public final void putCharOpaque(Object o, long offset, char x) {
2881         putCharVolatile(o, offset, x);
2882     }
2883 
2884     /** Opaque version of {@link #putIntVolatile(Object, long, int)} */
2885     @IntrinsicCandidate
2886     public final void putIntOpaque(Object o, long offset, int x) {
2887         putIntVolatile(o, offset, x);
2888     }
2889 
2890     /** Opaque version of {@link #putFloatVolatile(Object, long, float)} */
2891     @IntrinsicCandidate
2892     public final void putFloatOpaque(Object o, long offset, float x) {
2893         putFloatVolatile(o, offset, x);
2894     }
2895 
2896     /** Opaque version of {@link #putLongVolatile(Object, long, long)} */
2897     @IntrinsicCandidate
2898     public final void putLongOpaque(Object o, long offset, long x) {
2899         putLongVolatile(o, offset, x);
2900     }
2901 
2902     /** Opaque version of {@link #putDoubleVolatile(Object, long, double)} */
2903     @IntrinsicCandidate
2904     public final void putDoubleOpaque(Object o, long offset, double x) {
2905         putDoubleVolatile(o, offset, x);
2906     }
2907 
2908     @ForceInline
2909     private boolean compareAndSetFlatValueAsBytes(Object o, long offset, int layout, Class<?> valueType, Object expected, Object x) {
2910         // We turn the payload of an atomic value into a numeric value (of suitable type)
2911         // by storing the value into an array element (of matching layout) and by reading
2912         // back the array element as an integral value. After which we can implement the CAS
2913         // as a plain numeric CAS. Note: this only works if the payload contains no oops
2914         // (see VarHandles::isAtomicFlat).
2915         Object[] expectedArray = newSpecialArray(valueType, 1, layout);
2916         Object xArray = newSpecialArray(valueType, 1, layout);
2917         long base = arrayInstanceBaseOffset(expectedArray);
2918         int scale = arrayInstanceIndexScale(expectedArray);
2919         putFlatValue(expectedArray, base, layout, valueType, expected);
2920         putFlatValue(xArray, base, layout, valueType, x);
2921         switch (scale) {
2922             case 1: {
2923                 byte expectedByte = getByte(expectedArray, base);
2924                 byte xByte = getByte(xArray, base);
2925                 return compareAndSetByte(o, offset, expectedByte, xByte);
2926             }
2927             case 2: {
2928                 short expectedShort = getShort(expectedArray, base);
2929                 short xShort = getShort(xArray, base);
2930                 return compareAndSetShort(o, offset, expectedShort, xShort);
2931             }
2932             case 4: {
2933                 int expectedInt = getInt(expectedArray, base);
2934                 int xInt = getInt(xArray, base);
2935                 return compareAndSetInt(o, offset, expectedInt, xInt);
2936             }
2937             case 8: {
2938                 long expectedLong = getLong(expectedArray, base);
2939                 long xLong = getLong(xArray, base);
2940                 return compareAndSetLong(o, offset, expectedLong, xLong);
2941             }
2942             default: {
2943                 throw new UnsupportedOperationException();
2944             }
2945         }
2946     }
2947 
2948     /**
2949      * Unblocks the given thread blocked on {@code park}, or, if it is
2950      * not blocked, causes the subsequent call to {@code park} not to
2951      * block.  Note: this operation is "unsafe" solely because the
2952      * caller must somehow ensure that the thread has not been
2953      * destroyed. Nothing special is usually required to ensure this
2954      * when called from Java (in which there will ordinarily be a live
2955      * reference to the thread) but this is not nearly-automatically
2956      * so when calling from native code.
2957      *
2958      * @param thread the thread to unpark.
2959      */
2960     @IntrinsicCandidate
2961     public native void unpark(Object thread);
2962 
2963     /**
2964      * Blocks current thread, returning when a balancing
2965      * {@code unpark} occurs, or a balancing {@code unpark} has
2966      * already occurred, or the thread is interrupted, or, if not
2967      * absolute and time is not zero, the given time nanoseconds have
2968      * elapsed, or if absolute, the given deadline in milliseconds
2969      * since Epoch has passed, or spuriously (i.e., returning for no
2970      * "reason"). Note: This operation is in the Unsafe class only
2971      * because {@code unpark} is, so it would be strange to place it
2972      * elsewhere.
2973      */
2974     @IntrinsicCandidate
2975     public native void park(boolean isAbsolute, long time);
2976 
2977     /**
2978      * Gets the load average in the system run queue assigned
2979      * to the available processors averaged over various periods of time.
2980      * This method retrieves the given {@code nelem} samples and
2981      * assigns to the elements of the given {@code loadavg} array.
2982      * The system imposes a maximum of 3 samples, representing
2983      * averages over the last 1,  5,  and  15 minutes, respectively.
2984      *
2985      * @param loadavg an array of double of size nelems
2986      * @param nelems the number of samples to be retrieved and
2987      *        must be 1 to 3.
2988      *
2989      * @return the number of samples actually retrieved; or -1
2990      *         if the load average is unobtainable.
2991      */
2992     public int getLoadAverage(double[] loadavg, int nelems) {
2993         if (nelems < 0 || nelems > 3 || nelems > loadavg.length) {
2994             throw new ArrayIndexOutOfBoundsException();
2995         }
2996 
2997         return getLoadAverage0(loadavg, nelems);
2998     }
2999 
3000     // The following contain CAS-based Java implementations used on
3001     // platforms not supporting native instructions
3002 
3003     /**
3004      * Atomically adds the given value to the current value of a field
3005      * or array element within the given object {@code o}
3006      * at the given {@code offset}.
3007      *
3008      * @param o object/array to update the field/element in
3009      * @param offset field/element offset
3010      * @param delta the value to add
3011      * @return the previous value
3012      * @since 1.8
3013      */
3014     @IntrinsicCandidate
3015     public final int getAndAddInt(Object o, long offset, int delta) {
3016         int v;
3017         do {
3018             v = getIntVolatile(o, offset);
3019         } while (!weakCompareAndSetInt(o, offset, v, v + delta));
3020         return v;
3021     }
3022 
3023     @ForceInline
3024     public final int getAndAddIntRelease(Object o, long offset, int delta) {
3025         int v;
3026         do {
3027             v = getInt(o, offset);
3028         } while (!weakCompareAndSetIntRelease(o, offset, v, v + delta));
3029         return v;
3030     }
3031 
3032     @ForceInline
3033     public final int getAndAddIntAcquire(Object o, long offset, int delta) {
3034         int v;
3035         do {
3036             v = getIntAcquire(o, offset);
3037         } while (!weakCompareAndSetIntAcquire(o, offset, v, v + delta));
3038         return v;
3039     }
3040 
3041     /**
3042      * Atomically adds the given value to the current value of a field
3043      * or array element within the given object {@code o}
3044      * at the given {@code offset}.
3045      *
3046      * @param o object/array to update the field/element in
3047      * @param offset field/element offset
3048      * @param delta the value to add
3049      * @return the previous value
3050      * @since 1.8
3051      */
3052     @IntrinsicCandidate
3053     public final long getAndAddLong(Object o, long offset, long delta) {
3054         long v;
3055         do {
3056             v = getLongVolatile(o, offset);
3057         } while (!weakCompareAndSetLong(o, offset, v, v + delta));
3058         return v;
3059     }
3060 
3061     @ForceInline
3062     public final long getAndAddLongRelease(Object o, long offset, long delta) {
3063         long v;
3064         do {
3065             v = getLong(o, offset);
3066         } while (!weakCompareAndSetLongRelease(o, offset, v, v + delta));
3067         return v;
3068     }
3069 
3070     @ForceInline
3071     public final long getAndAddLongAcquire(Object o, long offset, long delta) {
3072         long v;
3073         do {
3074             v = getLongAcquire(o, offset);
3075         } while (!weakCompareAndSetLongAcquire(o, offset, v, v + delta));
3076         return v;
3077     }
3078 
3079     @IntrinsicCandidate
3080     public final byte getAndAddByte(Object o, long offset, byte delta) {
3081         byte v;
3082         do {
3083             v = getByteVolatile(o, offset);
3084         } while (!weakCompareAndSetByte(o, offset, v, (byte) (v + delta)));
3085         return v;
3086     }
3087 
3088     @ForceInline
3089     public final byte getAndAddByteRelease(Object o, long offset, byte delta) {
3090         byte v;
3091         do {
3092             v = getByte(o, offset);
3093         } while (!weakCompareAndSetByteRelease(o, offset, v, (byte) (v + delta)));
3094         return v;
3095     }
3096 
3097     @ForceInline
3098     public final byte getAndAddByteAcquire(Object o, long offset, byte delta) {
3099         byte v;
3100         do {
3101             v = getByteAcquire(o, offset);
3102         } while (!weakCompareAndSetByteAcquire(o, offset, v, (byte) (v + delta)));
3103         return v;
3104     }
3105 
3106     @IntrinsicCandidate
3107     public final short getAndAddShort(Object o, long offset, short delta) {
3108         short v;
3109         do {
3110             v = getShortVolatile(o, offset);
3111         } while (!weakCompareAndSetShort(o, offset, v, (short) (v + delta)));
3112         return v;
3113     }
3114 
3115     @ForceInline
3116     public final short getAndAddShortRelease(Object o, long offset, short delta) {
3117         short v;
3118         do {
3119             v = getShort(o, offset);
3120         } while (!weakCompareAndSetShortRelease(o, offset, v, (short) (v + delta)));
3121         return v;
3122     }
3123 
3124     @ForceInline
3125     public final short getAndAddShortAcquire(Object o, long offset, short delta) {
3126         short v;
3127         do {
3128             v = getShortAcquire(o, offset);
3129         } while (!weakCompareAndSetShortAcquire(o, offset, v, (short) (v + delta)));
3130         return v;
3131     }
3132 
3133     @ForceInline
3134     public final char getAndAddChar(Object o, long offset, char delta) {
3135         return (char) getAndAddShort(o, offset, (short) delta);
3136     }
3137 
3138     @ForceInline
3139     public final char getAndAddCharRelease(Object o, long offset, char delta) {
3140         return (char) getAndAddShortRelease(o, offset, (short) delta);
3141     }
3142 
3143     @ForceInline
3144     public final char getAndAddCharAcquire(Object o, long offset, char delta) {
3145         return (char) getAndAddShortAcquire(o, offset, (short) delta);
3146     }
3147 
3148     @ForceInline
3149     public final float getAndAddFloat(Object o, long offset, float delta) {
3150         int expectedBits;
3151         float v;
3152         do {
3153             // Load and CAS with the raw bits to avoid issues with NaNs and
3154             // possible bit conversion from signaling NaNs to quiet NaNs that
3155             // may result in the loop not terminating.
3156             expectedBits = getIntVolatile(o, offset);
3157             v = Float.intBitsToFloat(expectedBits);
3158         } while (!weakCompareAndSetInt(o, offset,
3159                                                 expectedBits, Float.floatToRawIntBits(v + delta)));
3160         return v;
3161     }
3162 
3163     @ForceInline
3164     public final float getAndAddFloatRelease(Object o, long offset, float delta) {
3165         int expectedBits;
3166         float v;
3167         do {
3168             // Load and CAS with the raw bits to avoid issues with NaNs and
3169             // possible bit conversion from signaling NaNs to quiet NaNs that
3170             // may result in the loop not terminating.
3171             expectedBits = getInt(o, offset);
3172             v = Float.intBitsToFloat(expectedBits);
3173         } while (!weakCompareAndSetIntRelease(o, offset,
3174                                                expectedBits, Float.floatToRawIntBits(v + delta)));
3175         return v;
3176     }
3177 
3178     @ForceInline
3179     public final float getAndAddFloatAcquire(Object o, long offset, float delta) {
3180         int expectedBits;
3181         float v;
3182         do {
3183             // Load and CAS with the raw bits to avoid issues with NaNs and
3184             // possible bit conversion from signaling NaNs to quiet NaNs that
3185             // may result in the loop not terminating.
3186             expectedBits = getIntAcquire(o, offset);
3187             v = Float.intBitsToFloat(expectedBits);
3188         } while (!weakCompareAndSetIntAcquire(o, offset,
3189                                                expectedBits, Float.floatToRawIntBits(v + delta)));
3190         return v;
3191     }
3192 
3193     @ForceInline
3194     public final double getAndAddDouble(Object o, long offset, double delta) {
3195         long expectedBits;
3196         double v;
3197         do {
3198             // Load and CAS with the raw bits to avoid issues with NaNs and
3199             // possible bit conversion from signaling NaNs to quiet NaNs that
3200             // may result in the loop not terminating.
3201             expectedBits = getLongVolatile(o, offset);
3202             v = Double.longBitsToDouble(expectedBits);
3203         } while (!weakCompareAndSetLong(o, offset,
3204                                                  expectedBits, Double.doubleToRawLongBits(v + delta)));
3205         return v;
3206     }
3207 
3208     @ForceInline
3209     public final double getAndAddDoubleRelease(Object o, long offset, double delta) {
3210         long expectedBits;
3211         double v;
3212         do {
3213             // Load and CAS with the raw bits to avoid issues with NaNs and
3214             // possible bit conversion from signaling NaNs to quiet NaNs that
3215             // may result in the loop not terminating.
3216             expectedBits = getLong(o, offset);
3217             v = Double.longBitsToDouble(expectedBits);
3218         } while (!weakCompareAndSetLongRelease(o, offset,
3219                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
3220         return v;
3221     }
3222 
3223     @ForceInline
3224     public final double getAndAddDoubleAcquire(Object o, long offset, double delta) {
3225         long expectedBits;
3226         double v;
3227         do {
3228             // Load and CAS with the raw bits to avoid issues with NaNs and
3229             // possible bit conversion from signaling NaNs to quiet NaNs that
3230             // may result in the loop not terminating.
3231             expectedBits = getLongAcquire(o, offset);
3232             v = Double.longBitsToDouble(expectedBits);
3233         } while (!weakCompareAndSetLongAcquire(o, offset,
3234                                                 expectedBits, Double.doubleToRawLongBits(v + delta)));
3235         return v;
3236     }
3237 
3238     /**
3239      * Atomically exchanges the given value with the current value of
3240      * a field or array element within the given object {@code o}
3241      * at the given {@code offset}.
3242      *
3243      * @param o object/array to update the field/element in
3244      * @param offset field/element offset
3245      * @param newValue new value
3246      * @return the previous value
3247      * @since 1.8
3248      */
3249     @IntrinsicCandidate
3250     public final int getAndSetInt(Object o, long offset, int newValue) {
3251         int v;
3252         do {
3253             v = getIntVolatile(o, offset);
3254         } while (!weakCompareAndSetInt(o, offset, v, newValue));
3255         return v;
3256     }
3257 
3258     @ForceInline
3259     public final int getAndSetIntRelease(Object o, long offset, int newValue) {
3260         int v;
3261         do {
3262             v = getInt(o, offset);
3263         } while (!weakCompareAndSetIntRelease(o, offset, v, newValue));
3264         return v;
3265     }
3266 
3267     @ForceInline
3268     public final int getAndSetIntAcquire(Object o, long offset, int newValue) {
3269         int v;
3270         do {
3271             v = getIntAcquire(o, offset);
3272         } while (!weakCompareAndSetIntAcquire(o, offset, v, newValue));
3273         return v;
3274     }
3275 
3276     /**
3277      * Atomically exchanges the given value with the current value of
3278      * a field or array element within the given object {@code o}
3279      * at the given {@code offset}.
3280      *
3281      * @param o object/array to update the field/element in
3282      * @param offset field/element offset
3283      * @param newValue new value
3284      * @return the previous value
3285      * @since 1.8
3286      */
3287     @IntrinsicCandidate
3288     public final long getAndSetLong(Object o, long offset, long newValue) {
3289         long v;
3290         do {
3291             v = getLongVolatile(o, offset);
3292         } while (!weakCompareAndSetLong(o, offset, v, newValue));
3293         return v;
3294     }
3295 
3296     @ForceInline
3297     public final long getAndSetLongRelease(Object o, long offset, long newValue) {
3298         long v;
3299         do {
3300             v = getLong(o, offset);
3301         } while (!weakCompareAndSetLongRelease(o, offset, v, newValue));
3302         return v;
3303     }
3304 
3305     @ForceInline
3306     public final long getAndSetLongAcquire(Object o, long offset, long newValue) {
3307         long v;
3308         do {
3309             v = getLongAcquire(o, offset);
3310         } while (!weakCompareAndSetLongAcquire(o, offset, v, newValue));
3311         return v;
3312     }
3313 
3314     /**
3315      * Atomically exchanges the given reference value with the current
3316      * reference value of a field or array element within the given
3317      * object {@code o} at the given {@code offset}.
3318      *
3319      * @param o object/array to update the field/element in
3320      * @param offset field/element offset
3321      * @param newValue new value
3322      * @return the previous value
3323      * @since 1.8
3324      */
3325     @IntrinsicCandidate
3326     public final Object getAndSetReference(Object o, long offset, Object newValue) {
3327         Object v;
3328         do {
3329             v = getReferenceVolatile(o, offset);
3330         } while (!weakCompareAndSetReference(o, offset, v, newValue));
3331         return v;
3332     }
3333 
3334     @ForceInline
3335     public final Object getAndSetReference(Object o, long offset, Class<?> valueType, Object newValue) {
3336         Object v;
3337         do {
3338             v = getReferenceVolatile(o, offset);
3339         } while (!compareAndSetReference(o, offset, valueType, v, newValue));
3340         return v;
3341     }
3342 
3343     @ForceInline
3344     public Object getAndSetFlatValue(Object o, long offset, int layoutKind, Class<?> valueType, Object newValue) {
3345         Object v;
3346         do {
3347             v = getFlatValueVolatile(o, offset, layoutKind, valueType);
3348         } while (!compareAndSetFlatValue(o, offset, layoutKind, valueType, v, newValue));
3349         return v;
3350     }
3351 
3352     @ForceInline
3353     public final Object getAndSetReferenceRelease(Object o, long offset, Object newValue) {
3354         Object v;
3355         do {
3356             v = getReference(o, offset);
3357         } while (!weakCompareAndSetReferenceRelease(o, offset, v, newValue));
3358         return v;
3359     }
3360 
3361     @ForceInline
3362     public final Object getAndSetReferenceRelease(Object o, long offset, Class<?> valueType, Object newValue) {
3363         return getAndSetReference(o, offset, valueType, newValue);
3364     }
3365 
3366     @ForceInline
3367     public Object getAndSetFlatValueRelease(Object o, long offset, int layoutKind, Class<?> valueType, Object x) {
3368         return getAndSetFlatValue(o, offset, layoutKind, valueType, x);
3369     }
3370 
3371     @ForceInline
3372     public final Object getAndSetReferenceAcquire(Object o, long offset, Object newValue) {
3373         Object v;
3374         do {
3375             v = getReferenceAcquire(o, offset);
3376         } while (!weakCompareAndSetReferenceAcquire(o, offset, v, newValue));
3377         return v;
3378     }
3379 
3380     @ForceInline
3381     public final Object getAndSetReferenceAcquire(Object o, long offset, Class<?> valueType, Object newValue) {
3382         return getAndSetReference(o, offset, valueType, newValue);
3383     }
3384 
3385     @ForceInline
3386     public Object getAndSetFlatValueAcquire(Object o, long offset, int layoutKind, Class<?> valueType, Object x) {
3387         return getAndSetFlatValue(o, offset, layoutKind, valueType, x);
3388     }
3389 
3390     @IntrinsicCandidate
3391     public final byte getAndSetByte(Object o, long offset, byte newValue) {
3392         byte v;
3393         do {
3394             v = getByteVolatile(o, offset);
3395         } while (!weakCompareAndSetByte(o, offset, v, newValue));
3396         return v;
3397     }
3398 
3399     @ForceInline
3400     public final byte getAndSetByteRelease(Object o, long offset, byte newValue) {
3401         byte v;
3402         do {
3403             v = getByte(o, offset);
3404         } while (!weakCompareAndSetByteRelease(o, offset, v, newValue));
3405         return v;
3406     }
3407 
3408     @ForceInline
3409     public final byte getAndSetByteAcquire(Object o, long offset, byte newValue) {
3410         byte v;
3411         do {
3412             v = getByteAcquire(o, offset);
3413         } while (!weakCompareAndSetByteAcquire(o, offset, v, newValue));
3414         return v;
3415     }
3416 
3417     @ForceInline
3418     public final boolean getAndSetBoolean(Object o, long offset, boolean newValue) {
3419         return byte2bool(getAndSetByte(o, offset, bool2byte(newValue)));
3420     }
3421 
3422     @ForceInline
3423     public final boolean getAndSetBooleanRelease(Object o, long offset, boolean newValue) {
3424         return byte2bool(getAndSetByteRelease(o, offset, bool2byte(newValue)));
3425     }
3426 
3427     @ForceInline
3428     public final boolean getAndSetBooleanAcquire(Object o, long offset, boolean newValue) {
3429         return byte2bool(getAndSetByteAcquire(o, offset, bool2byte(newValue)));
3430     }
3431 
3432     @IntrinsicCandidate
3433     public final short getAndSetShort(Object o, long offset, short newValue) {
3434         short v;
3435         do {
3436             v = getShortVolatile(o, offset);
3437         } while (!weakCompareAndSetShort(o, offset, v, newValue));
3438         return v;
3439     }
3440 
3441     @ForceInline
3442     public final short getAndSetShortRelease(Object o, long offset, short newValue) {
3443         short v;
3444         do {
3445             v = getShort(o, offset);
3446         } while (!weakCompareAndSetShortRelease(o, offset, v, newValue));
3447         return v;
3448     }
3449 
3450     @ForceInline
3451     public final short getAndSetShortAcquire(Object o, long offset, short newValue) {
3452         short v;
3453         do {
3454             v = getShortAcquire(o, offset);
3455         } while (!weakCompareAndSetShortAcquire(o, offset, v, newValue));
3456         return v;
3457     }
3458 
3459     @ForceInline
3460     public final char getAndSetChar(Object o, long offset, char newValue) {
3461         return s2c(getAndSetShort(o, offset, c2s(newValue)));
3462     }
3463 
3464     @ForceInline
3465     public final char getAndSetCharRelease(Object o, long offset, char newValue) {
3466         return s2c(getAndSetShortRelease(o, offset, c2s(newValue)));
3467     }
3468 
3469     @ForceInline
3470     public final char getAndSetCharAcquire(Object o, long offset, char newValue) {
3471         return s2c(getAndSetShortAcquire(o, offset, c2s(newValue)));
3472     }
3473 
3474     @ForceInline
3475     public final float getAndSetFloat(Object o, long offset, float newValue) {
3476         int v = getAndSetInt(o, offset, Float.floatToRawIntBits(newValue));
3477         return Float.intBitsToFloat(v);
3478     }
3479 
3480     @ForceInline
3481     public final float getAndSetFloatRelease(Object o, long offset, float newValue) {
3482         int v = getAndSetIntRelease(o, offset, Float.floatToRawIntBits(newValue));
3483         return Float.intBitsToFloat(v);
3484     }
3485 
3486     @ForceInline
3487     public final float getAndSetFloatAcquire(Object o, long offset, float newValue) {
3488         int v = getAndSetIntAcquire(o, offset, Float.floatToRawIntBits(newValue));
3489         return Float.intBitsToFloat(v);
3490     }
3491 
3492     @ForceInline
3493     public final double getAndSetDouble(Object o, long offset, double newValue) {
3494         long v = getAndSetLong(o, offset, Double.doubleToRawLongBits(newValue));
3495         return Double.longBitsToDouble(v);
3496     }
3497 
3498     @ForceInline
3499     public final double getAndSetDoubleRelease(Object o, long offset, double newValue) {
3500         long v = getAndSetLongRelease(o, offset, Double.doubleToRawLongBits(newValue));
3501         return Double.longBitsToDouble(v);
3502     }
3503 
3504     @ForceInline
3505     public final double getAndSetDoubleAcquire(Object o, long offset, double newValue) {
3506         long v = getAndSetLongAcquire(o, offset, Double.doubleToRawLongBits(newValue));
3507         return Double.longBitsToDouble(v);
3508     }
3509 
3510 
3511     // The following contain CAS-based Java implementations used on
3512     // platforms not supporting native instructions
3513 
3514     @ForceInline
3515     public final boolean getAndBitwiseOrBoolean(Object o, long offset, boolean mask) {
3516         return byte2bool(getAndBitwiseOrByte(o, offset, bool2byte(mask)));
3517     }
3518 
3519     @ForceInline
3520     public final boolean getAndBitwiseOrBooleanRelease(Object o, long offset, boolean mask) {
3521         return byte2bool(getAndBitwiseOrByteRelease(o, offset, bool2byte(mask)));
3522     }
3523 
3524     @ForceInline
3525     public final boolean getAndBitwiseOrBooleanAcquire(Object o, long offset, boolean mask) {
3526         return byte2bool(getAndBitwiseOrByteAcquire(o, offset, bool2byte(mask)));
3527     }
3528 
3529     @ForceInline
3530     public final boolean getAndBitwiseAndBoolean(Object o, long offset, boolean mask) {
3531         return byte2bool(getAndBitwiseAndByte(o, offset, bool2byte(mask)));
3532     }
3533 
3534     @ForceInline
3535     public final boolean getAndBitwiseAndBooleanRelease(Object o, long offset, boolean mask) {
3536         return byte2bool(getAndBitwiseAndByteRelease(o, offset, bool2byte(mask)));
3537     }
3538 
3539     @ForceInline
3540     public final boolean getAndBitwiseAndBooleanAcquire(Object o, long offset, boolean mask) {
3541         return byte2bool(getAndBitwiseAndByteAcquire(o, offset, bool2byte(mask)));
3542     }
3543 
3544     @ForceInline
3545     public final boolean getAndBitwiseXorBoolean(Object o, long offset, boolean mask) {
3546         return byte2bool(getAndBitwiseXorByte(o, offset, bool2byte(mask)));
3547     }
3548 
3549     @ForceInline
3550     public final boolean getAndBitwiseXorBooleanRelease(Object o, long offset, boolean mask) {
3551         return byte2bool(getAndBitwiseXorByteRelease(o, offset, bool2byte(mask)));
3552     }
3553 
3554     @ForceInline
3555     public final boolean getAndBitwiseXorBooleanAcquire(Object o, long offset, boolean mask) {
3556         return byte2bool(getAndBitwiseXorByteAcquire(o, offset, bool2byte(mask)));
3557     }
3558 
3559 
3560     @ForceInline
3561     public final byte getAndBitwiseOrByte(Object o, long offset, byte mask) {
3562         byte current;
3563         do {
3564             current = getByteVolatile(o, offset);
3565         } while (!weakCompareAndSetByte(o, offset,
3566                                                   current, (byte) (current | mask)));
3567         return current;
3568     }
3569 
3570     @ForceInline
3571     public final byte getAndBitwiseOrByteRelease(Object o, long offset, byte mask) {
3572         byte current;
3573         do {
3574             current = getByte(o, offset);
3575         } while (!weakCompareAndSetByteRelease(o, offset,
3576                                                  current, (byte) (current | mask)));
3577         return current;
3578     }
3579 
3580     @ForceInline
3581     public final byte getAndBitwiseOrByteAcquire(Object o, long offset, byte mask) {
3582         byte current;
3583         do {
3584             // Plain read, the value is a hint, the acquire CAS does the work
3585             current = getByte(o, offset);
3586         } while (!weakCompareAndSetByteAcquire(o, offset,
3587                                                  current, (byte) (current | mask)));
3588         return current;
3589     }
3590 
3591     @ForceInline
3592     public final byte getAndBitwiseAndByte(Object o, long offset, byte mask) {
3593         byte current;
3594         do {
3595             current = getByteVolatile(o, offset);
3596         } while (!weakCompareAndSetByte(o, offset,
3597                                                   current, (byte) (current & mask)));
3598         return current;
3599     }
3600 
3601     @ForceInline
3602     public final byte getAndBitwiseAndByteRelease(Object o, long offset, byte mask) {
3603         byte current;
3604         do {
3605             current = getByte(o, offset);
3606         } while (!weakCompareAndSetByteRelease(o, offset,
3607                                                  current, (byte) (current & mask)));
3608         return current;
3609     }
3610 
3611     @ForceInline
3612     public final byte getAndBitwiseAndByteAcquire(Object o, long offset, byte mask) {
3613         byte current;
3614         do {
3615             // Plain read, the value is a hint, the acquire CAS does the work
3616             current = getByte(o, offset);
3617         } while (!weakCompareAndSetByteAcquire(o, offset,
3618                                                  current, (byte) (current & mask)));
3619         return current;
3620     }
3621 
3622     @ForceInline
3623     public final byte getAndBitwiseXorByte(Object o, long offset, byte mask) {
3624         byte current;
3625         do {
3626             current = getByteVolatile(o, offset);
3627         } while (!weakCompareAndSetByte(o, offset,
3628                                                   current, (byte) (current ^ mask)));
3629         return current;
3630     }
3631 
3632     @ForceInline
3633     public final byte getAndBitwiseXorByteRelease(Object o, long offset, byte mask) {
3634         byte current;
3635         do {
3636             current = getByte(o, offset);
3637         } while (!weakCompareAndSetByteRelease(o, offset,
3638                                                  current, (byte) (current ^ mask)));
3639         return current;
3640     }
3641 
3642     @ForceInline
3643     public final byte getAndBitwiseXorByteAcquire(Object o, long offset, byte mask) {
3644         byte current;
3645         do {
3646             // Plain read, the value is a hint, the acquire CAS does the work
3647             current = getByte(o, offset);
3648         } while (!weakCompareAndSetByteAcquire(o, offset,
3649                                                  current, (byte) (current ^ mask)));
3650         return current;
3651     }
3652 
3653 
3654     @ForceInline
3655     public final char getAndBitwiseOrChar(Object o, long offset, char mask) {
3656         return s2c(getAndBitwiseOrShort(o, offset, c2s(mask)));
3657     }
3658 
3659     @ForceInline
3660     public final char getAndBitwiseOrCharRelease(Object o, long offset, char mask) {
3661         return s2c(getAndBitwiseOrShortRelease(o, offset, c2s(mask)));
3662     }
3663 
3664     @ForceInline
3665     public final char getAndBitwiseOrCharAcquire(Object o, long offset, char mask) {
3666         return s2c(getAndBitwiseOrShortAcquire(o, offset, c2s(mask)));
3667     }
3668 
3669     @ForceInline
3670     public final char getAndBitwiseAndChar(Object o, long offset, char mask) {
3671         return s2c(getAndBitwiseAndShort(o, offset, c2s(mask)));
3672     }
3673 
3674     @ForceInline
3675     public final char getAndBitwiseAndCharRelease(Object o, long offset, char mask) {
3676         return s2c(getAndBitwiseAndShortRelease(o, offset, c2s(mask)));
3677     }
3678 
3679     @ForceInline
3680     public final char getAndBitwiseAndCharAcquire(Object o, long offset, char mask) {
3681         return s2c(getAndBitwiseAndShortAcquire(o, offset, c2s(mask)));
3682     }
3683 
3684     @ForceInline
3685     public final char getAndBitwiseXorChar(Object o, long offset, char mask) {
3686         return s2c(getAndBitwiseXorShort(o, offset, c2s(mask)));
3687     }
3688 
3689     @ForceInline
3690     public final char getAndBitwiseXorCharRelease(Object o, long offset, char mask) {
3691         return s2c(getAndBitwiseXorShortRelease(o, offset, c2s(mask)));
3692     }
3693 
3694     @ForceInline
3695     public final char getAndBitwiseXorCharAcquire(Object o, long offset, char mask) {
3696         return s2c(getAndBitwiseXorShortAcquire(o, offset, c2s(mask)));
3697     }
3698 
3699 
3700     @ForceInline
3701     public final short getAndBitwiseOrShort(Object o, long offset, short mask) {
3702         short current;
3703         do {
3704             current = getShortVolatile(o, offset);
3705         } while (!weakCompareAndSetShort(o, offset,
3706                                                 current, (short) (current | mask)));
3707         return current;
3708     }
3709 
3710     @ForceInline
3711     public final short getAndBitwiseOrShortRelease(Object o, long offset, short mask) {
3712         short current;
3713         do {
3714             current = getShort(o, offset);
3715         } while (!weakCompareAndSetShortRelease(o, offset,
3716                                                current, (short) (current | mask)));
3717         return current;
3718     }
3719 
3720     @ForceInline
3721     public final short getAndBitwiseOrShortAcquire(Object o, long offset, short mask) {
3722         short current;
3723         do {
3724             // Plain read, the value is a hint, the acquire CAS does the work
3725             current = getShort(o, offset);
3726         } while (!weakCompareAndSetShortAcquire(o, offset,
3727                                                current, (short) (current | mask)));
3728         return current;
3729     }
3730 
3731     @ForceInline
3732     public final short getAndBitwiseAndShort(Object o, long offset, short mask) {
3733         short current;
3734         do {
3735             current = getShortVolatile(o, offset);
3736         } while (!weakCompareAndSetShort(o, offset,
3737                                                 current, (short) (current & mask)));
3738         return current;
3739     }
3740 
3741     @ForceInline
3742     public final short getAndBitwiseAndShortRelease(Object o, long offset, short mask) {
3743         short current;
3744         do {
3745             current = getShort(o, offset);
3746         } while (!weakCompareAndSetShortRelease(o, offset,
3747                                                current, (short) (current & mask)));
3748         return current;
3749     }
3750 
3751     @ForceInline
3752     public final short getAndBitwiseAndShortAcquire(Object o, long offset, short mask) {
3753         short current;
3754         do {
3755             // Plain read, the value is a hint, the acquire CAS does the work
3756             current = getShort(o, offset);
3757         } while (!weakCompareAndSetShortAcquire(o, offset,
3758                                                current, (short) (current & mask)));
3759         return current;
3760     }
3761 
3762     @ForceInline
3763     public final short getAndBitwiseXorShort(Object o, long offset, short mask) {
3764         short current;
3765         do {
3766             current = getShortVolatile(o, offset);
3767         } while (!weakCompareAndSetShort(o, offset,
3768                                                 current, (short) (current ^ mask)));
3769         return current;
3770     }
3771 
3772     @ForceInline
3773     public final short getAndBitwiseXorShortRelease(Object o, long offset, short mask) {
3774         short current;
3775         do {
3776             current = getShort(o, offset);
3777         } while (!weakCompareAndSetShortRelease(o, offset,
3778                                                current, (short) (current ^ mask)));
3779         return current;
3780     }
3781 
3782     @ForceInline
3783     public final short getAndBitwiseXorShortAcquire(Object o, long offset, short mask) {
3784         short current;
3785         do {
3786             // Plain read, the value is a hint, the acquire CAS does the work
3787             current = getShort(o, offset);
3788         } while (!weakCompareAndSetShortAcquire(o, offset,
3789                                                current, (short) (current ^ mask)));
3790         return current;
3791     }
3792 
3793 
3794     @ForceInline
3795     public final int getAndBitwiseOrInt(Object o, long offset, int mask) {
3796         int current;
3797         do {
3798             current = getIntVolatile(o, offset);
3799         } while (!weakCompareAndSetInt(o, offset,
3800                                                 current, current | mask));
3801         return current;
3802     }
3803 
3804     @ForceInline
3805     public final int getAndBitwiseOrIntRelease(Object o, long offset, int mask) {
3806         int current;
3807         do {
3808             current = getInt(o, offset);
3809         } while (!weakCompareAndSetIntRelease(o, offset,
3810                                                current, current | mask));
3811         return current;
3812     }
3813 
3814     @ForceInline
3815     public final int getAndBitwiseOrIntAcquire(Object o, long offset, int mask) {
3816         int current;
3817         do {
3818             // Plain read, the value is a hint, the acquire CAS does the work
3819             current = getInt(o, offset);
3820         } while (!weakCompareAndSetIntAcquire(o, offset,
3821                                                current, current | mask));
3822         return current;
3823     }
3824 
3825     /**
3826      * Atomically replaces the current value of a field or array element within
3827      * the given object with the result of bitwise AND between the current value
3828      * and mask.
3829      *
3830      * @param o object/array to update the field/element in
3831      * @param offset field/element offset
3832      * @param mask the mask value
3833      * @return the previous value
3834      * @since 9
3835      */
3836     @ForceInline
3837     public final int getAndBitwiseAndInt(Object o, long offset, int mask) {
3838         int current;
3839         do {
3840             current = getIntVolatile(o, offset);
3841         } while (!weakCompareAndSetInt(o, offset,
3842                                                 current, current & mask));
3843         return current;
3844     }
3845 
3846     @ForceInline
3847     public final int getAndBitwiseAndIntRelease(Object o, long offset, int mask) {
3848         int current;
3849         do {
3850             current = getInt(o, offset);
3851         } while (!weakCompareAndSetIntRelease(o, offset,
3852                                                current, current & mask));
3853         return current;
3854     }
3855 
3856     @ForceInline
3857     public final int getAndBitwiseAndIntAcquire(Object o, long offset, int mask) {
3858         int current;
3859         do {
3860             // Plain read, the value is a hint, the acquire CAS does the work
3861             current = getInt(o, offset);
3862         } while (!weakCompareAndSetIntAcquire(o, offset,
3863                                                current, current & mask));
3864         return current;
3865     }
3866 
3867     @ForceInline
3868     public final int getAndBitwiseXorInt(Object o, long offset, int mask) {
3869         int current;
3870         do {
3871             current = getIntVolatile(o, offset);
3872         } while (!weakCompareAndSetInt(o, offset,
3873                                                 current, current ^ mask));
3874         return current;
3875     }
3876 
3877     @ForceInline
3878     public final int getAndBitwiseXorIntRelease(Object o, long offset, int mask) {
3879         int current;
3880         do {
3881             current = getInt(o, offset);
3882         } while (!weakCompareAndSetIntRelease(o, offset,
3883                                                current, current ^ mask));
3884         return current;
3885     }
3886 
3887     @ForceInline
3888     public final int getAndBitwiseXorIntAcquire(Object o, long offset, int mask) {
3889         int current;
3890         do {
3891             // Plain read, the value is a hint, the acquire CAS does the work
3892             current = getInt(o, offset);
3893         } while (!weakCompareAndSetIntAcquire(o, offset,
3894                                                current, current ^ mask));
3895         return current;
3896     }
3897 
3898 
3899     @ForceInline
3900     public final long getAndBitwiseOrLong(Object o, long offset, long mask) {
3901         long current;
3902         do {
3903             current = getLongVolatile(o, offset);
3904         } while (!weakCompareAndSetLong(o, offset,
3905                                                 current, current | mask));
3906         return current;
3907     }
3908 
3909     @ForceInline
3910     public final long getAndBitwiseOrLongRelease(Object o, long offset, long mask) {
3911         long current;
3912         do {
3913             current = getLong(o, offset);
3914         } while (!weakCompareAndSetLongRelease(o, offset,
3915                                                current, current | mask));
3916         return current;
3917     }
3918 
3919     @ForceInline
3920     public final long getAndBitwiseOrLongAcquire(Object o, long offset, long mask) {
3921         long current;
3922         do {
3923             // Plain read, the value is a hint, the acquire CAS does the work
3924             current = getLong(o, offset);
3925         } while (!weakCompareAndSetLongAcquire(o, offset,
3926                                                current, current | mask));
3927         return current;
3928     }
3929 
3930     @ForceInline
3931     public final long getAndBitwiseAndLong(Object o, long offset, long mask) {
3932         long current;
3933         do {
3934             current = getLongVolatile(o, offset);
3935         } while (!weakCompareAndSetLong(o, offset,
3936                                                 current, current & mask));
3937         return current;
3938     }
3939 
3940     @ForceInline
3941     public final long getAndBitwiseAndLongRelease(Object o, long offset, long mask) {
3942         long current;
3943         do {
3944             current = getLong(o, offset);
3945         } while (!weakCompareAndSetLongRelease(o, offset,
3946                                                current, current & mask));
3947         return current;
3948     }
3949 
3950     @ForceInline
3951     public final long getAndBitwiseAndLongAcquire(Object o, long offset, long mask) {
3952         long current;
3953         do {
3954             // Plain read, the value is a hint, the acquire CAS does the work
3955             current = getLong(o, offset);
3956         } while (!weakCompareAndSetLongAcquire(o, offset,
3957                                                current, current & mask));
3958         return current;
3959     }
3960 
3961     @ForceInline
3962     public final long getAndBitwiseXorLong(Object o, long offset, long mask) {
3963         long current;
3964         do {
3965             current = getLongVolatile(o, offset);
3966         } while (!weakCompareAndSetLong(o, offset,
3967                                                 current, current ^ mask));
3968         return current;
3969     }
3970 
3971     @ForceInline
3972     public final long getAndBitwiseXorLongRelease(Object o, long offset, long mask) {
3973         long current;
3974         do {
3975             current = getLong(o, offset);
3976         } while (!weakCompareAndSetLongRelease(o, offset,
3977                                                current, current ^ mask));
3978         return current;
3979     }
3980 
3981     @ForceInline
3982     public final long getAndBitwiseXorLongAcquire(Object o, long offset, long mask) {
3983         long current;
3984         do {
3985             // Plain read, the value is a hint, the acquire CAS does the work
3986             current = getLong(o, offset);
3987         } while (!weakCompareAndSetLongAcquire(o, offset,
3988                                                current, current ^ mask));
3989         return current;
3990     }
3991 
3992 
3993 
3994     /**
3995      * Ensures that loads before the fence will not be reordered with loads and
3996      * stores after the fence; a "LoadLoad plus LoadStore barrier".
3997      *
3998      * Corresponds to C11 atomic_thread_fence(memory_order_acquire)
3999      * (an "acquire fence").
4000      *
4001      * Provides a LoadLoad barrier followed by a LoadStore barrier.
4002      *
4003      * @since 1.8
4004      */
4005     @IntrinsicCandidate
4006     public final void loadFence() {
4007         // If loadFence intrinsic is not available, fall back to full fence.
4008         fullFence();
4009     }
4010 
4011     /**
4012      * Ensures that loads and stores before the fence will not be reordered with
4013      * stores after the fence; a "StoreStore plus LoadStore barrier".
4014      *
4015      * Corresponds to C11 atomic_thread_fence(memory_order_release)
4016      * (a "release fence").
4017      *
4018      * Provides a StoreStore barrier followed by a LoadStore barrier.
4019      *
4020      * @since 1.8
4021      */
4022     @IntrinsicCandidate
4023     public final void storeFence() {
4024         // If storeFence intrinsic is not available, fall back to full fence.
4025         fullFence();
4026     }
4027 
4028     /**
4029      * Ensures that loads and stores before the fence will not be reordered
4030      * with loads and stores after the fence.  Implies the effects of both
4031      * loadFence() and storeFence(), and in addition, the effect of a StoreLoad
4032      * barrier.
4033      *
4034      * Corresponds to C11 atomic_thread_fence(memory_order_seq_cst).
4035      * @since 1.8
4036      */
4037     @IntrinsicCandidate
4038     public native void fullFence();
4039 
4040     /**
4041      * Ensures that loads before the fence will not be reordered with
4042      * loads after the fence.
4043      *
4044      * @implNote
4045      * This method is operationally equivalent to {@link #loadFence()}.
4046      *
4047      * @since 9
4048      */
4049     public final void loadLoadFence() {
4050         loadFence();
4051     }
4052 
4053     /**
4054      * Ensures that stores before the fence will not be reordered with
4055      * stores after the fence.
4056      *
4057      * @since 9
4058      */
4059     @IntrinsicCandidate
4060     public final void storeStoreFence() {
4061         // If storeStoreFence intrinsic is not available, fall back to storeFence.
4062         storeFence();
4063     }
4064 
4065     /**
4066      * Throws IllegalAccessError; for use by the VM for access control
4067      * error support.
4068      * @since 1.8
4069      */
4070     private static void throwIllegalAccessError() {
4071         throw new IllegalAccessError();
4072     }
4073 
4074     /**
4075      * Throws NoSuchMethodError; for use by the VM for redefinition support.
4076      * @since 13
4077      */
4078     private static void throwNoSuchMethodError() {
4079         throw new NoSuchMethodError();
4080     }
4081 
4082     /**
4083      * @return Returns true if the native byte ordering of this
4084      * platform is big-endian, false if it is little-endian.
4085      */
4086     public final boolean isBigEndian() { return BIG_ENDIAN; }
4087 
4088     /**
4089      * @return Returns true if this platform is capable of performing
4090      * accesses at addresses which are not aligned for the type of the
4091      * primitive type being accessed, false otherwise.
4092      */
4093     public final boolean unalignedAccess() { return UNALIGNED_ACCESS; }
4094 
4095     /**
4096      * Fetches a value at some byte offset into a given Java object.
4097      * More specifically, fetches a value within the given object
4098      * <code>o</code> at the given offset, or (if <code>o</code> is
4099      * null) from the memory address whose numerical value is the
4100      * given offset.  <p>
4101      *
4102      * The specification of this method is the same as {@link
4103      * #getLong(Object, long)} except that the offset does not need to
4104      * have been obtained from {@link #objectFieldOffset} on the
4105      * {@link java.lang.reflect.Field} of some Java field.  The value
4106      * in memory is raw data, and need not correspond to any Java
4107      * variable.  Unless <code>o</code> is null, the value accessed
4108      * must be entirely within the allocated object.  The endianness
4109      * of the value in memory is the endianness of the native platform.
4110      *
4111      * <p> The read will be atomic with respect to the largest power
4112      * of two that divides the GCD of the offset and the storage size.
4113      * For example, getLongUnaligned will make atomic reads of 2-, 4-,
4114      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
4115      * respectively.  There are no other guarantees of atomicity.
4116      * <p>
4117      * 8-byte atomicity is only guaranteed on platforms on which
4118      * support atomic accesses to longs.
4119      *
4120      * @param o Java heap object in which the value resides, if any, else
4121      *        null
4122      * @param offset The offset in bytes from the start of the object
4123      * @return the value fetched from the indicated object
4124      * @throws RuntimeException No defined exceptions are thrown, not even
4125      *         {@link NullPointerException}
4126      * @since 9
4127      */
4128     @IntrinsicCandidate
4129     public final long getLongUnaligned(Object o, long offset) {
4130         if ((offset & 7) == 0) {
4131             return getLong(o, offset);
4132         } else if ((offset & 3) == 0) {
4133             return makeLong(getInt(o, offset),
4134                             getInt(o, offset + 4));
4135         } else if ((offset & 1) == 0) {
4136             return makeLong(getShort(o, offset),
4137                             getShort(o, offset + 2),
4138                             getShort(o, offset + 4),
4139                             getShort(o, offset + 6));
4140         } else {
4141             return makeLong(getByte(o, offset),
4142                             getByte(o, offset + 1),
4143                             getByte(o, offset + 2),
4144                             getByte(o, offset + 3),
4145                             getByte(o, offset + 4),
4146                             getByte(o, offset + 5),
4147                             getByte(o, offset + 6),
4148                             getByte(o, offset + 7));
4149         }
4150     }
4151     /**
4152      * As {@link #getLongUnaligned(Object, long)} but with an
4153      * additional argument which specifies the endianness of the value
4154      * as stored in memory.
4155      *
4156      * @param o Java heap object in which the variable resides
4157      * @param offset The offset in bytes from the start of the object
4158      * @param bigEndian The endianness of the value
4159      * @return the value fetched from the indicated object
4160      * @since 9
4161      */
4162     public final long getLongUnaligned(Object o, long offset, boolean bigEndian) {
4163         return convEndian(bigEndian, getLongUnaligned(o, offset));
4164     }
4165 
4166     /** @see #getLongUnaligned(Object, long) */
4167     @IntrinsicCandidate
4168     public final int getIntUnaligned(Object o, long offset) {
4169         if ((offset & 3) == 0) {
4170             return getInt(o, offset);
4171         } else if ((offset & 1) == 0) {
4172             return makeInt(getShort(o, offset),
4173                            getShort(o, offset + 2));
4174         } else {
4175             return makeInt(getByte(o, offset),
4176                            getByte(o, offset + 1),
4177                            getByte(o, offset + 2),
4178                            getByte(o, offset + 3));
4179         }
4180     }
4181     /** @see #getLongUnaligned(Object, long, boolean) */
4182     public final int getIntUnaligned(Object o, long offset, boolean bigEndian) {
4183         return convEndian(bigEndian, getIntUnaligned(o, offset));
4184     }
4185 
4186     /** @see #getLongUnaligned(Object, long) */
4187     @IntrinsicCandidate
4188     public final short getShortUnaligned(Object o, long offset) {
4189         if ((offset & 1) == 0) {
4190             return getShort(o, offset);
4191         } else {
4192             return makeShort(getByte(o, offset),
4193                              getByte(o, offset + 1));
4194         }
4195     }
4196     /** @see #getLongUnaligned(Object, long, boolean) */
4197     public final short getShortUnaligned(Object o, long offset, boolean bigEndian) {
4198         return convEndian(bigEndian, getShortUnaligned(o, offset));
4199     }
4200 
4201     /** @see #getLongUnaligned(Object, long) */
4202     @IntrinsicCandidate
4203     public final char getCharUnaligned(Object o, long offset) {
4204         if ((offset & 1) == 0) {
4205             return getChar(o, offset);
4206         } else {
4207             return (char)makeShort(getByte(o, offset),
4208                                    getByte(o, offset + 1));
4209         }
4210     }
4211 
4212     /** @see #getLongUnaligned(Object, long, boolean) */
4213     public final char getCharUnaligned(Object o, long offset, boolean bigEndian) {
4214         return convEndian(bigEndian, getCharUnaligned(o, offset));
4215     }
4216 
4217     /**
4218      * Stores a value at some byte offset into a given Java object.
4219      * <p>
4220      * The specification of this method is the same as {@link
4221      * #getLong(Object, long)} except that the offset does not need to
4222      * have been obtained from {@link #objectFieldOffset} on the
4223      * {@link java.lang.reflect.Field} of some Java field.  The value
4224      * in memory is raw data, and need not correspond to any Java
4225      * variable.  The endianness of the value in memory is the
4226      * endianness of the native platform.
4227      * <p>
4228      * The write will be atomic with respect to the largest power of
4229      * two that divides the GCD of the offset and the storage size.
4230      * For example, putLongUnaligned will make atomic writes of 2-, 4-,
4231      * or 8-byte storage units if the offset is zero mod 2, 4, or 8,
4232      * respectively.  There are no other guarantees of atomicity.
4233      * <p>
4234      * 8-byte atomicity is only guaranteed on platforms on which
4235      * support atomic accesses to longs.
4236      *
4237      * @param o Java heap object in which the value resides, if any, else
4238      *        null
4239      * @param offset The offset in bytes from the start of the object
4240      * @param x the value to store
4241      * @throws RuntimeException No defined exceptions are thrown, not even
4242      *         {@link NullPointerException}
4243      * @since 9
4244      */
4245     @IntrinsicCandidate
4246     public final void putLongUnaligned(Object o, long offset, long x) {
4247         if ((offset & 7) == 0) {
4248             putLong(o, offset, x);
4249         } else if ((offset & 3) == 0) {
4250             putLongParts(o, offset,
4251                          (int)(x >> 0),
4252                          (int)(x >>> 32));
4253         } else if ((offset & 1) == 0) {
4254             putLongParts(o, offset,
4255                          (short)(x >>> 0),
4256                          (short)(x >>> 16),
4257                          (short)(x >>> 32),
4258                          (short)(x >>> 48));
4259         } else {
4260             putLongParts(o, offset,
4261                          (byte)(x >>> 0),
4262                          (byte)(x >>> 8),
4263                          (byte)(x >>> 16),
4264                          (byte)(x >>> 24),
4265                          (byte)(x >>> 32),
4266                          (byte)(x >>> 40),
4267                          (byte)(x >>> 48),
4268                          (byte)(x >>> 56));
4269         }
4270     }
4271 
4272     /**
4273      * As {@link #putLongUnaligned(Object, long, long)} but with an additional
4274      * argument which specifies the endianness of the value as stored in memory.
4275      * @param o Java heap object in which the value resides
4276      * @param offset The offset in bytes from the start of the object
4277      * @param x the value to store
4278      * @param bigEndian The endianness of the value
4279      * @throws RuntimeException No defined exceptions are thrown, not even
4280      *         {@link NullPointerException}
4281      * @since 9
4282      */
4283     public final void putLongUnaligned(Object o, long offset, long x, boolean bigEndian) {
4284         putLongUnaligned(o, offset, convEndian(bigEndian, x));
4285     }
4286 
4287     /** @see #putLongUnaligned(Object, long, long) */
4288     @IntrinsicCandidate
4289     public final void putIntUnaligned(Object o, long offset, int x) {
4290         if ((offset & 3) == 0) {
4291             putInt(o, offset, x);
4292         } else if ((offset & 1) == 0) {
4293             putIntParts(o, offset,
4294                         (short)(x >> 0),
4295                         (short)(x >>> 16));
4296         } else {
4297             putIntParts(o, offset,
4298                         (byte)(x >>> 0),
4299                         (byte)(x >>> 8),
4300                         (byte)(x >>> 16),
4301                         (byte)(x >>> 24));
4302         }
4303     }
4304     /** @see #putLongUnaligned(Object, long, long, boolean) */
4305     public final void putIntUnaligned(Object o, long offset, int x, boolean bigEndian) {
4306         putIntUnaligned(o, offset, convEndian(bigEndian, x));
4307     }
4308 
4309     /** @see #putLongUnaligned(Object, long, long) */
4310     @IntrinsicCandidate
4311     public final void putShortUnaligned(Object o, long offset, short x) {
4312         if ((offset & 1) == 0) {
4313             putShort(o, offset, x);
4314         } else {
4315             putShortParts(o, offset,
4316                           (byte)(x >>> 0),
4317                           (byte)(x >>> 8));
4318         }
4319     }
4320     /** @see #putLongUnaligned(Object, long, long, boolean) */
4321     public final void putShortUnaligned(Object o, long offset, short x, boolean bigEndian) {
4322         putShortUnaligned(o, offset, convEndian(bigEndian, x));
4323     }
4324 
4325     /** @see #putLongUnaligned(Object, long, long) */
4326     @IntrinsicCandidate
4327     public final void putCharUnaligned(Object o, long offset, char x) {
4328         putShortUnaligned(o, offset, (short)x);
4329     }
4330     /** @see #putLongUnaligned(Object, long, long, boolean) */
4331     public final void putCharUnaligned(Object o, long offset, char x, boolean bigEndian) {
4332         putCharUnaligned(o, offset, convEndian(bigEndian, x));
4333     }
4334 
4335     private static int pickPos(int top, int pos) { return BIG_ENDIAN ? top - pos : pos; }
4336 
4337     // These methods construct integers from bytes.  The byte ordering
4338     // is the native endianness of this platform.
4339     private static long makeLong(byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
4340         return ((toUnsignedLong(i0) << pickPos(56, 0))
4341               | (toUnsignedLong(i1) << pickPos(56, 8))
4342               | (toUnsignedLong(i2) << pickPos(56, 16))
4343               | (toUnsignedLong(i3) << pickPos(56, 24))
4344               | (toUnsignedLong(i4) << pickPos(56, 32))
4345               | (toUnsignedLong(i5) << pickPos(56, 40))
4346               | (toUnsignedLong(i6) << pickPos(56, 48))
4347               | (toUnsignedLong(i7) << pickPos(56, 56)));
4348     }
4349     private static long makeLong(short i0, short i1, short i2, short i3) {
4350         return ((toUnsignedLong(i0) << pickPos(48, 0))
4351               | (toUnsignedLong(i1) << pickPos(48, 16))
4352               | (toUnsignedLong(i2) << pickPos(48, 32))
4353               | (toUnsignedLong(i3) << pickPos(48, 48)));
4354     }
4355     private static long makeLong(int i0, int i1) {
4356         return (toUnsignedLong(i0) << pickPos(32, 0))
4357              | (toUnsignedLong(i1) << pickPos(32, 32));
4358     }
4359     private static int makeInt(short i0, short i1) {
4360         return (toUnsignedInt(i0) << pickPos(16, 0))
4361              | (toUnsignedInt(i1) << pickPos(16, 16));
4362     }
4363     private static int makeInt(byte i0, byte i1, byte i2, byte i3) {
4364         return ((toUnsignedInt(i0) << pickPos(24, 0))
4365               | (toUnsignedInt(i1) << pickPos(24, 8))
4366               | (toUnsignedInt(i2) << pickPos(24, 16))
4367               | (toUnsignedInt(i3) << pickPos(24, 24)));
4368     }
4369     private static short makeShort(byte i0, byte i1) {
4370         return (short)((toUnsignedInt(i0) << pickPos(8, 0))
4371                      | (toUnsignedInt(i1) << pickPos(8, 8)));
4372     }
4373 
4374     private static byte  pick(byte  le, byte  be) { return BIG_ENDIAN ? be : le; }
4375     private static short pick(short le, short be) { return BIG_ENDIAN ? be : le; }
4376     private static int   pick(int   le, int   be) { return BIG_ENDIAN ? be : le; }
4377 
4378     // These methods write integers to memory from smaller parts
4379     // provided by their caller.  The ordering in which these parts
4380     // are written is the native endianness of this platform.
4381     private void putLongParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3, byte i4, byte i5, byte i6, byte i7) {
4382         putByte(o, offset + 0, pick(i0, i7));
4383         putByte(o, offset + 1, pick(i1, i6));
4384         putByte(o, offset + 2, pick(i2, i5));
4385         putByte(o, offset + 3, pick(i3, i4));
4386         putByte(o, offset + 4, pick(i4, i3));
4387         putByte(o, offset + 5, pick(i5, i2));
4388         putByte(o, offset + 6, pick(i6, i1));
4389         putByte(o, offset + 7, pick(i7, i0));
4390     }
4391     private void putLongParts(Object o, long offset, short i0, short i1, short i2, short i3) {
4392         putShort(o, offset + 0, pick(i0, i3));
4393         putShort(o, offset + 2, pick(i1, i2));
4394         putShort(o, offset + 4, pick(i2, i1));
4395         putShort(o, offset + 6, pick(i3, i0));
4396     }
4397     private void putLongParts(Object o, long offset, int i0, int i1) {
4398         putInt(o, offset + 0, pick(i0, i1));
4399         putInt(o, offset + 4, pick(i1, i0));
4400     }
4401     private void putIntParts(Object o, long offset, short i0, short i1) {
4402         putShort(o, offset + 0, pick(i0, i1));
4403         putShort(o, offset + 2, pick(i1, i0));
4404     }
4405     private void putIntParts(Object o, long offset, byte i0, byte i1, byte i2, byte i3) {
4406         putByte(o, offset + 0, pick(i0, i3));
4407         putByte(o, offset + 1, pick(i1, i2));
4408         putByte(o, offset + 2, pick(i2, i1));
4409         putByte(o, offset + 3, pick(i3, i0));
4410     }
4411     private void putShortParts(Object o, long offset, byte i0, byte i1) {
4412         putByte(o, offset + 0, pick(i0, i1));
4413         putByte(o, offset + 1, pick(i1, i0));
4414     }
4415 
4416     // Zero-extend an integer
4417     private static int toUnsignedInt(byte n)    { return n & 0xff; }
4418     private static int toUnsignedInt(short n)   { return n & 0xffff; }
4419     private static long toUnsignedLong(byte n)  { return n & 0xffl; }
4420     private static long toUnsignedLong(short n) { return n & 0xffffl; }
4421     private static long toUnsignedLong(int n)   { return n & 0xffffffffl; }
4422 
4423     // Maybe byte-reverse an integer
4424     private static char convEndian(boolean big, char n)   { return big == BIG_ENDIAN ? n : Character.reverseBytes(n); }
4425     private static short convEndian(boolean big, short n) { return big == BIG_ENDIAN ? n : Short.reverseBytes(n)    ; }
4426     private static int convEndian(boolean big, int n)     { return big == BIG_ENDIAN ? n : Integer.reverseBytes(n)  ; }
4427     private static long convEndian(boolean big, long n)   { return big == BIG_ENDIAN ? n : Long.reverseBytes(n)     ; }
4428 
4429 
4430 
4431     private native long allocateMemory0(long bytes);
4432     private native long reallocateMemory0(long address, long bytes);
4433     private native void freeMemory0(long address);
4434     @IntrinsicCandidate
4435     private native void setMemory0(Object o, long offset, long bytes, byte value);
4436     @IntrinsicCandidate
4437     private native void copyMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes);
4438     private native void copySwapMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes, long elemSize);
4439     private native long objectFieldOffset0(Field f); // throws IAE
4440     private native long knownObjectFieldOffset0(Class<?> c, String name); // error code: -1 not found, -2 static
4441     private native long staticFieldOffset0(Field f); // throws IAE
4442     private native Object staticFieldBase0(Field f); // throws IAE
4443     private native boolean shouldBeInitialized0(Class<?> c);
4444     private native void ensureClassInitialized0(Class<?> c);
4445     private native void notifyStrictStaticAccess0(Class<?> c, long staticFieldOffset, boolean writing);
4446     private native int arrayBaseOffset0(Class<?> arrayClass); // public version returns long to promote correct arithmetic
4447     private native int arrayInstanceBaseOffset0(Object[] array);
4448     private native int arrayIndexScale0(Class<?> arrayClass);
4449     private native int arrayInstanceIndexScale0(Object[] array);
4450     private native long getObjectSize0(Object o);
4451     private native int getLoadAverage0(double[] loadavg, int nelems);
4452 
4453 
4454     /**
4455      * Invokes the given direct byte buffer's cleaner, if any.
4456      *
4457      * @param directBuffer a direct byte buffer
4458      * @throws NullPointerException     if {@code directBuffer} is null
4459      * @throws IllegalArgumentException if {@code directBuffer} is non-direct,
4460      *                                  or is a {@link java.nio.Buffer#slice slice}, or is a
4461      *                                  {@link java.nio.Buffer#duplicate duplicate}
4462      */
4463     public void invokeCleaner(java.nio.ByteBuffer directBuffer) {
4464         if (!directBuffer.isDirect())
4465             throw new IllegalArgumentException("buffer is non-direct");
4466 
4467         DirectBuffer db = (DirectBuffer) directBuffer;
4468         if (db.attachment() != null)
4469             throw new IllegalArgumentException("duplicate or slice");
4470 
4471         Cleaner cleaner = db.cleaner();
4472         if (cleaner != null) {
4473             cleaner.clean();
4474         }
4475     }
4476 }