1 /* 2 * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/macroAssembler.hpp" 27 #include "ci/ciUtilities.inline.hpp" 28 #include "classfile/vmIntrinsics.hpp" 29 #include "compiler/compileBroker.hpp" 30 #include "compiler/compileLog.hpp" 31 #include "gc/shared/barrierSet.hpp" 32 #include "jfr/support/jfrIntrinsics.hpp" 33 #include "memory/resourceArea.hpp" 34 #include "oops/klass.inline.hpp" 35 #include "oops/objArrayKlass.hpp" 36 #include "opto/addnode.hpp" 37 #include "opto/arraycopynode.hpp" 38 #include "opto/c2compiler.hpp" 39 #include "opto/castnode.hpp" 40 #include "opto/cfgnode.hpp" 41 #include "opto/convertnode.hpp" 42 #include "opto/countbitsnode.hpp" 43 #include "opto/idealKit.hpp" 44 #include "opto/library_call.hpp" 45 #include "opto/mathexactnode.hpp" 46 #include "opto/mulnode.hpp" 47 #include "opto/narrowptrnode.hpp" 48 #include "opto/opaquenode.hpp" 49 #include "opto/parse.hpp" 50 #include "opto/runtime.hpp" 51 #include "opto/rootnode.hpp" 52 #include "opto/subnode.hpp" 53 #include "prims/unsafe.hpp" 54 #include "runtime/jniHandles.inline.hpp" 55 #include "runtime/objectMonitor.hpp" 56 #include "runtime/sharedRuntime.hpp" 57 #include "runtime/stubRoutines.hpp" 58 #include "utilities/macros.hpp" 59 #include "utilities/powerOfTwo.hpp" 60 61 //---------------------------make_vm_intrinsic---------------------------- 62 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) { 63 vmIntrinsicID id = m->intrinsic_id(); 64 assert(id != vmIntrinsics::_none, "must be a VM intrinsic"); 65 66 if (!m->is_loaded()) { 67 // Do not attempt to inline unloaded methods. 68 return NULL; 69 } 70 71 C2Compiler* compiler = (C2Compiler*)CompileBroker::compiler(CompLevel_full_optimization); 72 bool is_available = false; 73 74 { 75 // For calling is_intrinsic_supported and is_intrinsic_disabled_by_flag 76 // the compiler must transition to '_thread_in_vm' state because both 77 // methods access VM-internal data. 78 VM_ENTRY_MARK; 79 methodHandle mh(THREAD, m->get_Method()); 80 is_available = compiler != NULL && compiler->is_intrinsic_supported(mh, is_virtual) && 81 !C->directive()->is_intrinsic_disabled(mh) && 82 !vmIntrinsics::is_disabled_by_flags(mh); 83 84 } 85 86 if (is_available) { 87 assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility"); 88 assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?"); 89 return new LibraryIntrinsic(m, is_virtual, 90 vmIntrinsics::predicates_needed(id), 91 vmIntrinsics::does_virtual_dispatch(id), 92 id); 93 } else { 94 return NULL; 95 } 96 } 97 98 JVMState* LibraryIntrinsic::generate(JVMState* jvms) { 99 LibraryCallKit kit(jvms, this); 100 Compile* C = kit.C; 101 int nodes = C->unique(); 102 #ifndef PRODUCT 103 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 104 char buf[1000]; 105 const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf)); 106 tty->print_cr("Intrinsic %s", str); 107 } 108 #endif 109 ciMethod* callee = kit.callee(); 110 const int bci = kit.bci(); 111 #ifdef ASSERT 112 Node* ctrl = kit.control(); 113 #endif 114 // Try to inline the intrinsic. 115 if (callee->check_intrinsic_candidate() && 116 kit.try_to_inline(_last_predicate)) { 117 const char *inline_msg = is_virtual() ? "(intrinsic, virtual)" 118 : "(intrinsic)"; 119 CompileTask::print_inlining_ul(callee, jvms->depth() - 1, bci, inline_msg); 120 if (C->print_intrinsics() || C->print_inlining()) { 121 C->print_inlining(callee, jvms->depth() - 1, bci, inline_msg); 122 } 123 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked); 124 if (C->log()) { 125 C->log()->elem("intrinsic id='%s'%s nodes='%d'", 126 vmIntrinsics::name_at(intrinsic_id()), 127 (is_virtual() ? " virtual='1'" : ""), 128 C->unique() - nodes); 129 } 130 // Push the result from the inlined method onto the stack. 131 kit.push_result(); 132 C->print_inlining_update(this); 133 return kit.transfer_exceptions_into_jvms(); 134 } 135 136 // The intrinsic bailed out 137 assert(ctrl == kit.control(), "Control flow was added although the intrinsic bailed out"); 138 if (jvms->has_method()) { 139 // Not a root compile. 140 const char* msg; 141 if (callee->intrinsic_candidate()) { 142 msg = is_virtual() ? "failed to inline (intrinsic, virtual)" : "failed to inline (intrinsic)"; 143 } else { 144 msg = is_virtual() ? "failed to inline (intrinsic, virtual), method not annotated" 145 : "failed to inline (intrinsic), method not annotated"; 146 } 147 CompileTask::print_inlining_ul(callee, jvms->depth() - 1, bci, msg); 148 if (C->print_intrinsics() || C->print_inlining()) { 149 C->print_inlining(callee, jvms->depth() - 1, bci, msg); 150 } 151 } else { 152 // Root compile 153 ResourceMark rm; 154 stringStream msg_stream; 155 msg_stream.print("Did not generate intrinsic %s%s at bci:%d in", 156 vmIntrinsics::name_at(intrinsic_id()), 157 is_virtual() ? " (virtual)" : "", bci); 158 const char *msg = msg_stream.freeze(); 159 log_debug(jit, inlining)("%s", msg); 160 if (C->print_intrinsics() || C->print_inlining()) { 161 tty->print("%s", msg); 162 } 163 } 164 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed); 165 C->print_inlining_update(this); 166 167 return NULL; 168 } 169 170 Node* LibraryIntrinsic::generate_predicate(JVMState* jvms, int predicate) { 171 LibraryCallKit kit(jvms, this); 172 Compile* C = kit.C; 173 int nodes = C->unique(); 174 _last_predicate = predicate; 175 #ifndef PRODUCT 176 assert(is_predicated() && predicate < predicates_count(), "sanity"); 177 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 178 char buf[1000]; 179 const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf)); 180 tty->print_cr("Predicate for intrinsic %s", str); 181 } 182 #endif 183 ciMethod* callee = kit.callee(); 184 const int bci = kit.bci(); 185 186 Node* slow_ctl = kit.try_to_predicate(predicate); 187 if (!kit.failing()) { 188 const char *inline_msg = is_virtual() ? "(intrinsic, virtual, predicate)" 189 : "(intrinsic, predicate)"; 190 CompileTask::print_inlining_ul(callee, jvms->depth() - 1, bci, inline_msg); 191 if (C->print_intrinsics() || C->print_inlining()) { 192 C->print_inlining(callee, jvms->depth() - 1, bci, inline_msg); 193 } 194 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked); 195 if (C->log()) { 196 C->log()->elem("predicate_intrinsic id='%s'%s nodes='%d'", 197 vmIntrinsics::name_at(intrinsic_id()), 198 (is_virtual() ? " virtual='1'" : ""), 199 C->unique() - nodes); 200 } 201 return slow_ctl; // Could be NULL if the check folds. 202 } 203 204 // The intrinsic bailed out 205 if (jvms->has_method()) { 206 // Not a root compile. 207 const char* msg = "failed to generate predicate for intrinsic"; 208 CompileTask::print_inlining_ul(kit.callee(), jvms->depth() - 1, bci, msg); 209 if (C->print_intrinsics() || C->print_inlining()) { 210 C->print_inlining(kit.callee(), jvms->depth() - 1, bci, msg); 211 } 212 } else { 213 // Root compile 214 ResourceMark rm; 215 stringStream msg_stream; 216 msg_stream.print("Did not generate intrinsic %s%s at bci:%d in", 217 vmIntrinsics::name_at(intrinsic_id()), 218 is_virtual() ? " (virtual)" : "", bci); 219 const char *msg = msg_stream.freeze(); 220 log_debug(jit, inlining)("%s", msg); 221 if (C->print_intrinsics() || C->print_inlining()) { 222 C->print_inlining_stream()->print("%s", msg); 223 } 224 } 225 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed); 226 return NULL; 227 } 228 229 bool LibraryCallKit::try_to_inline(int predicate) { 230 // Handle symbolic names for otherwise undistinguished boolean switches: 231 const bool is_store = true; 232 const bool is_compress = true; 233 const bool is_static = true; 234 const bool is_volatile = true; 235 236 if (!jvms()->has_method()) { 237 // Root JVMState has a null method. 238 assert(map()->memory()->Opcode() == Op_Parm, ""); 239 // Insert the memory aliasing node 240 set_all_memory(reset_memory()); 241 } 242 assert(merged_memory(), ""); 243 244 switch (intrinsic_id()) { 245 case vmIntrinsics::_hashCode: return inline_native_hashcode(intrinsic()->is_virtual(), !is_static); 246 case vmIntrinsics::_identityHashCode: return inline_native_hashcode(/*!virtual*/ false, is_static); 247 case vmIntrinsics::_getClass: return inline_native_getClass(); 248 249 case vmIntrinsics::_ceil: 250 case vmIntrinsics::_floor: 251 case vmIntrinsics::_rint: 252 case vmIntrinsics::_dsin: 253 case vmIntrinsics::_dcos: 254 case vmIntrinsics::_dtan: 255 case vmIntrinsics::_dabs: 256 case vmIntrinsics::_fabs: 257 case vmIntrinsics::_iabs: 258 case vmIntrinsics::_labs: 259 case vmIntrinsics::_datan2: 260 case vmIntrinsics::_dsqrt: 261 case vmIntrinsics::_dsqrt_strict: 262 case vmIntrinsics::_dexp: 263 case vmIntrinsics::_dlog: 264 case vmIntrinsics::_dlog10: 265 case vmIntrinsics::_dpow: 266 case vmIntrinsics::_dcopySign: 267 case vmIntrinsics::_fcopySign: 268 case vmIntrinsics::_dsignum: 269 case vmIntrinsics::_roundF: 270 case vmIntrinsics::_roundD: 271 case vmIntrinsics::_fsignum: return inline_math_native(intrinsic_id()); 272 273 case vmIntrinsics::_notify: 274 case vmIntrinsics::_notifyAll: 275 return inline_notify(intrinsic_id()); 276 277 case vmIntrinsics::_addExactI: return inline_math_addExactI(false /* add */); 278 case vmIntrinsics::_addExactL: return inline_math_addExactL(false /* add */); 279 case vmIntrinsics::_decrementExactI: return inline_math_subtractExactI(true /* decrement */); 280 case vmIntrinsics::_decrementExactL: return inline_math_subtractExactL(true /* decrement */); 281 case vmIntrinsics::_incrementExactI: return inline_math_addExactI(true /* increment */); 282 case vmIntrinsics::_incrementExactL: return inline_math_addExactL(true /* increment */); 283 case vmIntrinsics::_multiplyExactI: return inline_math_multiplyExactI(); 284 case vmIntrinsics::_multiplyExactL: return inline_math_multiplyExactL(); 285 case vmIntrinsics::_multiplyHigh: return inline_math_multiplyHigh(); 286 case vmIntrinsics::_unsignedMultiplyHigh: return inline_math_unsignedMultiplyHigh(); 287 case vmIntrinsics::_negateExactI: return inline_math_negateExactI(); 288 case vmIntrinsics::_negateExactL: return inline_math_negateExactL(); 289 case vmIntrinsics::_subtractExactI: return inline_math_subtractExactI(false /* subtract */); 290 case vmIntrinsics::_subtractExactL: return inline_math_subtractExactL(false /* subtract */); 291 292 case vmIntrinsics::_arraycopy: return inline_arraycopy(); 293 294 case vmIntrinsics::_compareToL: return inline_string_compareTo(StrIntrinsicNode::LL); 295 case vmIntrinsics::_compareToU: return inline_string_compareTo(StrIntrinsicNode::UU); 296 case vmIntrinsics::_compareToLU: return inline_string_compareTo(StrIntrinsicNode::LU); 297 case vmIntrinsics::_compareToUL: return inline_string_compareTo(StrIntrinsicNode::UL); 298 299 case vmIntrinsics::_indexOfL: return inline_string_indexOf(StrIntrinsicNode::LL); 300 case vmIntrinsics::_indexOfU: return inline_string_indexOf(StrIntrinsicNode::UU); 301 case vmIntrinsics::_indexOfUL: return inline_string_indexOf(StrIntrinsicNode::UL); 302 case vmIntrinsics::_indexOfIL: return inline_string_indexOfI(StrIntrinsicNode::LL); 303 case vmIntrinsics::_indexOfIU: return inline_string_indexOfI(StrIntrinsicNode::UU); 304 case vmIntrinsics::_indexOfIUL: return inline_string_indexOfI(StrIntrinsicNode::UL); 305 case vmIntrinsics::_indexOfU_char: return inline_string_indexOfChar(StrIntrinsicNode::U); 306 case vmIntrinsics::_indexOfL_char: return inline_string_indexOfChar(StrIntrinsicNode::L); 307 308 case vmIntrinsics::_equalsL: return inline_string_equals(StrIntrinsicNode::LL); 309 case vmIntrinsics::_equalsU: return inline_string_equals(StrIntrinsicNode::UU); 310 311 case vmIntrinsics::_vectorizedHashCode: return inline_vectorizedHashCode(); 312 313 case vmIntrinsics::_toBytesStringU: return inline_string_toBytesU(); 314 case vmIntrinsics::_getCharsStringU: return inline_string_getCharsU(); 315 case vmIntrinsics::_getCharStringU: return inline_string_char_access(!is_store); 316 case vmIntrinsics::_putCharStringU: return inline_string_char_access( is_store); 317 318 case vmIntrinsics::_compressStringC: 319 case vmIntrinsics::_compressStringB: return inline_string_copy( is_compress); 320 case vmIntrinsics::_inflateStringC: 321 case vmIntrinsics::_inflateStringB: return inline_string_copy(!is_compress); 322 323 case vmIntrinsics::_getReference: return inline_unsafe_access(!is_store, T_OBJECT, Relaxed, false); 324 case vmIntrinsics::_getBoolean: return inline_unsafe_access(!is_store, T_BOOLEAN, Relaxed, false); 325 case vmIntrinsics::_getByte: return inline_unsafe_access(!is_store, T_BYTE, Relaxed, false); 326 case vmIntrinsics::_getShort: return inline_unsafe_access(!is_store, T_SHORT, Relaxed, false); 327 case vmIntrinsics::_getChar: return inline_unsafe_access(!is_store, T_CHAR, Relaxed, false); 328 case vmIntrinsics::_getInt: return inline_unsafe_access(!is_store, T_INT, Relaxed, false); 329 case vmIntrinsics::_getLong: return inline_unsafe_access(!is_store, T_LONG, Relaxed, false); 330 case vmIntrinsics::_getFloat: return inline_unsafe_access(!is_store, T_FLOAT, Relaxed, false); 331 case vmIntrinsics::_getDouble: return inline_unsafe_access(!is_store, T_DOUBLE, Relaxed, false); 332 333 case vmIntrinsics::_putReference: return inline_unsafe_access( is_store, T_OBJECT, Relaxed, false); 334 case vmIntrinsics::_putBoolean: return inline_unsafe_access( is_store, T_BOOLEAN, Relaxed, false); 335 case vmIntrinsics::_putByte: return inline_unsafe_access( is_store, T_BYTE, Relaxed, false); 336 case vmIntrinsics::_putShort: return inline_unsafe_access( is_store, T_SHORT, Relaxed, false); 337 case vmIntrinsics::_putChar: return inline_unsafe_access( is_store, T_CHAR, Relaxed, false); 338 case vmIntrinsics::_putInt: return inline_unsafe_access( is_store, T_INT, Relaxed, false); 339 case vmIntrinsics::_putLong: return inline_unsafe_access( is_store, T_LONG, Relaxed, false); 340 case vmIntrinsics::_putFloat: return inline_unsafe_access( is_store, T_FLOAT, Relaxed, false); 341 case vmIntrinsics::_putDouble: return inline_unsafe_access( is_store, T_DOUBLE, Relaxed, false); 342 343 case vmIntrinsics::_getReferenceVolatile: return inline_unsafe_access(!is_store, T_OBJECT, Volatile, false); 344 case vmIntrinsics::_getBooleanVolatile: return inline_unsafe_access(!is_store, T_BOOLEAN, Volatile, false); 345 case vmIntrinsics::_getByteVolatile: return inline_unsafe_access(!is_store, T_BYTE, Volatile, false); 346 case vmIntrinsics::_getShortVolatile: return inline_unsafe_access(!is_store, T_SHORT, Volatile, false); 347 case vmIntrinsics::_getCharVolatile: return inline_unsafe_access(!is_store, T_CHAR, Volatile, false); 348 case vmIntrinsics::_getIntVolatile: return inline_unsafe_access(!is_store, T_INT, Volatile, false); 349 case vmIntrinsics::_getLongVolatile: return inline_unsafe_access(!is_store, T_LONG, Volatile, false); 350 case vmIntrinsics::_getFloatVolatile: return inline_unsafe_access(!is_store, T_FLOAT, Volatile, false); 351 case vmIntrinsics::_getDoubleVolatile: return inline_unsafe_access(!is_store, T_DOUBLE, Volatile, false); 352 353 case vmIntrinsics::_putReferenceVolatile: return inline_unsafe_access( is_store, T_OBJECT, Volatile, false); 354 case vmIntrinsics::_putBooleanVolatile: return inline_unsafe_access( is_store, T_BOOLEAN, Volatile, false); 355 case vmIntrinsics::_putByteVolatile: return inline_unsafe_access( is_store, T_BYTE, Volatile, false); 356 case vmIntrinsics::_putShortVolatile: return inline_unsafe_access( is_store, T_SHORT, Volatile, false); 357 case vmIntrinsics::_putCharVolatile: return inline_unsafe_access( is_store, T_CHAR, Volatile, false); 358 case vmIntrinsics::_putIntVolatile: return inline_unsafe_access( is_store, T_INT, Volatile, false); 359 case vmIntrinsics::_putLongVolatile: return inline_unsafe_access( is_store, T_LONG, Volatile, false); 360 case vmIntrinsics::_putFloatVolatile: return inline_unsafe_access( is_store, T_FLOAT, Volatile, false); 361 case vmIntrinsics::_putDoubleVolatile: return inline_unsafe_access( is_store, T_DOUBLE, Volatile, false); 362 363 case vmIntrinsics::_getShortUnaligned: return inline_unsafe_access(!is_store, T_SHORT, Relaxed, true); 364 case vmIntrinsics::_getCharUnaligned: return inline_unsafe_access(!is_store, T_CHAR, Relaxed, true); 365 case vmIntrinsics::_getIntUnaligned: return inline_unsafe_access(!is_store, T_INT, Relaxed, true); 366 case vmIntrinsics::_getLongUnaligned: return inline_unsafe_access(!is_store, T_LONG, Relaxed, true); 367 368 case vmIntrinsics::_putShortUnaligned: return inline_unsafe_access( is_store, T_SHORT, Relaxed, true); 369 case vmIntrinsics::_putCharUnaligned: return inline_unsafe_access( is_store, T_CHAR, Relaxed, true); 370 case vmIntrinsics::_putIntUnaligned: return inline_unsafe_access( is_store, T_INT, Relaxed, true); 371 case vmIntrinsics::_putLongUnaligned: return inline_unsafe_access( is_store, T_LONG, Relaxed, true); 372 373 case vmIntrinsics::_getReferenceAcquire: return inline_unsafe_access(!is_store, T_OBJECT, Acquire, false); 374 case vmIntrinsics::_getBooleanAcquire: return inline_unsafe_access(!is_store, T_BOOLEAN, Acquire, false); 375 case vmIntrinsics::_getByteAcquire: return inline_unsafe_access(!is_store, T_BYTE, Acquire, false); 376 case vmIntrinsics::_getShortAcquire: return inline_unsafe_access(!is_store, T_SHORT, Acquire, false); 377 case vmIntrinsics::_getCharAcquire: return inline_unsafe_access(!is_store, T_CHAR, Acquire, false); 378 case vmIntrinsics::_getIntAcquire: return inline_unsafe_access(!is_store, T_INT, Acquire, false); 379 case vmIntrinsics::_getLongAcquire: return inline_unsafe_access(!is_store, T_LONG, Acquire, false); 380 case vmIntrinsics::_getFloatAcquire: return inline_unsafe_access(!is_store, T_FLOAT, Acquire, false); 381 case vmIntrinsics::_getDoubleAcquire: return inline_unsafe_access(!is_store, T_DOUBLE, Acquire, false); 382 383 case vmIntrinsics::_putReferenceRelease: return inline_unsafe_access( is_store, T_OBJECT, Release, false); 384 case vmIntrinsics::_putBooleanRelease: return inline_unsafe_access( is_store, T_BOOLEAN, Release, false); 385 case vmIntrinsics::_putByteRelease: return inline_unsafe_access( is_store, T_BYTE, Release, false); 386 case vmIntrinsics::_putShortRelease: return inline_unsafe_access( is_store, T_SHORT, Release, false); 387 case vmIntrinsics::_putCharRelease: return inline_unsafe_access( is_store, T_CHAR, Release, false); 388 case vmIntrinsics::_putIntRelease: return inline_unsafe_access( is_store, T_INT, Release, false); 389 case vmIntrinsics::_putLongRelease: return inline_unsafe_access( is_store, T_LONG, Release, false); 390 case vmIntrinsics::_putFloatRelease: return inline_unsafe_access( is_store, T_FLOAT, Release, false); 391 case vmIntrinsics::_putDoubleRelease: return inline_unsafe_access( is_store, T_DOUBLE, Release, false); 392 393 case vmIntrinsics::_getReferenceOpaque: return inline_unsafe_access(!is_store, T_OBJECT, Opaque, false); 394 case vmIntrinsics::_getBooleanOpaque: return inline_unsafe_access(!is_store, T_BOOLEAN, Opaque, false); 395 case vmIntrinsics::_getByteOpaque: return inline_unsafe_access(!is_store, T_BYTE, Opaque, false); 396 case vmIntrinsics::_getShortOpaque: return inline_unsafe_access(!is_store, T_SHORT, Opaque, false); 397 case vmIntrinsics::_getCharOpaque: return inline_unsafe_access(!is_store, T_CHAR, Opaque, false); 398 case vmIntrinsics::_getIntOpaque: return inline_unsafe_access(!is_store, T_INT, Opaque, false); 399 case vmIntrinsics::_getLongOpaque: return inline_unsafe_access(!is_store, T_LONG, Opaque, false); 400 case vmIntrinsics::_getFloatOpaque: return inline_unsafe_access(!is_store, T_FLOAT, Opaque, false); 401 case vmIntrinsics::_getDoubleOpaque: return inline_unsafe_access(!is_store, T_DOUBLE, Opaque, false); 402 403 case vmIntrinsics::_putReferenceOpaque: return inline_unsafe_access( is_store, T_OBJECT, Opaque, false); 404 case vmIntrinsics::_putBooleanOpaque: return inline_unsafe_access( is_store, T_BOOLEAN, Opaque, false); 405 case vmIntrinsics::_putByteOpaque: return inline_unsafe_access( is_store, T_BYTE, Opaque, false); 406 case vmIntrinsics::_putShortOpaque: return inline_unsafe_access( is_store, T_SHORT, Opaque, false); 407 case vmIntrinsics::_putCharOpaque: return inline_unsafe_access( is_store, T_CHAR, Opaque, false); 408 case vmIntrinsics::_putIntOpaque: return inline_unsafe_access( is_store, T_INT, Opaque, false); 409 case vmIntrinsics::_putLongOpaque: return inline_unsafe_access( is_store, T_LONG, Opaque, false); 410 case vmIntrinsics::_putFloatOpaque: return inline_unsafe_access( is_store, T_FLOAT, Opaque, false); 411 case vmIntrinsics::_putDoubleOpaque: return inline_unsafe_access( is_store, T_DOUBLE, Opaque, false); 412 413 case vmIntrinsics::_compareAndSetReference: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap, Volatile); 414 case vmIntrinsics::_compareAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap, Volatile); 415 case vmIntrinsics::_compareAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap, Volatile); 416 case vmIntrinsics::_compareAndSetInt: return inline_unsafe_load_store(T_INT, LS_cmp_swap, Volatile); 417 case vmIntrinsics::_compareAndSetLong: return inline_unsafe_load_store(T_LONG, LS_cmp_swap, Volatile); 418 419 case vmIntrinsics::_weakCompareAndSetReferencePlain: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Relaxed); 420 case vmIntrinsics::_weakCompareAndSetReferenceAcquire: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Acquire); 421 case vmIntrinsics::_weakCompareAndSetReferenceRelease: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Release); 422 case vmIntrinsics::_weakCompareAndSetReference: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Volatile); 423 case vmIntrinsics::_weakCompareAndSetBytePlain: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Relaxed); 424 case vmIntrinsics::_weakCompareAndSetByteAcquire: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Acquire); 425 case vmIntrinsics::_weakCompareAndSetByteRelease: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Release); 426 case vmIntrinsics::_weakCompareAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Volatile); 427 case vmIntrinsics::_weakCompareAndSetShortPlain: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Relaxed); 428 case vmIntrinsics::_weakCompareAndSetShortAcquire: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Acquire); 429 case vmIntrinsics::_weakCompareAndSetShortRelease: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Release); 430 case vmIntrinsics::_weakCompareAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Volatile); 431 case vmIntrinsics::_weakCompareAndSetIntPlain: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Relaxed); 432 case vmIntrinsics::_weakCompareAndSetIntAcquire: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Acquire); 433 case vmIntrinsics::_weakCompareAndSetIntRelease: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Release); 434 case vmIntrinsics::_weakCompareAndSetInt: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Volatile); 435 case vmIntrinsics::_weakCompareAndSetLongPlain: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Relaxed); 436 case vmIntrinsics::_weakCompareAndSetLongAcquire: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Acquire); 437 case vmIntrinsics::_weakCompareAndSetLongRelease: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Release); 438 case vmIntrinsics::_weakCompareAndSetLong: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Volatile); 439 440 case vmIntrinsics::_compareAndExchangeReference: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Volatile); 441 case vmIntrinsics::_compareAndExchangeReferenceAcquire: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Acquire); 442 case vmIntrinsics::_compareAndExchangeReferenceRelease: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Release); 443 case vmIntrinsics::_compareAndExchangeByte: return inline_unsafe_load_store(T_BYTE, LS_cmp_exchange, Volatile); 444 case vmIntrinsics::_compareAndExchangeByteAcquire: return inline_unsafe_load_store(T_BYTE, LS_cmp_exchange, Acquire); 445 case vmIntrinsics::_compareAndExchangeByteRelease: return inline_unsafe_load_store(T_BYTE, LS_cmp_exchange, Release); 446 case vmIntrinsics::_compareAndExchangeShort: return inline_unsafe_load_store(T_SHORT, LS_cmp_exchange, Volatile); 447 case vmIntrinsics::_compareAndExchangeShortAcquire: return inline_unsafe_load_store(T_SHORT, LS_cmp_exchange, Acquire); 448 case vmIntrinsics::_compareAndExchangeShortRelease: return inline_unsafe_load_store(T_SHORT, LS_cmp_exchange, Release); 449 case vmIntrinsics::_compareAndExchangeInt: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Volatile); 450 case vmIntrinsics::_compareAndExchangeIntAcquire: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Acquire); 451 case vmIntrinsics::_compareAndExchangeIntRelease: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Release); 452 case vmIntrinsics::_compareAndExchangeLong: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Volatile); 453 case vmIntrinsics::_compareAndExchangeLongAcquire: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Acquire); 454 case vmIntrinsics::_compareAndExchangeLongRelease: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Release); 455 456 case vmIntrinsics::_getAndAddByte: return inline_unsafe_load_store(T_BYTE, LS_get_add, Volatile); 457 case vmIntrinsics::_getAndAddShort: return inline_unsafe_load_store(T_SHORT, LS_get_add, Volatile); 458 case vmIntrinsics::_getAndAddInt: return inline_unsafe_load_store(T_INT, LS_get_add, Volatile); 459 case vmIntrinsics::_getAndAddLong: return inline_unsafe_load_store(T_LONG, LS_get_add, Volatile); 460 461 case vmIntrinsics::_getAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_get_set, Volatile); 462 case vmIntrinsics::_getAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_get_set, Volatile); 463 case vmIntrinsics::_getAndSetInt: return inline_unsafe_load_store(T_INT, LS_get_set, Volatile); 464 case vmIntrinsics::_getAndSetLong: return inline_unsafe_load_store(T_LONG, LS_get_set, Volatile); 465 case vmIntrinsics::_getAndSetReference: return inline_unsafe_load_store(T_OBJECT, LS_get_set, Volatile); 466 467 case vmIntrinsics::_loadFence: 468 case vmIntrinsics::_storeFence: 469 case vmIntrinsics::_storeStoreFence: 470 case vmIntrinsics::_fullFence: return inline_unsafe_fence(intrinsic_id()); 471 472 case vmIntrinsics::_onSpinWait: return inline_onspinwait(); 473 474 case vmIntrinsics::_currentCarrierThread: return inline_native_currentCarrierThread(); 475 case vmIntrinsics::_currentThread: return inline_native_currentThread(); 476 case vmIntrinsics::_setCurrentThread: return inline_native_setCurrentThread(); 477 478 case vmIntrinsics::_scopedValueCache: return inline_native_scopedValueCache(); 479 case vmIntrinsics::_setScopedValueCache: return inline_native_setScopedValueCache(); 480 481 #ifdef JFR_HAVE_INTRINSICS 482 case vmIntrinsics::_counterTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, JfrTime::time_function()), "counterTime"); 483 case vmIntrinsics::_getEventWriter: return inline_native_getEventWriter(); 484 #endif 485 case vmIntrinsics::_currentTimeMillis: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis"); 486 case vmIntrinsics::_nanoTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime"); 487 case vmIntrinsics::_writeback0: return inline_unsafe_writeback0(); 488 case vmIntrinsics::_writebackPreSync0: return inline_unsafe_writebackSync0(true); 489 case vmIntrinsics::_writebackPostSync0: return inline_unsafe_writebackSync0(false); 490 case vmIntrinsics::_allocateInstance: return inline_unsafe_allocate(); 491 case vmIntrinsics::_copyMemory: return inline_unsafe_copyMemory(); 492 case vmIntrinsics::_getLength: return inline_native_getLength(); 493 case vmIntrinsics::_copyOf: return inline_array_copyOf(false); 494 case vmIntrinsics::_copyOfRange: return inline_array_copyOf(true); 495 case vmIntrinsics::_equalsB: return inline_array_equals(StrIntrinsicNode::LL); 496 case vmIntrinsics::_equalsC: return inline_array_equals(StrIntrinsicNode::UU); 497 case vmIntrinsics::_Preconditions_checkIndex: return inline_preconditions_checkIndex(T_INT); 498 case vmIntrinsics::_Preconditions_checkLongIndex: return inline_preconditions_checkIndex(T_LONG); 499 case vmIntrinsics::_clone: return inline_native_clone(intrinsic()->is_virtual()); 500 501 case vmIntrinsics::_allocateUninitializedArray: return inline_unsafe_newArray(true); 502 case vmIntrinsics::_newArray: return inline_unsafe_newArray(false); 503 504 case vmIntrinsics::_isAssignableFrom: return inline_native_subtype_check(); 505 506 case vmIntrinsics::_isInstance: 507 case vmIntrinsics::_getModifiers: 508 case vmIntrinsics::_isInterface: 509 case vmIntrinsics::_isArray: 510 case vmIntrinsics::_isPrimitive: 511 case vmIntrinsics::_isHidden: 512 case vmIntrinsics::_getSuperclass: 513 case vmIntrinsics::_getClassAccessFlags: return inline_native_Class_query(intrinsic_id()); 514 515 case vmIntrinsics::_floatToRawIntBits: 516 case vmIntrinsics::_floatToIntBits: 517 case vmIntrinsics::_intBitsToFloat: 518 case vmIntrinsics::_doubleToRawLongBits: 519 case vmIntrinsics::_doubleToLongBits: 520 case vmIntrinsics::_longBitsToDouble: 521 case vmIntrinsics::_floatToFloat16: 522 case vmIntrinsics::_float16ToFloat: return inline_fp_conversions(intrinsic_id()); 523 524 case vmIntrinsics::_floatIsFinite: 525 case vmIntrinsics::_floatIsInfinite: 526 case vmIntrinsics::_doubleIsFinite: 527 case vmIntrinsics::_doubleIsInfinite: return inline_fp_range_check(intrinsic_id()); 528 529 case vmIntrinsics::_numberOfLeadingZeros_i: 530 case vmIntrinsics::_numberOfLeadingZeros_l: 531 case vmIntrinsics::_numberOfTrailingZeros_i: 532 case vmIntrinsics::_numberOfTrailingZeros_l: 533 case vmIntrinsics::_bitCount_i: 534 case vmIntrinsics::_bitCount_l: 535 case vmIntrinsics::_reverse_i: 536 case vmIntrinsics::_reverse_l: 537 case vmIntrinsics::_reverseBytes_i: 538 case vmIntrinsics::_reverseBytes_l: 539 case vmIntrinsics::_reverseBytes_s: 540 case vmIntrinsics::_reverseBytes_c: return inline_number_methods(intrinsic_id()); 541 542 case vmIntrinsics::_compress_i: 543 case vmIntrinsics::_compress_l: 544 case vmIntrinsics::_expand_i: 545 case vmIntrinsics::_expand_l: return inline_bitshuffle_methods(intrinsic_id()); 546 547 case vmIntrinsics::_compareUnsigned_i: 548 case vmIntrinsics::_compareUnsigned_l: return inline_compare_unsigned(intrinsic_id()); 549 550 case vmIntrinsics::_divideUnsigned_i: 551 case vmIntrinsics::_divideUnsigned_l: 552 case vmIntrinsics::_remainderUnsigned_i: 553 case vmIntrinsics::_remainderUnsigned_l: return inline_divmod_methods(intrinsic_id()); 554 555 case vmIntrinsics::_getCallerClass: return inline_native_Reflection_getCallerClass(); 556 557 case vmIntrinsics::_Reference_get: return inline_reference_get(); 558 case vmIntrinsics::_Reference_refersTo0: return inline_reference_refersTo0(false); 559 case vmIntrinsics::_PhantomReference_refersTo0: return inline_reference_refersTo0(true); 560 561 case vmIntrinsics::_Class_cast: return inline_Class_cast(); 562 563 case vmIntrinsics::_aescrypt_encryptBlock: 564 case vmIntrinsics::_aescrypt_decryptBlock: return inline_aescrypt_Block(intrinsic_id()); 565 566 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt: 567 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt: 568 return inline_cipherBlockChaining_AESCrypt(intrinsic_id()); 569 570 case vmIntrinsics::_electronicCodeBook_encryptAESCrypt: 571 case vmIntrinsics::_electronicCodeBook_decryptAESCrypt: 572 return inline_electronicCodeBook_AESCrypt(intrinsic_id()); 573 574 case vmIntrinsics::_counterMode_AESCrypt: 575 return inline_counterMode_AESCrypt(intrinsic_id()); 576 577 case vmIntrinsics::_galoisCounterMode_AESCrypt: 578 return inline_galoisCounterMode_AESCrypt(); 579 580 case vmIntrinsics::_md5_implCompress: 581 case vmIntrinsics::_sha_implCompress: 582 case vmIntrinsics::_sha2_implCompress: 583 case vmIntrinsics::_sha5_implCompress: 584 case vmIntrinsics::_sha3_implCompress: 585 return inline_digestBase_implCompress(intrinsic_id()); 586 587 case vmIntrinsics::_digestBase_implCompressMB: 588 return inline_digestBase_implCompressMB(predicate); 589 590 case vmIntrinsics::_multiplyToLen: 591 return inline_multiplyToLen(); 592 593 case vmIntrinsics::_squareToLen: 594 return inline_squareToLen(); 595 596 case vmIntrinsics::_mulAdd: 597 return inline_mulAdd(); 598 599 case vmIntrinsics::_montgomeryMultiply: 600 return inline_montgomeryMultiply(); 601 case vmIntrinsics::_montgomerySquare: 602 return inline_montgomerySquare(); 603 604 case vmIntrinsics::_bigIntegerRightShiftWorker: 605 return inline_bigIntegerShift(true); 606 case vmIntrinsics::_bigIntegerLeftShiftWorker: 607 return inline_bigIntegerShift(false); 608 609 case vmIntrinsics::_vectorizedMismatch: 610 return inline_vectorizedMismatch(); 611 612 case vmIntrinsics::_ghash_processBlocks: 613 return inline_ghash_processBlocks(); 614 case vmIntrinsics::_chacha20Block: 615 return inline_chacha20Block(); 616 case vmIntrinsics::_base64_encodeBlock: 617 return inline_base64_encodeBlock(); 618 case vmIntrinsics::_base64_decodeBlock: 619 return inline_base64_decodeBlock(); 620 case vmIntrinsics::_poly1305_processBlocks: 621 return inline_poly1305_processBlocks(); 622 623 case vmIntrinsics::_encodeISOArray: 624 case vmIntrinsics::_encodeByteISOArray: 625 return inline_encodeISOArray(false); 626 case vmIntrinsics::_encodeAsciiArray: 627 return inline_encodeISOArray(true); 628 629 case vmIntrinsics::_updateCRC32: 630 return inline_updateCRC32(); 631 case vmIntrinsics::_updateBytesCRC32: 632 return inline_updateBytesCRC32(); 633 case vmIntrinsics::_updateByteBufferCRC32: 634 return inline_updateByteBufferCRC32(); 635 636 case vmIntrinsics::_updateBytesCRC32C: 637 return inline_updateBytesCRC32C(); 638 case vmIntrinsics::_updateDirectByteBufferCRC32C: 639 return inline_updateDirectByteBufferCRC32C(); 640 641 case vmIntrinsics::_updateBytesAdler32: 642 return inline_updateBytesAdler32(); 643 case vmIntrinsics::_updateByteBufferAdler32: 644 return inline_updateByteBufferAdler32(); 645 646 case vmIntrinsics::_profileBoolean: 647 return inline_profileBoolean(); 648 case vmIntrinsics::_isCompileConstant: 649 return inline_isCompileConstant(); 650 651 case vmIntrinsics::_countPositives: 652 return inline_countPositives(); 653 654 case vmIntrinsics::_fmaD: 655 case vmIntrinsics::_fmaF: 656 return inline_fma(intrinsic_id()); 657 658 case vmIntrinsics::_isDigit: 659 case vmIntrinsics::_isLowerCase: 660 case vmIntrinsics::_isUpperCase: 661 case vmIntrinsics::_isWhitespace: 662 return inline_character_compare(intrinsic_id()); 663 664 case vmIntrinsics::_min: 665 case vmIntrinsics::_max: 666 case vmIntrinsics::_min_strict: 667 case vmIntrinsics::_max_strict: 668 return inline_min_max(intrinsic_id()); 669 670 case vmIntrinsics::_maxF: 671 case vmIntrinsics::_minF: 672 case vmIntrinsics::_maxD: 673 case vmIntrinsics::_minD: 674 case vmIntrinsics::_maxF_strict: 675 case vmIntrinsics::_minF_strict: 676 case vmIntrinsics::_maxD_strict: 677 case vmIntrinsics::_minD_strict: 678 return inline_fp_min_max(intrinsic_id()); 679 680 case vmIntrinsics::_VectorUnaryOp: 681 return inline_vector_nary_operation(1); 682 case vmIntrinsics::_VectorBinaryOp: 683 return inline_vector_nary_operation(2); 684 case vmIntrinsics::_VectorTernaryOp: 685 return inline_vector_nary_operation(3); 686 case vmIntrinsics::_VectorFromBitsCoerced: 687 return inline_vector_frombits_coerced(); 688 case vmIntrinsics::_VectorShuffleIota: 689 return inline_vector_shuffle_iota(); 690 case vmIntrinsics::_VectorMaskOp: 691 return inline_vector_mask_operation(); 692 case vmIntrinsics::_VectorShuffleToVector: 693 return inline_vector_shuffle_to_vector(); 694 case vmIntrinsics::_VectorLoadOp: 695 return inline_vector_mem_operation(/*is_store=*/false); 696 case vmIntrinsics::_VectorLoadMaskedOp: 697 return inline_vector_mem_masked_operation(/*is_store*/false); 698 case vmIntrinsics::_VectorStoreOp: 699 return inline_vector_mem_operation(/*is_store=*/true); 700 case vmIntrinsics::_VectorStoreMaskedOp: 701 return inline_vector_mem_masked_operation(/*is_store=*/true); 702 case vmIntrinsics::_VectorGatherOp: 703 return inline_vector_gather_scatter(/*is_scatter*/ false); 704 case vmIntrinsics::_VectorScatterOp: 705 return inline_vector_gather_scatter(/*is_scatter*/ true); 706 case vmIntrinsics::_VectorReductionCoerced: 707 return inline_vector_reduction(); 708 case vmIntrinsics::_VectorTest: 709 return inline_vector_test(); 710 case vmIntrinsics::_VectorBlend: 711 return inline_vector_blend(); 712 case vmIntrinsics::_VectorRearrange: 713 return inline_vector_rearrange(); 714 case vmIntrinsics::_VectorCompare: 715 return inline_vector_compare(); 716 case vmIntrinsics::_VectorBroadcastInt: 717 return inline_vector_broadcast_int(); 718 case vmIntrinsics::_VectorConvert: 719 return inline_vector_convert(); 720 case vmIntrinsics::_VectorInsert: 721 return inline_vector_insert(); 722 case vmIntrinsics::_VectorExtract: 723 return inline_vector_extract(); 724 case vmIntrinsics::_VectorCompressExpand: 725 return inline_vector_compress_expand(); 726 case vmIntrinsics::_IndexVector: 727 return inline_index_vector(); 728 729 case vmIntrinsics::_getObjectSize: 730 return inline_getObjectSize(); 731 732 case vmIntrinsics::_blackhole: 733 return inline_blackhole(); 734 735 case vmIntrinsics::_shipilev_magic_timestamp: 736 return inline_timestamp(false); 737 case vmIntrinsics::_shipilev_magic_timestamp_serial: 738 return inline_timestamp(true); 739 case vmIntrinsics::_shipilev_magic_sizeOf: 740 return inline_sizeOf(); 741 case vmIntrinsics::_shipilev_magic_addressOf: 742 return inline_addressOf(); 743 744 default: 745 // If you get here, it may be that someone has added a new intrinsic 746 // to the list in vmIntrinsics.hpp without implementing it here. 747 #ifndef PRODUCT 748 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) { 749 tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)", 750 vmIntrinsics::name_at(intrinsic_id()), vmIntrinsics::as_int(intrinsic_id())); 751 } 752 #endif 753 return false; 754 } 755 } 756 757 Node* LibraryCallKit::try_to_predicate(int predicate) { 758 if (!jvms()->has_method()) { 759 // Root JVMState has a null method. 760 assert(map()->memory()->Opcode() == Op_Parm, ""); 761 // Insert the memory aliasing node 762 set_all_memory(reset_memory()); 763 } 764 assert(merged_memory(), ""); 765 766 switch (intrinsic_id()) { 767 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt: 768 return inline_cipherBlockChaining_AESCrypt_predicate(false); 769 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt: 770 return inline_cipherBlockChaining_AESCrypt_predicate(true); 771 case vmIntrinsics::_electronicCodeBook_encryptAESCrypt: 772 return inline_electronicCodeBook_AESCrypt_predicate(false); 773 case vmIntrinsics::_electronicCodeBook_decryptAESCrypt: 774 return inline_electronicCodeBook_AESCrypt_predicate(true); 775 case vmIntrinsics::_counterMode_AESCrypt: 776 return inline_counterMode_AESCrypt_predicate(); 777 case vmIntrinsics::_digestBase_implCompressMB: 778 return inline_digestBase_implCompressMB_predicate(predicate); 779 case vmIntrinsics::_galoisCounterMode_AESCrypt: 780 return inline_galoisCounterMode_AESCrypt_predicate(); 781 782 default: 783 // If you get here, it may be that someone has added a new intrinsic 784 // to the list in vmIntrinsics.hpp without implementing it here. 785 #ifndef PRODUCT 786 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) { 787 tty->print_cr("*** Warning: Unimplemented predicate for intrinsic %s(%d)", 788 vmIntrinsics::name_at(intrinsic_id()), vmIntrinsics::as_int(intrinsic_id())); 789 } 790 #endif 791 Node* slow_ctl = control(); 792 set_control(top()); // No fast path intrinsic 793 return slow_ctl; 794 } 795 } 796 797 //------------------------------set_result------------------------------- 798 // Helper function for finishing intrinsics. 799 void LibraryCallKit::set_result(RegionNode* region, PhiNode* value) { 800 record_for_igvn(region); 801 set_control(_gvn.transform(region)); 802 set_result( _gvn.transform(value)); 803 assert(value->type()->basic_type() == result()->bottom_type()->basic_type(), "sanity"); 804 } 805 806 //------------------------------generate_guard--------------------------- 807 // Helper function for generating guarded fast-slow graph structures. 808 // The given 'test', if true, guards a slow path. If the test fails 809 // then a fast path can be taken. (We generally hope it fails.) 810 // In all cases, GraphKit::control() is updated to the fast path. 811 // The returned value represents the control for the slow path. 812 // The return value is never 'top'; it is either a valid control 813 // or NULL if it is obvious that the slow path can never be taken. 814 // Also, if region and the slow control are not NULL, the slow edge 815 // is appended to the region. 816 Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) { 817 if (stopped()) { 818 // Already short circuited. 819 return NULL; 820 } 821 822 // Build an if node and its projections. 823 // If test is true we take the slow path, which we assume is uncommon. 824 if (_gvn.type(test) == TypeInt::ZERO) { 825 // The slow branch is never taken. No need to build this guard. 826 return NULL; 827 } 828 829 IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN); 830 831 Node* if_slow = _gvn.transform(new IfTrueNode(iff)); 832 if (if_slow == top()) { 833 // The slow branch is never taken. No need to build this guard. 834 return NULL; 835 } 836 837 if (region != NULL) 838 region->add_req(if_slow); 839 840 Node* if_fast = _gvn.transform(new IfFalseNode(iff)); 841 set_control(if_fast); 842 843 return if_slow; 844 } 845 846 inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) { 847 return generate_guard(test, region, PROB_UNLIKELY_MAG(3)); 848 } 849 inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) { 850 return generate_guard(test, region, PROB_FAIR); 851 } 852 853 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region, 854 Node* *pos_index) { 855 if (stopped()) 856 return NULL; // already stopped 857 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint] 858 return NULL; // index is already adequately typed 859 Node* cmp_lt = _gvn.transform(new CmpINode(index, intcon(0))); 860 Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt)); 861 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN); 862 if (is_neg != NULL && pos_index != NULL) { 863 // Emulate effect of Parse::adjust_map_after_if. 864 Node* ccast = new CastIINode(index, TypeInt::POS); 865 ccast->set_req(0, control()); 866 (*pos_index) = _gvn.transform(ccast); 867 } 868 return is_neg; 869 } 870 871 // Make sure that 'position' is a valid limit index, in [0..length]. 872 // There are two equivalent plans for checking this: 873 // A. (offset + copyLength) unsigned<= arrayLength 874 // B. offset <= (arrayLength - copyLength) 875 // We require that all of the values above, except for the sum and 876 // difference, are already known to be non-negative. 877 // Plan A is robust in the face of overflow, if offset and copyLength 878 // are both hugely positive. 879 // 880 // Plan B is less direct and intuitive, but it does not overflow at 881 // all, since the difference of two non-negatives is always 882 // representable. Whenever Java methods must perform the equivalent 883 // check they generally use Plan B instead of Plan A. 884 // For the moment we use Plan A. 885 inline Node* LibraryCallKit::generate_limit_guard(Node* offset, 886 Node* subseq_length, 887 Node* array_length, 888 RegionNode* region) { 889 if (stopped()) 890 return NULL; // already stopped 891 bool zero_offset = _gvn.type(offset) == TypeInt::ZERO; 892 if (zero_offset && subseq_length->eqv_uncast(array_length)) 893 return NULL; // common case of whole-array copy 894 Node* last = subseq_length; 895 if (!zero_offset) // last += offset 896 last = _gvn.transform(new AddINode(last, offset)); 897 Node* cmp_lt = _gvn.transform(new CmpUNode(array_length, last)); 898 Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt)); 899 Node* is_over = generate_guard(bol_lt, region, PROB_MIN); 900 return is_over; 901 } 902 903 // Emit range checks for the given String.value byte array 904 void LibraryCallKit::generate_string_range_check(Node* array, Node* offset, Node* count, bool char_count) { 905 if (stopped()) { 906 return; // already stopped 907 } 908 RegionNode* bailout = new RegionNode(1); 909 record_for_igvn(bailout); 910 if (char_count) { 911 // Convert char count to byte count 912 count = _gvn.transform(new LShiftINode(count, intcon(1))); 913 } 914 915 // Offset and count must not be negative 916 generate_negative_guard(offset, bailout); 917 generate_negative_guard(count, bailout); 918 // Offset + count must not exceed length of array 919 generate_limit_guard(offset, count, load_array_length(array), bailout); 920 921 if (bailout->req() > 1) { 922 PreserveJVMState pjvms(this); 923 set_control(_gvn.transform(bailout)); 924 uncommon_trap(Deoptimization::Reason_intrinsic, 925 Deoptimization::Action_maybe_recompile); 926 } 927 } 928 929 Node* LibraryCallKit::current_thread_helper(Node*& tls_output, ByteSize handle_offset, 930 bool is_immutable) { 931 ciKlass* thread_klass = env()->Thread_klass(); 932 const Type* thread_type 933 = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull); 934 935 Node* thread = _gvn.transform(new ThreadLocalNode()); 936 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(handle_offset)); 937 tls_output = thread; 938 939 Node* thread_obj_handle 940 = (is_immutable 941 ? LoadNode::make(_gvn, NULL, immutable_memory(), p, p->bottom_type()->is_ptr(), 942 TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered) 943 : make_load(NULL, p, p->bottom_type()->is_ptr(), T_ADDRESS, MemNode::unordered)); 944 thread_obj_handle = _gvn.transform(thread_obj_handle); 945 946 DecoratorSet decorators = IN_NATIVE; 947 if (is_immutable) { 948 decorators |= C2_IMMUTABLE_MEMORY; 949 } 950 return access_load(thread_obj_handle, thread_type, T_OBJECT, decorators); 951 } 952 953 //--------------------------generate_current_thread-------------------- 954 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) { 955 return current_thread_helper(tls_output, JavaThread::threadObj_offset(), 956 /*is_immutable*/false); 957 } 958 959 //--------------------------generate_virtual_thread-------------------- 960 Node* LibraryCallKit::generate_virtual_thread(Node* tls_output) { 961 return current_thread_helper(tls_output, JavaThread::vthread_offset(), 962 !C->method()->changes_current_thread()); 963 } 964 965 //------------------------------make_string_method_node------------------------ 966 // Helper method for String intrinsic functions. This version is called with 967 // str1 and str2 pointing to byte[] nodes containing Latin1 or UTF16 encoded 968 // characters (depending on 'is_byte'). cnt1 and cnt2 are pointing to Int nodes 969 // containing the lengths of str1 and str2. 970 Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae) { 971 Node* result = NULL; 972 switch (opcode) { 973 case Op_StrIndexOf: 974 result = new StrIndexOfNode(control(), memory(TypeAryPtr::BYTES), 975 str1_start, cnt1, str2_start, cnt2, ae); 976 break; 977 case Op_StrComp: 978 result = new StrCompNode(control(), memory(TypeAryPtr::BYTES), 979 str1_start, cnt1, str2_start, cnt2, ae); 980 break; 981 case Op_StrEquals: 982 // We already know that cnt1 == cnt2 here (checked in 'inline_string_equals'). 983 // Use the constant length if there is one because optimized match rule may exist. 984 result = new StrEqualsNode(control(), memory(TypeAryPtr::BYTES), 985 str1_start, str2_start, cnt2->is_Con() ? cnt2 : cnt1, ae); 986 break; 987 default: 988 ShouldNotReachHere(); 989 return NULL; 990 } 991 992 // All these intrinsics have checks. 993 C->set_has_split_ifs(true); // Has chance for split-if optimization 994 clear_upper_avx(); 995 996 return _gvn.transform(result); 997 } 998 999 //------------------------------inline_string_compareTo------------------------ 1000 bool LibraryCallKit::inline_string_compareTo(StrIntrinsicNode::ArgEnc ae) { 1001 Node* arg1 = argument(0); 1002 Node* arg2 = argument(1); 1003 1004 arg1 = must_be_not_null(arg1, true); 1005 arg2 = must_be_not_null(arg2, true); 1006 1007 // Get start addr and length of first argument 1008 Node* arg1_start = array_element_address(arg1, intcon(0), T_BYTE); 1009 Node* arg1_cnt = load_array_length(arg1); 1010 1011 // Get start addr and length of second argument 1012 Node* arg2_start = array_element_address(arg2, intcon(0), T_BYTE); 1013 Node* arg2_cnt = load_array_length(arg2); 1014 1015 Node* result = make_string_method_node(Op_StrComp, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae); 1016 set_result(result); 1017 return true; 1018 } 1019 1020 //------------------------------inline_string_equals------------------------ 1021 bool LibraryCallKit::inline_string_equals(StrIntrinsicNode::ArgEnc ae) { 1022 Node* arg1 = argument(0); 1023 Node* arg2 = argument(1); 1024 1025 // paths (plus control) merge 1026 RegionNode* region = new RegionNode(3); 1027 Node* phi = new PhiNode(region, TypeInt::BOOL); 1028 1029 if (!stopped()) { 1030 1031 arg1 = must_be_not_null(arg1, true); 1032 arg2 = must_be_not_null(arg2, true); 1033 1034 // Get start addr and length of first argument 1035 Node* arg1_start = array_element_address(arg1, intcon(0), T_BYTE); 1036 Node* arg1_cnt = load_array_length(arg1); 1037 1038 // Get start addr and length of second argument 1039 Node* arg2_start = array_element_address(arg2, intcon(0), T_BYTE); 1040 Node* arg2_cnt = load_array_length(arg2); 1041 1042 // Check for arg1_cnt != arg2_cnt 1043 Node* cmp = _gvn.transform(new CmpINode(arg1_cnt, arg2_cnt)); 1044 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne)); 1045 Node* if_ne = generate_slow_guard(bol, NULL); 1046 if (if_ne != NULL) { 1047 phi->init_req(2, intcon(0)); 1048 region->init_req(2, if_ne); 1049 } 1050 1051 // Check for count == 0 is done by assembler code for StrEquals. 1052 1053 if (!stopped()) { 1054 Node* equals = make_string_method_node(Op_StrEquals, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae); 1055 phi->init_req(1, equals); 1056 region->init_req(1, control()); 1057 } 1058 } 1059 1060 // post merge 1061 set_control(_gvn.transform(region)); 1062 record_for_igvn(region); 1063 1064 set_result(_gvn.transform(phi)); 1065 return true; 1066 } 1067 1068 //------------------------------inline_array_equals---------------------------- 1069 bool LibraryCallKit::inline_array_equals(StrIntrinsicNode::ArgEnc ae) { 1070 assert(ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::LL, "unsupported array types"); 1071 Node* arg1 = argument(0); 1072 Node* arg2 = argument(1); 1073 1074 const TypeAryPtr* mtype = (ae == StrIntrinsicNode::UU) ? TypeAryPtr::CHARS : TypeAryPtr::BYTES; 1075 set_result(_gvn.transform(new AryEqNode(control(), memory(mtype), arg1, arg2, ae))); 1076 clear_upper_avx(); 1077 1078 return true; 1079 } 1080 1081 1082 //------------------------------inline_countPositives------------------------------ 1083 bool LibraryCallKit::inline_countPositives() { 1084 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 1085 return false; 1086 } 1087 1088 assert(callee()->signature()->size() == 3, "countPositives has 3 parameters"); 1089 // no receiver since it is static method 1090 Node* ba = argument(0); 1091 Node* offset = argument(1); 1092 Node* len = argument(2); 1093 1094 ba = must_be_not_null(ba, true); 1095 1096 // Range checks 1097 generate_string_range_check(ba, offset, len, false); 1098 if (stopped()) { 1099 return true; 1100 } 1101 Node* ba_start = array_element_address(ba, offset, T_BYTE); 1102 Node* result = new CountPositivesNode(control(), memory(TypeAryPtr::BYTES), ba_start, len); 1103 set_result(_gvn.transform(result)); 1104 return true; 1105 } 1106 1107 bool LibraryCallKit::inline_preconditions_checkIndex(BasicType bt) { 1108 Node* index = argument(0); 1109 Node* length = bt == T_INT ? argument(1) : argument(2); 1110 if (too_many_traps(Deoptimization::Reason_intrinsic) || too_many_traps(Deoptimization::Reason_range_check)) { 1111 return false; 1112 } 1113 1114 // check that length is positive 1115 Node* len_pos_cmp = _gvn.transform(CmpNode::make(length, integercon(0, bt), bt)); 1116 Node* len_pos_bol = _gvn.transform(new BoolNode(len_pos_cmp, BoolTest::ge)); 1117 1118 { 1119 BuildCutout unless(this, len_pos_bol, PROB_MAX); 1120 uncommon_trap(Deoptimization::Reason_intrinsic, 1121 Deoptimization::Action_make_not_entrant); 1122 } 1123 1124 if (stopped()) { 1125 // Length is known to be always negative during compilation and the IR graph so far constructed is good so return success 1126 return true; 1127 } 1128 1129 // length is now known positive, add a cast node to make this explicit 1130 jlong upper_bound = _gvn.type(length)->is_integer(bt)->hi_as_long(); 1131 Node* casted_length = ConstraintCastNode::make(control(), length, TypeInteger::make(0, upper_bound, Type::WidenMax, bt), ConstraintCastNode::RegularDependency, bt); 1132 casted_length = _gvn.transform(casted_length); 1133 replace_in_map(length, casted_length); 1134 length = casted_length; 1135 1136 // Use an unsigned comparison for the range check itself 1137 Node* rc_cmp = _gvn.transform(CmpNode::make(index, length, bt, true)); 1138 BoolTest::mask btest = BoolTest::lt; 1139 Node* rc_bool = _gvn.transform(new BoolNode(rc_cmp, btest)); 1140 RangeCheckNode* rc = new RangeCheckNode(control(), rc_bool, PROB_MAX, COUNT_UNKNOWN); 1141 _gvn.set_type(rc, rc->Value(&_gvn)); 1142 if (!rc_bool->is_Con()) { 1143 record_for_igvn(rc); 1144 } 1145 set_control(_gvn.transform(new IfTrueNode(rc))); 1146 { 1147 PreserveJVMState pjvms(this); 1148 set_control(_gvn.transform(new IfFalseNode(rc))); 1149 uncommon_trap(Deoptimization::Reason_range_check, 1150 Deoptimization::Action_make_not_entrant); 1151 } 1152 1153 if (stopped()) { 1154 // Range check is known to always fail during compilation and the IR graph so far constructed is good so return success 1155 return true; 1156 } 1157 1158 // index is now known to be >= 0 and < length, cast it 1159 Node* result = ConstraintCastNode::make(control(), index, TypeInteger::make(0, upper_bound, Type::WidenMax, bt), ConstraintCastNode::RegularDependency, bt); 1160 result = _gvn.transform(result); 1161 set_result(result); 1162 replace_in_map(index, result); 1163 return true; 1164 } 1165 1166 //------------------------------inline_string_indexOf------------------------ 1167 bool LibraryCallKit::inline_string_indexOf(StrIntrinsicNode::ArgEnc ae) { 1168 if (!Matcher::match_rule_supported(Op_StrIndexOf)) { 1169 return false; 1170 } 1171 Node* src = argument(0); 1172 Node* tgt = argument(1); 1173 1174 // Make the merge point 1175 RegionNode* result_rgn = new RegionNode(4); 1176 Node* result_phi = new PhiNode(result_rgn, TypeInt::INT); 1177 1178 src = must_be_not_null(src, true); 1179 tgt = must_be_not_null(tgt, true); 1180 1181 // Get start addr and length of source string 1182 Node* src_start = array_element_address(src, intcon(0), T_BYTE); 1183 Node* src_count = load_array_length(src); 1184 1185 // Get start addr and length of substring 1186 Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE); 1187 Node* tgt_count = load_array_length(tgt); 1188 1189 if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) { 1190 // Divide src size by 2 if String is UTF16 encoded 1191 src_count = _gvn.transform(new RShiftINode(src_count, intcon(1))); 1192 } 1193 if (ae == StrIntrinsicNode::UU) { 1194 // Divide substring size by 2 if String is UTF16 encoded 1195 tgt_count = _gvn.transform(new RShiftINode(tgt_count, intcon(1))); 1196 } 1197 1198 Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, result_rgn, result_phi, ae); 1199 if (result != NULL) { 1200 result_phi->init_req(3, result); 1201 result_rgn->init_req(3, control()); 1202 } 1203 set_control(_gvn.transform(result_rgn)); 1204 record_for_igvn(result_rgn); 1205 set_result(_gvn.transform(result_phi)); 1206 1207 return true; 1208 } 1209 1210 //-----------------------------inline_string_indexOf----------------------- 1211 bool LibraryCallKit::inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae) { 1212 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 1213 return false; 1214 } 1215 if (!Matcher::match_rule_supported(Op_StrIndexOf)) { 1216 return false; 1217 } 1218 assert(callee()->signature()->size() == 5, "String.indexOf() has 5 arguments"); 1219 Node* src = argument(0); // byte[] 1220 Node* src_count = argument(1); // char count 1221 Node* tgt = argument(2); // byte[] 1222 Node* tgt_count = argument(3); // char count 1223 Node* from_index = argument(4); // char index 1224 1225 src = must_be_not_null(src, true); 1226 tgt = must_be_not_null(tgt, true); 1227 1228 // Multiply byte array index by 2 if String is UTF16 encoded 1229 Node* src_offset = (ae == StrIntrinsicNode::LL) ? from_index : _gvn.transform(new LShiftINode(from_index, intcon(1))); 1230 src_count = _gvn.transform(new SubINode(src_count, from_index)); 1231 Node* src_start = array_element_address(src, src_offset, T_BYTE); 1232 Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE); 1233 1234 // Range checks 1235 generate_string_range_check(src, src_offset, src_count, ae != StrIntrinsicNode::LL); 1236 generate_string_range_check(tgt, intcon(0), tgt_count, ae == StrIntrinsicNode::UU); 1237 if (stopped()) { 1238 return true; 1239 } 1240 1241 RegionNode* region = new RegionNode(5); 1242 Node* phi = new PhiNode(region, TypeInt::INT); 1243 1244 Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, region, phi, ae); 1245 if (result != NULL) { 1246 // The result is index relative to from_index if substring was found, -1 otherwise. 1247 // Generate code which will fold into cmove. 1248 Node* cmp = _gvn.transform(new CmpINode(result, intcon(0))); 1249 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::lt)); 1250 1251 Node* if_lt = generate_slow_guard(bol, NULL); 1252 if (if_lt != NULL) { 1253 // result == -1 1254 phi->init_req(3, result); 1255 region->init_req(3, if_lt); 1256 } 1257 if (!stopped()) { 1258 result = _gvn.transform(new AddINode(result, from_index)); 1259 phi->init_req(4, result); 1260 region->init_req(4, control()); 1261 } 1262 } 1263 1264 set_control(_gvn.transform(region)); 1265 record_for_igvn(region); 1266 set_result(_gvn.transform(phi)); 1267 clear_upper_avx(); 1268 1269 return true; 1270 } 1271 1272 // Create StrIndexOfNode with fast path checks 1273 Node* LibraryCallKit::make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_start, Node* tgt_count, 1274 RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae) { 1275 // Check for substr count > string count 1276 Node* cmp = _gvn.transform(new CmpINode(tgt_count, src_count)); 1277 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::gt)); 1278 Node* if_gt = generate_slow_guard(bol, NULL); 1279 if (if_gt != NULL) { 1280 phi->init_req(1, intcon(-1)); 1281 region->init_req(1, if_gt); 1282 } 1283 if (!stopped()) { 1284 // Check for substr count == 0 1285 cmp = _gvn.transform(new CmpINode(tgt_count, intcon(0))); 1286 bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq)); 1287 Node* if_zero = generate_slow_guard(bol, NULL); 1288 if (if_zero != NULL) { 1289 phi->init_req(2, intcon(0)); 1290 region->init_req(2, if_zero); 1291 } 1292 } 1293 if (!stopped()) { 1294 return make_string_method_node(Op_StrIndexOf, src_start, src_count, tgt_start, tgt_count, ae); 1295 } 1296 return NULL; 1297 } 1298 1299 //-----------------------------inline_string_indexOfChar----------------------- 1300 bool LibraryCallKit::inline_string_indexOfChar(StrIntrinsicNode::ArgEnc ae) { 1301 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 1302 return false; 1303 } 1304 if (!Matcher::match_rule_supported(Op_StrIndexOfChar)) { 1305 return false; 1306 } 1307 assert(callee()->signature()->size() == 4, "String.indexOfChar() has 4 arguments"); 1308 Node* src = argument(0); // byte[] 1309 Node* tgt = argument(1); // tgt is int ch 1310 Node* from_index = argument(2); 1311 Node* max = argument(3); 1312 1313 src = must_be_not_null(src, true); 1314 1315 Node* src_offset = ae == StrIntrinsicNode::L ? from_index : _gvn.transform(new LShiftINode(from_index, intcon(1))); 1316 Node* src_start = array_element_address(src, src_offset, T_BYTE); 1317 Node* src_count = _gvn.transform(new SubINode(max, from_index)); 1318 1319 // Range checks 1320 generate_string_range_check(src, src_offset, src_count, ae == StrIntrinsicNode::U); 1321 if (stopped()) { 1322 return true; 1323 } 1324 1325 RegionNode* region = new RegionNode(3); 1326 Node* phi = new PhiNode(region, TypeInt::INT); 1327 1328 Node* result = new StrIndexOfCharNode(control(), memory(TypeAryPtr::BYTES), src_start, src_count, tgt, ae); 1329 C->set_has_split_ifs(true); // Has chance for split-if optimization 1330 _gvn.transform(result); 1331 1332 Node* cmp = _gvn.transform(new CmpINode(result, intcon(0))); 1333 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::lt)); 1334 1335 Node* if_lt = generate_slow_guard(bol, NULL); 1336 if (if_lt != NULL) { 1337 // result == -1 1338 phi->init_req(2, result); 1339 region->init_req(2, if_lt); 1340 } 1341 if (!stopped()) { 1342 result = _gvn.transform(new AddINode(result, from_index)); 1343 phi->init_req(1, result); 1344 region->init_req(1, control()); 1345 } 1346 set_control(_gvn.transform(region)); 1347 record_for_igvn(region); 1348 set_result(_gvn.transform(phi)); 1349 1350 return true; 1351 } 1352 //---------------------------inline_string_copy--------------------- 1353 // compressIt == true --> generate a compressed copy operation (compress char[]/byte[] to byte[]) 1354 // int StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) 1355 // int StringUTF16.compress(byte[] src, int srcOff, byte[] dst, int dstOff, int len) 1356 // compressIt == false --> generate an inflated copy operation (inflate byte[] to char[]/byte[]) 1357 // void StringLatin1.inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) 1358 // void StringLatin1.inflate(byte[] src, int srcOff, byte[] dst, int dstOff, int len) 1359 bool LibraryCallKit::inline_string_copy(bool compress) { 1360 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 1361 return false; 1362 } 1363 int nargs = 5; // 2 oops, 3 ints 1364 assert(callee()->signature()->size() == nargs, "string copy has 5 arguments"); 1365 1366 Node* src = argument(0); 1367 Node* src_offset = argument(1); 1368 Node* dst = argument(2); 1369 Node* dst_offset = argument(3); 1370 Node* length = argument(4); 1371 1372 // Check for allocation before we add nodes that would confuse 1373 // tightly_coupled_allocation() 1374 AllocateArrayNode* alloc = tightly_coupled_allocation(dst); 1375 1376 // Figure out the size and type of the elements we will be copying. 1377 const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr(); 1378 const TypeAryPtr* dst_type = dst->Value(&_gvn)->isa_aryptr(); 1379 if (src_type == nullptr || dst_type == nullptr) { 1380 return false; 1381 } 1382 BasicType src_elem = src_type->elem()->array_element_basic_type(); 1383 BasicType dst_elem = dst_type->elem()->array_element_basic_type(); 1384 assert((compress && dst_elem == T_BYTE && (src_elem == T_BYTE || src_elem == T_CHAR)) || 1385 (!compress && src_elem == T_BYTE && (dst_elem == T_BYTE || dst_elem == T_CHAR)), 1386 "Unsupported array types for inline_string_copy"); 1387 1388 src = must_be_not_null(src, true); 1389 dst = must_be_not_null(dst, true); 1390 1391 // Convert char[] offsets to byte[] offsets 1392 bool convert_src = (compress && src_elem == T_BYTE); 1393 bool convert_dst = (!compress && dst_elem == T_BYTE); 1394 if (convert_src) { 1395 src_offset = _gvn.transform(new LShiftINode(src_offset, intcon(1))); 1396 } else if (convert_dst) { 1397 dst_offset = _gvn.transform(new LShiftINode(dst_offset, intcon(1))); 1398 } 1399 1400 // Range checks 1401 generate_string_range_check(src, src_offset, length, convert_src); 1402 generate_string_range_check(dst, dst_offset, length, convert_dst); 1403 if (stopped()) { 1404 return true; 1405 } 1406 1407 Node* src_start = array_element_address(src, src_offset, src_elem); 1408 Node* dst_start = array_element_address(dst, dst_offset, dst_elem); 1409 // 'src_start' points to src array + scaled offset 1410 // 'dst_start' points to dst array + scaled offset 1411 Node* count = NULL; 1412 if (compress) { 1413 count = compress_string(src_start, TypeAryPtr::get_array_body_type(src_elem), dst_start, length); 1414 } else { 1415 inflate_string(src_start, dst_start, TypeAryPtr::get_array_body_type(dst_elem), length); 1416 } 1417 1418 if (alloc != NULL) { 1419 if (alloc->maybe_set_complete(&_gvn)) { 1420 // "You break it, you buy it." 1421 InitializeNode* init = alloc->initialization(); 1422 assert(init->is_complete(), "we just did this"); 1423 init->set_complete_with_arraycopy(); 1424 assert(dst->is_CheckCastPP(), "sanity"); 1425 assert(dst->in(0)->in(0) == init, "dest pinned"); 1426 } 1427 // Do not let stores that initialize this object be reordered with 1428 // a subsequent store that would make this object accessible by 1429 // other threads. 1430 // Record what AllocateNode this StoreStore protects so that 1431 // escape analysis can go from the MemBarStoreStoreNode to the 1432 // AllocateNode and eliminate the MemBarStoreStoreNode if possible 1433 // based on the escape status of the AllocateNode. 1434 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 1435 } 1436 if (compress) { 1437 set_result(_gvn.transform(count)); 1438 } 1439 clear_upper_avx(); 1440 1441 return true; 1442 } 1443 1444 #ifdef _LP64 1445 #define XTOP ,top() /*additional argument*/ 1446 #else //_LP64 1447 #define XTOP /*no additional argument*/ 1448 #endif //_LP64 1449 1450 //------------------------inline_string_toBytesU-------------------------- 1451 // public static byte[] StringUTF16.toBytes(char[] value, int off, int len) 1452 bool LibraryCallKit::inline_string_toBytesU() { 1453 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 1454 return false; 1455 } 1456 // Get the arguments. 1457 Node* value = argument(0); 1458 Node* offset = argument(1); 1459 Node* length = argument(2); 1460 1461 Node* newcopy = NULL; 1462 1463 // Set the original stack and the reexecute bit for the interpreter to reexecute 1464 // the bytecode that invokes StringUTF16.toBytes() if deoptimization happens. 1465 { PreserveReexecuteState preexecs(this); 1466 jvms()->set_should_reexecute(true); 1467 1468 // Check if a null path was taken unconditionally. 1469 value = null_check(value); 1470 1471 RegionNode* bailout = new RegionNode(1); 1472 record_for_igvn(bailout); 1473 1474 // Range checks 1475 generate_negative_guard(offset, bailout); 1476 generate_negative_guard(length, bailout); 1477 generate_limit_guard(offset, length, load_array_length(value), bailout); 1478 // Make sure that resulting byte[] length does not overflow Integer.MAX_VALUE 1479 generate_limit_guard(length, intcon(0), intcon(max_jint/2), bailout); 1480 1481 if (bailout->req() > 1) { 1482 PreserveJVMState pjvms(this); 1483 set_control(_gvn.transform(bailout)); 1484 uncommon_trap(Deoptimization::Reason_intrinsic, 1485 Deoptimization::Action_maybe_recompile); 1486 } 1487 if (stopped()) { 1488 return true; 1489 } 1490 1491 Node* size = _gvn.transform(new LShiftINode(length, intcon(1))); 1492 Node* klass_node = makecon(TypeKlassPtr::make(ciTypeArrayKlass::make(T_BYTE))); 1493 newcopy = new_array(klass_node, size, 0); // no arguments to push 1494 AllocateArrayNode* alloc = tightly_coupled_allocation(newcopy); 1495 guarantee(alloc != NULL, "created above"); 1496 1497 // Calculate starting addresses. 1498 Node* src_start = array_element_address(value, offset, T_CHAR); 1499 Node* dst_start = basic_plus_adr(newcopy, arrayOopDesc::base_offset_in_bytes(T_BYTE)); 1500 1501 // Check if src array address is aligned to HeapWordSize (dst is always aligned) 1502 const TypeInt* toffset = gvn().type(offset)->is_int(); 1503 bool aligned = toffset->is_con() && ((toffset->get_con() * type2aelembytes(T_CHAR)) % HeapWordSize == 0); 1504 1505 // Figure out which arraycopy runtime method to call (disjoint, uninitialized). 1506 const char* copyfunc_name = "arraycopy"; 1507 address copyfunc_addr = StubRoutines::select_arraycopy_function(T_CHAR, aligned, true, copyfunc_name, true); 1508 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, 1509 OptoRuntime::fast_arraycopy_Type(), 1510 copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM, 1511 src_start, dst_start, ConvI2X(length) XTOP); 1512 // Do not let reads from the cloned object float above the arraycopy. 1513 if (alloc->maybe_set_complete(&_gvn)) { 1514 // "You break it, you buy it." 1515 InitializeNode* init = alloc->initialization(); 1516 assert(init->is_complete(), "we just did this"); 1517 init->set_complete_with_arraycopy(); 1518 assert(newcopy->is_CheckCastPP(), "sanity"); 1519 assert(newcopy->in(0)->in(0) == init, "dest pinned"); 1520 } 1521 // Do not let stores that initialize this object be reordered with 1522 // a subsequent store that would make this object accessible by 1523 // other threads. 1524 // Record what AllocateNode this StoreStore protects so that 1525 // escape analysis can go from the MemBarStoreStoreNode to the 1526 // AllocateNode and eliminate the MemBarStoreStoreNode if possible 1527 // based on the escape status of the AllocateNode. 1528 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 1529 } // original reexecute is set back here 1530 1531 C->set_has_split_ifs(true); // Has chance for split-if optimization 1532 if (!stopped()) { 1533 set_result(newcopy); 1534 } 1535 clear_upper_avx(); 1536 1537 return true; 1538 } 1539 1540 //------------------------inline_string_getCharsU-------------------------- 1541 // public void StringUTF16.getChars(byte[] src, int srcBegin, int srcEnd, char dst[], int dstBegin) 1542 bool LibraryCallKit::inline_string_getCharsU() { 1543 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 1544 return false; 1545 } 1546 1547 // Get the arguments. 1548 Node* src = argument(0); 1549 Node* src_begin = argument(1); 1550 Node* src_end = argument(2); // exclusive offset (i < src_end) 1551 Node* dst = argument(3); 1552 Node* dst_begin = argument(4); 1553 1554 // Check for allocation before we add nodes that would confuse 1555 // tightly_coupled_allocation() 1556 AllocateArrayNode* alloc = tightly_coupled_allocation(dst); 1557 1558 // Check if a null path was taken unconditionally. 1559 src = null_check(src); 1560 dst = null_check(dst); 1561 if (stopped()) { 1562 return true; 1563 } 1564 1565 // Get length and convert char[] offset to byte[] offset 1566 Node* length = _gvn.transform(new SubINode(src_end, src_begin)); 1567 src_begin = _gvn.transform(new LShiftINode(src_begin, intcon(1))); 1568 1569 // Range checks 1570 generate_string_range_check(src, src_begin, length, true); 1571 generate_string_range_check(dst, dst_begin, length, false); 1572 if (stopped()) { 1573 return true; 1574 } 1575 1576 if (!stopped()) { 1577 // Calculate starting addresses. 1578 Node* src_start = array_element_address(src, src_begin, T_BYTE); 1579 Node* dst_start = array_element_address(dst, dst_begin, T_CHAR); 1580 1581 // Check if array addresses are aligned to HeapWordSize 1582 const TypeInt* tsrc = gvn().type(src_begin)->is_int(); 1583 const TypeInt* tdst = gvn().type(dst_begin)->is_int(); 1584 bool aligned = tsrc->is_con() && ((tsrc->get_con() * type2aelembytes(T_BYTE)) % HeapWordSize == 0) && 1585 tdst->is_con() && ((tdst->get_con() * type2aelembytes(T_CHAR)) % HeapWordSize == 0); 1586 1587 // Figure out which arraycopy runtime method to call (disjoint, uninitialized). 1588 const char* copyfunc_name = "arraycopy"; 1589 address copyfunc_addr = StubRoutines::select_arraycopy_function(T_CHAR, aligned, true, copyfunc_name, true); 1590 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, 1591 OptoRuntime::fast_arraycopy_Type(), 1592 copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM, 1593 src_start, dst_start, ConvI2X(length) XTOP); 1594 // Do not let reads from the cloned object float above the arraycopy. 1595 if (alloc != NULL) { 1596 if (alloc->maybe_set_complete(&_gvn)) { 1597 // "You break it, you buy it." 1598 InitializeNode* init = alloc->initialization(); 1599 assert(init->is_complete(), "we just did this"); 1600 init->set_complete_with_arraycopy(); 1601 assert(dst->is_CheckCastPP(), "sanity"); 1602 assert(dst->in(0)->in(0) == init, "dest pinned"); 1603 } 1604 // Do not let stores that initialize this object be reordered with 1605 // a subsequent store that would make this object accessible by 1606 // other threads. 1607 // Record what AllocateNode this StoreStore protects so that 1608 // escape analysis can go from the MemBarStoreStoreNode to the 1609 // AllocateNode and eliminate the MemBarStoreStoreNode if possible 1610 // based on the escape status of the AllocateNode. 1611 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 1612 } else { 1613 insert_mem_bar(Op_MemBarCPUOrder); 1614 } 1615 } 1616 1617 C->set_has_split_ifs(true); // Has chance for split-if optimization 1618 return true; 1619 } 1620 1621 //----------------------inline_string_char_access---------------------------- 1622 // Store/Load char to/from byte[] array. 1623 // static void StringUTF16.putChar(byte[] val, int index, int c) 1624 // static char StringUTF16.getChar(byte[] val, int index) 1625 bool LibraryCallKit::inline_string_char_access(bool is_store) { 1626 Node* value = argument(0); 1627 Node* index = argument(1); 1628 Node* ch = is_store ? argument(2) : NULL; 1629 1630 // This intrinsic accesses byte[] array as char[] array. Computing the offsets 1631 // correctly requires matched array shapes. 1632 assert (arrayOopDesc::base_offset_in_bytes(T_CHAR) == arrayOopDesc::base_offset_in_bytes(T_BYTE), 1633 "sanity: byte[] and char[] bases agree"); 1634 assert (type2aelembytes(T_CHAR) == type2aelembytes(T_BYTE)*2, 1635 "sanity: byte[] and char[] scales agree"); 1636 1637 // Bail when getChar over constants is requested: constant folding would 1638 // reject folding mismatched char access over byte[]. A normal inlining for getChar 1639 // Java method would constant fold nicely instead. 1640 if (!is_store && value->is_Con() && index->is_Con()) { 1641 return false; 1642 } 1643 1644 // Save state and restore on bailout 1645 uint old_sp = sp(); 1646 SafePointNode* old_map = clone_map(); 1647 1648 value = must_be_not_null(value, true); 1649 1650 Node* adr = array_element_address(value, index, T_CHAR); 1651 if (adr->is_top()) { 1652 set_map(old_map); 1653 set_sp(old_sp); 1654 return false; 1655 } 1656 old_map->destruct(&_gvn); 1657 if (is_store) { 1658 access_store_at(value, adr, TypeAryPtr::BYTES, ch, TypeInt::CHAR, T_CHAR, IN_HEAP | MO_UNORDERED | C2_MISMATCHED); 1659 } else { 1660 ch = access_load_at(value, adr, TypeAryPtr::BYTES, TypeInt::CHAR, T_CHAR, IN_HEAP | MO_UNORDERED | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD | C2_UNKNOWN_CONTROL_LOAD); 1661 set_result(ch); 1662 } 1663 return true; 1664 } 1665 1666 //--------------------------round_double_node-------------------------------- 1667 // Round a double node if necessary. 1668 Node* LibraryCallKit::round_double_node(Node* n) { 1669 if (Matcher::strict_fp_requires_explicit_rounding) { 1670 #ifdef IA32 1671 if (UseSSE < 2) { 1672 n = _gvn.transform(new RoundDoubleNode(NULL, n)); 1673 } 1674 #else 1675 Unimplemented(); 1676 #endif // IA32 1677 } 1678 return n; 1679 } 1680 1681 //------------------------------inline_math----------------------------------- 1682 // public static double Math.abs(double) 1683 // public static double Math.sqrt(double) 1684 // public static double Math.log(double) 1685 // public static double Math.log10(double) 1686 // public static double Math.round(double) 1687 bool LibraryCallKit::inline_double_math(vmIntrinsics::ID id) { 1688 Node* arg = round_double_node(argument(0)); 1689 Node* n = NULL; 1690 switch (id) { 1691 case vmIntrinsics::_dabs: n = new AbsDNode( arg); break; 1692 case vmIntrinsics::_dsqrt: 1693 case vmIntrinsics::_dsqrt_strict: 1694 n = new SqrtDNode(C, control(), arg); break; 1695 case vmIntrinsics::_ceil: n = RoundDoubleModeNode::make(_gvn, arg, RoundDoubleModeNode::rmode_ceil); break; 1696 case vmIntrinsics::_floor: n = RoundDoubleModeNode::make(_gvn, arg, RoundDoubleModeNode::rmode_floor); break; 1697 case vmIntrinsics::_rint: n = RoundDoubleModeNode::make(_gvn, arg, RoundDoubleModeNode::rmode_rint); break; 1698 case vmIntrinsics::_roundD: n = new RoundDNode(arg); break; 1699 case vmIntrinsics::_dcopySign: n = CopySignDNode::make(_gvn, arg, round_double_node(argument(2))); break; 1700 case vmIntrinsics::_dsignum: n = SignumDNode::make(_gvn, arg); break; 1701 default: fatal_unexpected_iid(id); break; 1702 } 1703 set_result(_gvn.transform(n)); 1704 return true; 1705 } 1706 1707 //------------------------------inline_math----------------------------------- 1708 // public static float Math.abs(float) 1709 // public static int Math.abs(int) 1710 // public static long Math.abs(long) 1711 bool LibraryCallKit::inline_math(vmIntrinsics::ID id) { 1712 Node* arg = argument(0); 1713 Node* n = NULL; 1714 switch (id) { 1715 case vmIntrinsics::_fabs: n = new AbsFNode( arg); break; 1716 case vmIntrinsics::_iabs: n = new AbsINode( arg); break; 1717 case vmIntrinsics::_labs: n = new AbsLNode( arg); break; 1718 case vmIntrinsics::_fcopySign: n = new CopySignFNode(arg, argument(1)); break; 1719 case vmIntrinsics::_fsignum: n = SignumFNode::make(_gvn, arg); break; 1720 case vmIntrinsics::_roundF: n = new RoundFNode(arg); break; 1721 default: fatal_unexpected_iid(id); break; 1722 } 1723 set_result(_gvn.transform(n)); 1724 return true; 1725 } 1726 1727 //------------------------------runtime_math----------------------------- 1728 bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) { 1729 assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(), 1730 "must be (DD)D or (D)D type"); 1731 1732 // Inputs 1733 Node* a = round_double_node(argument(0)); 1734 Node* b = (call_type == OptoRuntime::Math_DD_D_Type()) ? round_double_node(argument(2)) : NULL; 1735 1736 const TypePtr* no_memory_effects = NULL; 1737 Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName, 1738 no_memory_effects, 1739 a, top(), b, b ? top() : NULL); 1740 Node* value = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+0)); 1741 #ifdef ASSERT 1742 Node* value_top = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+1)); 1743 assert(value_top == top(), "second value must be top"); 1744 #endif 1745 1746 set_result(value); 1747 return true; 1748 } 1749 1750 //------------------------------inline_math_pow----------------------------- 1751 bool LibraryCallKit::inline_math_pow() { 1752 Node* exp = round_double_node(argument(2)); 1753 const TypeD* d = _gvn.type(exp)->isa_double_constant(); 1754 if (d != NULL) { 1755 if (d->getd() == 2.0) { 1756 // Special case: pow(x, 2.0) => x * x 1757 Node* base = round_double_node(argument(0)); 1758 set_result(_gvn.transform(new MulDNode(base, base))); 1759 return true; 1760 } else if (d->getd() == 0.5 && Matcher::match_rule_supported(Op_SqrtD)) { 1761 // Special case: pow(x, 0.5) => sqrt(x) 1762 Node* base = round_double_node(argument(0)); 1763 Node* zero = _gvn.zerocon(T_DOUBLE); 1764 1765 RegionNode* region = new RegionNode(3); 1766 Node* phi = new PhiNode(region, Type::DOUBLE); 1767 1768 Node* cmp = _gvn.transform(new CmpDNode(base, zero)); 1769 // According to the API specs, pow(-0.0, 0.5) = 0.0 and sqrt(-0.0) = -0.0. 1770 // So pow(-0.0, 0.5) shouldn't be replaced with sqrt(-0.0). 1771 // -0.0/+0.0 are both excluded since floating-point comparison doesn't distinguish -0.0 from +0.0. 1772 Node* test = _gvn.transform(new BoolNode(cmp, BoolTest::le)); 1773 1774 Node* if_pow = generate_slow_guard(test, NULL); 1775 Node* value_sqrt = _gvn.transform(new SqrtDNode(C, control(), base)); 1776 phi->init_req(1, value_sqrt); 1777 region->init_req(1, control()); 1778 1779 if (if_pow != NULL) { 1780 set_control(if_pow); 1781 address target = StubRoutines::dpow() != NULL ? StubRoutines::dpow() : 1782 CAST_FROM_FN_PTR(address, SharedRuntime::dpow); 1783 const TypePtr* no_memory_effects = NULL; 1784 Node* trig = make_runtime_call(RC_LEAF, OptoRuntime::Math_DD_D_Type(), target, "POW", 1785 no_memory_effects, base, top(), exp, top()); 1786 Node* value_pow = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+0)); 1787 #ifdef ASSERT 1788 Node* value_top = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+1)); 1789 assert(value_top == top(), "second value must be top"); 1790 #endif 1791 phi->init_req(2, value_pow); 1792 region->init_req(2, _gvn.transform(new ProjNode(trig, TypeFunc::Control))); 1793 } 1794 1795 C->set_has_split_ifs(true); // Has chance for split-if optimization 1796 set_control(_gvn.transform(region)); 1797 record_for_igvn(region); 1798 set_result(_gvn.transform(phi)); 1799 1800 return true; 1801 } 1802 } 1803 1804 return StubRoutines::dpow() != NULL ? 1805 runtime_math(OptoRuntime::Math_DD_D_Type(), StubRoutines::dpow(), "dpow") : 1806 runtime_math(OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW"); 1807 } 1808 1809 //------------------------------inline_math_native----------------------------- 1810 bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) { 1811 switch (id) { 1812 case vmIntrinsics::_dsin: 1813 return StubRoutines::dsin() != NULL ? 1814 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dsin(), "dsin") : 1815 runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dsin), "SIN"); 1816 case vmIntrinsics::_dcos: 1817 return StubRoutines::dcos() != NULL ? 1818 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dcos(), "dcos") : 1819 runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dcos), "COS"); 1820 case vmIntrinsics::_dtan: 1821 return StubRoutines::dtan() != NULL ? 1822 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dtan(), "dtan") : 1823 runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dtan), "TAN"); 1824 case vmIntrinsics::_dexp: 1825 return StubRoutines::dexp() != NULL ? 1826 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dexp(), "dexp") : 1827 runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dexp), "EXP"); 1828 case vmIntrinsics::_dlog: 1829 return StubRoutines::dlog() != NULL ? 1830 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog(), "dlog") : 1831 runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dlog), "LOG"); 1832 case vmIntrinsics::_dlog10: 1833 return StubRoutines::dlog10() != NULL ? 1834 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog10(), "dlog10") : 1835 runtime_math(OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dlog10), "LOG10"); 1836 1837 case vmIntrinsics::_roundD: return Matcher::match_rule_supported(Op_RoundD) ? inline_double_math(id) : false; 1838 case vmIntrinsics::_ceil: 1839 case vmIntrinsics::_floor: 1840 case vmIntrinsics::_rint: return Matcher::match_rule_supported(Op_RoundDoubleMode) ? inline_double_math(id) : false; 1841 1842 case vmIntrinsics::_dsqrt: 1843 case vmIntrinsics::_dsqrt_strict: 1844 return Matcher::match_rule_supported(Op_SqrtD) ? inline_double_math(id) : false; 1845 case vmIntrinsics::_dabs: return Matcher::has_match_rule(Op_AbsD) ? inline_double_math(id) : false; 1846 case vmIntrinsics::_fabs: return Matcher::match_rule_supported(Op_AbsF) ? inline_math(id) : false; 1847 case vmIntrinsics::_iabs: return Matcher::match_rule_supported(Op_AbsI) ? inline_math(id) : false; 1848 case vmIntrinsics::_labs: return Matcher::match_rule_supported(Op_AbsL) ? inline_math(id) : false; 1849 1850 case vmIntrinsics::_dpow: return inline_math_pow(); 1851 case vmIntrinsics::_dcopySign: return inline_double_math(id); 1852 case vmIntrinsics::_fcopySign: return inline_math(id); 1853 case vmIntrinsics::_dsignum: return Matcher::match_rule_supported(Op_SignumD) ? inline_double_math(id) : false; 1854 case vmIntrinsics::_fsignum: return Matcher::match_rule_supported(Op_SignumF) ? inline_math(id) : false; 1855 case vmIntrinsics::_roundF: return Matcher::match_rule_supported(Op_RoundF) ? inline_math(id) : false; 1856 1857 // These intrinsics are not yet correctly implemented 1858 case vmIntrinsics::_datan2: 1859 return false; 1860 1861 default: 1862 fatal_unexpected_iid(id); 1863 return false; 1864 } 1865 } 1866 1867 //----------------------------inline_notify-----------------------------------* 1868 bool LibraryCallKit::inline_notify(vmIntrinsics::ID id) { 1869 const TypeFunc* ftype = OptoRuntime::monitor_notify_Type(); 1870 address func; 1871 if (id == vmIntrinsics::_notify) { 1872 func = OptoRuntime::monitor_notify_Java(); 1873 } else { 1874 func = OptoRuntime::monitor_notifyAll_Java(); 1875 } 1876 Node* call = make_runtime_call(RC_NO_LEAF, ftype, func, NULL, TypeRawPtr::BOTTOM, argument(0)); 1877 make_slow_call_ex(call, env()->Throwable_klass(), false); 1878 return true; 1879 } 1880 1881 1882 //----------------------------inline_min_max----------------------------------- 1883 bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) { 1884 set_result(generate_min_max(id, argument(0), argument(1))); 1885 return true; 1886 } 1887 1888 void LibraryCallKit::inline_math_mathExact(Node* math, Node *test) { 1889 Node* bol = _gvn.transform( new BoolNode(test, BoolTest::overflow) ); 1890 IfNode* check = create_and_map_if(control(), bol, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN); 1891 Node* fast_path = _gvn.transform( new IfFalseNode(check)); 1892 Node* slow_path = _gvn.transform( new IfTrueNode(check) ); 1893 1894 { 1895 PreserveJVMState pjvms(this); 1896 PreserveReexecuteState preexecs(this); 1897 jvms()->set_should_reexecute(true); 1898 1899 set_control(slow_path); 1900 set_i_o(i_o()); 1901 1902 uncommon_trap(Deoptimization::Reason_intrinsic, 1903 Deoptimization::Action_none); 1904 } 1905 1906 set_control(fast_path); 1907 set_result(math); 1908 } 1909 1910 template <typename OverflowOp> 1911 bool LibraryCallKit::inline_math_overflow(Node* arg1, Node* arg2) { 1912 typedef typename OverflowOp::MathOp MathOp; 1913 1914 MathOp* mathOp = new MathOp(arg1, arg2); 1915 Node* operation = _gvn.transform( mathOp ); 1916 Node* ofcheck = _gvn.transform( new OverflowOp(arg1, arg2) ); 1917 inline_math_mathExact(operation, ofcheck); 1918 return true; 1919 } 1920 1921 bool LibraryCallKit::inline_math_addExactI(bool is_increment) { 1922 return inline_math_overflow<OverflowAddINode>(argument(0), is_increment ? intcon(1) : argument(1)); 1923 } 1924 1925 bool LibraryCallKit::inline_math_addExactL(bool is_increment) { 1926 return inline_math_overflow<OverflowAddLNode>(argument(0), is_increment ? longcon(1) : argument(2)); 1927 } 1928 1929 bool LibraryCallKit::inline_math_subtractExactI(bool is_decrement) { 1930 return inline_math_overflow<OverflowSubINode>(argument(0), is_decrement ? intcon(1) : argument(1)); 1931 } 1932 1933 bool LibraryCallKit::inline_math_subtractExactL(bool is_decrement) { 1934 return inline_math_overflow<OverflowSubLNode>(argument(0), is_decrement ? longcon(1) : argument(2)); 1935 } 1936 1937 bool LibraryCallKit::inline_math_negateExactI() { 1938 return inline_math_overflow<OverflowSubINode>(intcon(0), argument(0)); 1939 } 1940 1941 bool LibraryCallKit::inline_math_negateExactL() { 1942 return inline_math_overflow<OverflowSubLNode>(longcon(0), argument(0)); 1943 } 1944 1945 bool LibraryCallKit::inline_math_multiplyExactI() { 1946 return inline_math_overflow<OverflowMulINode>(argument(0), argument(1)); 1947 } 1948 1949 bool LibraryCallKit::inline_math_multiplyExactL() { 1950 return inline_math_overflow<OverflowMulLNode>(argument(0), argument(2)); 1951 } 1952 1953 bool LibraryCallKit::inline_math_multiplyHigh() { 1954 set_result(_gvn.transform(new MulHiLNode(argument(0), argument(2)))); 1955 return true; 1956 } 1957 1958 bool LibraryCallKit::inline_math_unsignedMultiplyHigh() { 1959 set_result(_gvn.transform(new UMulHiLNode(argument(0), argument(2)))); 1960 return true; 1961 } 1962 1963 Node* 1964 LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) { 1965 Node* result_val = NULL; 1966 switch (id) { 1967 case vmIntrinsics::_min: 1968 case vmIntrinsics::_min_strict: 1969 result_val = _gvn.transform(new MinINode(x0, y0)); 1970 break; 1971 case vmIntrinsics::_max: 1972 case vmIntrinsics::_max_strict: 1973 result_val = _gvn.transform(new MaxINode(x0, y0)); 1974 break; 1975 default: 1976 fatal_unexpected_iid(id); 1977 break; 1978 } 1979 return result_val; 1980 } 1981 1982 inline int 1983 LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset, BasicType type) { 1984 const TypePtr* base_type = TypePtr::NULL_PTR; 1985 if (base != NULL) base_type = _gvn.type(base)->isa_ptr(); 1986 if (base_type == NULL) { 1987 // Unknown type. 1988 return Type::AnyPtr; 1989 } else if (base_type == TypePtr::NULL_PTR) { 1990 // Since this is a NULL+long form, we have to switch to a rawptr. 1991 base = _gvn.transform(new CastX2PNode(offset)); 1992 offset = MakeConX(0); 1993 return Type::RawPtr; 1994 } else if (base_type->base() == Type::RawPtr) { 1995 return Type::RawPtr; 1996 } else if (base_type->isa_oopptr()) { 1997 // Base is never null => always a heap address. 1998 if (!TypePtr::NULL_PTR->higher_equal(base_type)) { 1999 return Type::OopPtr; 2000 } 2001 // Offset is small => always a heap address. 2002 const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t(); 2003 if (offset_type != NULL && 2004 base_type->offset() == 0 && // (should always be?) 2005 offset_type->_lo >= 0 && 2006 !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) { 2007 return Type::OopPtr; 2008 } else if (type == T_OBJECT) { 2009 // off heap access to an oop doesn't make any sense. Has to be on 2010 // heap. 2011 return Type::OopPtr; 2012 } 2013 // Otherwise, it might either be oop+off or NULL+addr. 2014 return Type::AnyPtr; 2015 } else { 2016 // No information: 2017 return Type::AnyPtr; 2018 } 2019 } 2020 2021 Node* LibraryCallKit::make_unsafe_address(Node*& base, Node* offset, BasicType type, bool can_cast) { 2022 Node* uncasted_base = base; 2023 int kind = classify_unsafe_addr(uncasted_base, offset, type); 2024 if (kind == Type::RawPtr) { 2025 return basic_plus_adr(top(), uncasted_base, offset); 2026 } else if (kind == Type::AnyPtr) { 2027 assert(base == uncasted_base, "unexpected base change"); 2028 if (can_cast) { 2029 if (!_gvn.type(base)->speculative_maybe_null() && 2030 !too_many_traps(Deoptimization::Reason_speculate_null_check)) { 2031 // According to profiling, this access is always on 2032 // heap. Casting the base to not null and thus avoiding membars 2033 // around the access should allow better optimizations 2034 Node* null_ctl = top(); 2035 base = null_check_oop(base, &null_ctl, true, true, true); 2036 assert(null_ctl->is_top(), "no null control here"); 2037 return basic_plus_adr(base, offset); 2038 } else if (_gvn.type(base)->speculative_always_null() && 2039 !too_many_traps(Deoptimization::Reason_speculate_null_assert)) { 2040 // According to profiling, this access is always off 2041 // heap. 2042 base = null_assert(base); 2043 Node* raw_base = _gvn.transform(new CastX2PNode(offset)); 2044 offset = MakeConX(0); 2045 return basic_plus_adr(top(), raw_base, offset); 2046 } 2047 } 2048 // We don't know if it's an on heap or off heap access. Fall back 2049 // to raw memory access. 2050 Node* raw = _gvn.transform(new CheckCastPPNode(control(), base, TypeRawPtr::BOTTOM)); 2051 return basic_plus_adr(top(), raw, offset); 2052 } else { 2053 assert(base == uncasted_base, "unexpected base change"); 2054 // We know it's an on heap access so base can't be null 2055 if (TypePtr::NULL_PTR->higher_equal(_gvn.type(base))) { 2056 base = must_be_not_null(base, true); 2057 } 2058 return basic_plus_adr(base, offset); 2059 } 2060 } 2061 2062 //--------------------------inline_number_methods----------------------------- 2063 // inline int Integer.numberOfLeadingZeros(int) 2064 // inline int Long.numberOfLeadingZeros(long) 2065 // 2066 // inline int Integer.numberOfTrailingZeros(int) 2067 // inline int Long.numberOfTrailingZeros(long) 2068 // 2069 // inline int Integer.bitCount(int) 2070 // inline int Long.bitCount(long) 2071 // 2072 // inline char Character.reverseBytes(char) 2073 // inline short Short.reverseBytes(short) 2074 // inline int Integer.reverseBytes(int) 2075 // inline long Long.reverseBytes(long) 2076 bool LibraryCallKit::inline_number_methods(vmIntrinsics::ID id) { 2077 Node* arg = argument(0); 2078 Node* n = NULL; 2079 switch (id) { 2080 case vmIntrinsics::_numberOfLeadingZeros_i: n = new CountLeadingZerosINode( arg); break; 2081 case vmIntrinsics::_numberOfLeadingZeros_l: n = new CountLeadingZerosLNode( arg); break; 2082 case vmIntrinsics::_numberOfTrailingZeros_i: n = new CountTrailingZerosINode(arg); break; 2083 case vmIntrinsics::_numberOfTrailingZeros_l: n = new CountTrailingZerosLNode(arg); break; 2084 case vmIntrinsics::_bitCount_i: n = new PopCountINode( arg); break; 2085 case vmIntrinsics::_bitCount_l: n = new PopCountLNode( arg); break; 2086 case vmIntrinsics::_reverseBytes_c: n = new ReverseBytesUSNode(0, arg); break; 2087 case vmIntrinsics::_reverseBytes_s: n = new ReverseBytesSNode( 0, arg); break; 2088 case vmIntrinsics::_reverseBytes_i: n = new ReverseBytesINode( 0, arg); break; 2089 case vmIntrinsics::_reverseBytes_l: n = new ReverseBytesLNode( 0, arg); break; 2090 case vmIntrinsics::_reverse_i: n = new ReverseINode(0, arg); break; 2091 case vmIntrinsics::_reverse_l: n = new ReverseLNode(0, arg); break; 2092 default: fatal_unexpected_iid(id); break; 2093 } 2094 set_result(_gvn.transform(n)); 2095 return true; 2096 } 2097 2098 //--------------------------inline_bitshuffle_methods----------------------------- 2099 // inline int Integer.compress(int, int) 2100 // inline int Integer.expand(int, int) 2101 // inline long Long.compress(long, long) 2102 // inline long Long.expand(long, long) 2103 bool LibraryCallKit::inline_bitshuffle_methods(vmIntrinsics::ID id) { 2104 Node* n = NULL; 2105 switch (id) { 2106 case vmIntrinsics::_compress_i: n = new CompressBitsNode(argument(0), argument(1), TypeInt::INT); break; 2107 case vmIntrinsics::_expand_i: n = new ExpandBitsNode(argument(0), argument(1), TypeInt::INT); break; 2108 case vmIntrinsics::_compress_l: n = new CompressBitsNode(argument(0), argument(2), TypeLong::LONG); break; 2109 case vmIntrinsics::_expand_l: n = new ExpandBitsNode(argument(0), argument(2), TypeLong::LONG); break; 2110 default: fatal_unexpected_iid(id); break; 2111 } 2112 set_result(_gvn.transform(n)); 2113 return true; 2114 } 2115 2116 //--------------------------inline_number_methods----------------------------- 2117 // inline int Integer.compareUnsigned(int, int) 2118 // inline int Long.compareUnsigned(long, long) 2119 bool LibraryCallKit::inline_compare_unsigned(vmIntrinsics::ID id) { 2120 Node* arg1 = argument(0); 2121 Node* arg2 = (id == vmIntrinsics::_compareUnsigned_l) ? argument(2) : argument(1); 2122 Node* n = NULL; 2123 switch (id) { 2124 case vmIntrinsics::_compareUnsigned_i: n = new CmpU3Node(arg1, arg2); break; 2125 case vmIntrinsics::_compareUnsigned_l: n = new CmpUL3Node(arg1, arg2); break; 2126 default: fatal_unexpected_iid(id); break; 2127 } 2128 set_result(_gvn.transform(n)); 2129 return true; 2130 } 2131 2132 //--------------------------inline_unsigned_divmod_methods----------------------------- 2133 // inline int Integer.divideUnsigned(int, int) 2134 // inline int Integer.remainderUnsigned(int, int) 2135 // inline long Long.divideUnsigned(long, long) 2136 // inline long Long.remainderUnsigned(long, long) 2137 bool LibraryCallKit::inline_divmod_methods(vmIntrinsics::ID id) { 2138 Node* n = NULL; 2139 switch (id) { 2140 case vmIntrinsics::_divideUnsigned_i: { 2141 zero_check_int(argument(1)); 2142 // Compile-time detect of null-exception 2143 if (stopped()) { 2144 return true; // keep the graph constructed so far 2145 } 2146 n = new UDivINode(control(), argument(0), argument(1)); 2147 break; 2148 } 2149 case vmIntrinsics::_divideUnsigned_l: { 2150 zero_check_long(argument(2)); 2151 // Compile-time detect of null-exception 2152 if (stopped()) { 2153 return true; // keep the graph constructed so far 2154 } 2155 n = new UDivLNode(control(), argument(0), argument(2)); 2156 break; 2157 } 2158 case vmIntrinsics::_remainderUnsigned_i: { 2159 zero_check_int(argument(1)); 2160 // Compile-time detect of null-exception 2161 if (stopped()) { 2162 return true; // keep the graph constructed so far 2163 } 2164 n = new UModINode(control(), argument(0), argument(1)); 2165 break; 2166 } 2167 case vmIntrinsics::_remainderUnsigned_l: { 2168 zero_check_long(argument(2)); 2169 // Compile-time detect of null-exception 2170 if (stopped()) { 2171 return true; // keep the graph constructed so far 2172 } 2173 n = new UModLNode(control(), argument(0), argument(2)); 2174 break; 2175 } 2176 default: fatal_unexpected_iid(id); break; 2177 } 2178 set_result(_gvn.transform(n)); 2179 return true; 2180 } 2181 2182 //----------------------------inline_unsafe_access---------------------------- 2183 2184 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type) { 2185 // Attempt to infer a sharper value type from the offset and base type. 2186 ciKlass* sharpened_klass = NULL; 2187 2188 // See if it is an instance field, with an object type. 2189 if (alias_type->field() != NULL) { 2190 if (alias_type->field()->type()->is_klass()) { 2191 sharpened_klass = alias_type->field()->type()->as_klass(); 2192 } 2193 } 2194 2195 const TypeOopPtr* result = NULL; 2196 // See if it is a narrow oop array. 2197 if (adr_type->isa_aryptr()) { 2198 if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) { 2199 const TypeOopPtr* elem_type = adr_type->is_aryptr()->elem()->make_oopptr(); 2200 if (elem_type != NULL && elem_type->is_loaded()) { 2201 // Sharpen the value type. 2202 result = elem_type; 2203 } 2204 } 2205 } 2206 2207 // The sharpened class might be unloaded if there is no class loader 2208 // contraint in place. 2209 if (result == NULL && sharpened_klass != NULL && sharpened_klass->is_loaded()) { 2210 // Sharpen the value type. 2211 result = TypeOopPtr::make_from_klass(sharpened_klass); 2212 } 2213 if (result != NULL) { 2214 #ifndef PRODUCT 2215 if (C->print_intrinsics() || C->print_inlining()) { 2216 tty->print(" from base type: "); adr_type->dump(); tty->cr(); 2217 tty->print(" sharpened value: "); result->dump(); tty->cr(); 2218 } 2219 #endif 2220 } 2221 return result; 2222 } 2223 2224 DecoratorSet LibraryCallKit::mo_decorator_for_access_kind(AccessKind kind) { 2225 switch (kind) { 2226 case Relaxed: 2227 return MO_UNORDERED; 2228 case Opaque: 2229 return MO_RELAXED; 2230 case Acquire: 2231 return MO_ACQUIRE; 2232 case Release: 2233 return MO_RELEASE; 2234 case Volatile: 2235 return MO_SEQ_CST; 2236 default: 2237 ShouldNotReachHere(); 2238 return 0; 2239 } 2240 } 2241 2242 bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, const AccessKind kind, const bool unaligned) { 2243 if (callee()->is_static()) return false; // caller must have the capability! 2244 DecoratorSet decorators = C2_UNSAFE_ACCESS; 2245 guarantee(!is_store || kind != Acquire, "Acquire accesses can be produced only for loads"); 2246 guarantee( is_store || kind != Release, "Release accesses can be produced only for stores"); 2247 assert(type != T_OBJECT || !unaligned, "unaligned access not supported with object type"); 2248 2249 if (is_reference_type(type)) { 2250 decorators |= ON_UNKNOWN_OOP_REF; 2251 } 2252 2253 if (unaligned) { 2254 decorators |= C2_UNALIGNED; 2255 } 2256 2257 #ifndef PRODUCT 2258 { 2259 ResourceMark rm; 2260 // Check the signatures. 2261 ciSignature* sig = callee()->signature(); 2262 #ifdef ASSERT 2263 if (!is_store) { 2264 // Object getReference(Object base, int/long offset), etc. 2265 BasicType rtype = sig->return_type()->basic_type(); 2266 assert(rtype == type, "getter must return the expected value"); 2267 assert(sig->count() == 2, "oop getter has 2 arguments"); 2268 assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object"); 2269 assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct"); 2270 } else { 2271 // void putReference(Object base, int/long offset, Object x), etc. 2272 assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value"); 2273 assert(sig->count() == 3, "oop putter has 3 arguments"); 2274 assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object"); 2275 assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct"); 2276 BasicType vtype = sig->type_at(sig->count()-1)->basic_type(); 2277 assert(vtype == type, "putter must accept the expected value"); 2278 } 2279 #endif // ASSERT 2280 } 2281 #endif //PRODUCT 2282 2283 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". 2284 2285 Node* receiver = argument(0); // type: oop 2286 2287 // Build address expression. 2288 Node* heap_base_oop = top(); 2289 2290 // The base is either a Java object or a value produced by Unsafe.staticFieldBase 2291 Node* base = argument(1); // type: oop 2292 // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset 2293 Node* offset = argument(2); // type: long 2294 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset 2295 // to be plain byte offsets, which are also the same as those accepted 2296 // by oopDesc::field_addr. 2297 assert(Unsafe_field_offset_to_byte_offset(11) == 11, 2298 "fieldOffset must be byte-scaled"); 2299 // 32-bit machines ignore the high half! 2300 offset = ConvL2X(offset); 2301 2302 // Save state and restore on bailout 2303 uint old_sp = sp(); 2304 SafePointNode* old_map = clone_map(); 2305 2306 Node* adr = make_unsafe_address(base, offset, type, kind == Relaxed); 2307 2308 if (_gvn.type(base)->isa_ptr() == TypePtr::NULL_PTR) { 2309 if (type != T_OBJECT) { 2310 decorators |= IN_NATIVE; // off-heap primitive access 2311 } else { 2312 set_map(old_map); 2313 set_sp(old_sp); 2314 return false; // off-heap oop accesses are not supported 2315 } 2316 } else { 2317 heap_base_oop = base; // on-heap or mixed access 2318 } 2319 2320 // Can base be NULL? Otherwise, always on-heap access. 2321 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(_gvn.type(base)); 2322 2323 if (!can_access_non_heap) { 2324 decorators |= IN_HEAP; 2325 } 2326 2327 Node* val = is_store ? argument(4) : NULL; 2328 2329 const TypePtr* adr_type = _gvn.type(adr)->isa_ptr(); 2330 if (adr_type == TypePtr::NULL_PTR) { 2331 set_map(old_map); 2332 set_sp(old_sp); 2333 return false; // off-heap access with zero address 2334 } 2335 2336 // Try to categorize the address. 2337 Compile::AliasType* alias_type = C->alias_type(adr_type); 2338 assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here"); 2339 2340 if (alias_type->adr_type() == TypeInstPtr::KLASS || 2341 alias_type->adr_type() == TypeAryPtr::RANGE) { 2342 set_map(old_map); 2343 set_sp(old_sp); 2344 return false; // not supported 2345 } 2346 2347 bool mismatched = false; 2348 BasicType bt = alias_type->basic_type(); 2349 if (bt != T_ILLEGAL) { 2350 assert(alias_type->adr_type()->is_oopptr(), "should be on-heap access"); 2351 if (bt == T_BYTE && adr_type->isa_aryptr()) { 2352 // Alias type doesn't differentiate between byte[] and boolean[]). 2353 // Use address type to get the element type. 2354 bt = adr_type->is_aryptr()->elem()->array_element_basic_type(); 2355 } 2356 if (is_reference_type(bt, true)) { 2357 // accessing an array field with getReference is not a mismatch 2358 bt = T_OBJECT; 2359 } 2360 if ((bt == T_OBJECT) != (type == T_OBJECT)) { 2361 // Don't intrinsify mismatched object accesses 2362 set_map(old_map); 2363 set_sp(old_sp); 2364 return false; 2365 } 2366 mismatched = (bt != type); 2367 } else if (alias_type->adr_type()->isa_oopptr()) { 2368 mismatched = true; // conservatively mark all "wide" on-heap accesses as mismatched 2369 } 2370 2371 old_map->destruct(&_gvn); 2372 assert(!mismatched || alias_type->adr_type()->is_oopptr(), "off-heap access can't be mismatched"); 2373 2374 if (mismatched) { 2375 decorators |= C2_MISMATCHED; 2376 } 2377 2378 // First guess at the value type. 2379 const Type *value_type = Type::get_const_basic_type(type); 2380 2381 // Figure out the memory ordering. 2382 decorators |= mo_decorator_for_access_kind(kind); 2383 2384 if (!is_store && type == T_OBJECT) { 2385 const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type); 2386 if (tjp != NULL) { 2387 value_type = tjp; 2388 } 2389 } 2390 2391 receiver = null_check(receiver); 2392 if (stopped()) { 2393 return true; 2394 } 2395 // Heap pointers get a null-check from the interpreter, 2396 // as a courtesy. However, this is not guaranteed by Unsafe, 2397 // and it is not possible to fully distinguish unintended nulls 2398 // from intended ones in this API. 2399 2400 if (!is_store) { 2401 Node* p = NULL; 2402 // Try to constant fold a load from a constant field 2403 ciField* field = alias_type->field(); 2404 if (heap_base_oop != top() && field != NULL && field->is_constant() && !mismatched) { 2405 // final or stable field 2406 p = make_constant_from_field(field, heap_base_oop); 2407 } 2408 2409 if (p == NULL) { // Could not constant fold the load 2410 p = access_load_at(heap_base_oop, adr, adr_type, value_type, type, decorators); 2411 // Normalize the value returned by getBoolean in the following cases 2412 if (type == T_BOOLEAN && 2413 (mismatched || 2414 heap_base_oop == top() || // - heap_base_oop is NULL or 2415 (can_access_non_heap && field == NULL)) // - heap_base_oop is potentially NULL 2416 // and the unsafe access is made to large offset 2417 // (i.e., larger than the maximum offset necessary for any 2418 // field access) 2419 ) { 2420 IdealKit ideal = IdealKit(this); 2421 #define __ ideal. 2422 IdealVariable normalized_result(ideal); 2423 __ declarations_done(); 2424 __ set(normalized_result, p); 2425 __ if_then(p, BoolTest::ne, ideal.ConI(0)); 2426 __ set(normalized_result, ideal.ConI(1)); 2427 ideal.end_if(); 2428 final_sync(ideal); 2429 p = __ value(normalized_result); 2430 #undef __ 2431 } 2432 } 2433 if (type == T_ADDRESS) { 2434 p = gvn().transform(new CastP2XNode(NULL, p)); 2435 p = ConvX2UL(p); 2436 } 2437 // The load node has the control of the preceding MemBarCPUOrder. All 2438 // following nodes will have the control of the MemBarCPUOrder inserted at 2439 // the end of this method. So, pushing the load onto the stack at a later 2440 // point is fine. 2441 set_result(p); 2442 } else { 2443 if (bt == T_ADDRESS) { 2444 // Repackage the long as a pointer. 2445 val = ConvL2X(val); 2446 val = gvn().transform(new CastX2PNode(val)); 2447 } 2448 access_store_at(heap_base_oop, adr, adr_type, val, value_type, type, decorators); 2449 } 2450 2451 return true; 2452 } 2453 2454 //----------------------------inline_unsafe_load_store---------------------------- 2455 // This method serves a couple of different customers (depending on LoadStoreKind): 2456 // 2457 // LS_cmp_swap: 2458 // 2459 // boolean compareAndSetReference(Object o, long offset, Object expected, Object x); 2460 // boolean compareAndSetInt( Object o, long offset, int expected, int x); 2461 // boolean compareAndSetLong( Object o, long offset, long expected, long x); 2462 // 2463 // LS_cmp_swap_weak: 2464 // 2465 // boolean weakCompareAndSetReference( Object o, long offset, Object expected, Object x); 2466 // boolean weakCompareAndSetReferencePlain( Object o, long offset, Object expected, Object x); 2467 // boolean weakCompareAndSetReferenceAcquire(Object o, long offset, Object expected, Object x); 2468 // boolean weakCompareAndSetReferenceRelease(Object o, long offset, Object expected, Object x); 2469 // 2470 // boolean weakCompareAndSetInt( Object o, long offset, int expected, int x); 2471 // boolean weakCompareAndSetIntPlain( Object o, long offset, int expected, int x); 2472 // boolean weakCompareAndSetIntAcquire( Object o, long offset, int expected, int x); 2473 // boolean weakCompareAndSetIntRelease( Object o, long offset, int expected, int x); 2474 // 2475 // boolean weakCompareAndSetLong( Object o, long offset, long expected, long x); 2476 // boolean weakCompareAndSetLongPlain( Object o, long offset, long expected, long x); 2477 // boolean weakCompareAndSetLongAcquire( Object o, long offset, long expected, long x); 2478 // boolean weakCompareAndSetLongRelease( Object o, long offset, long expected, long x); 2479 // 2480 // LS_cmp_exchange: 2481 // 2482 // Object compareAndExchangeReferenceVolatile(Object o, long offset, Object expected, Object x); 2483 // Object compareAndExchangeReferenceAcquire( Object o, long offset, Object expected, Object x); 2484 // Object compareAndExchangeReferenceRelease( Object o, long offset, Object expected, Object x); 2485 // 2486 // Object compareAndExchangeIntVolatile( Object o, long offset, Object expected, Object x); 2487 // Object compareAndExchangeIntAcquire( Object o, long offset, Object expected, Object x); 2488 // Object compareAndExchangeIntRelease( Object o, long offset, Object expected, Object x); 2489 // 2490 // Object compareAndExchangeLongVolatile( Object o, long offset, Object expected, Object x); 2491 // Object compareAndExchangeLongAcquire( Object o, long offset, Object expected, Object x); 2492 // Object compareAndExchangeLongRelease( Object o, long offset, Object expected, Object x); 2493 // 2494 // LS_get_add: 2495 // 2496 // int getAndAddInt( Object o, long offset, int delta) 2497 // long getAndAddLong(Object o, long offset, long delta) 2498 // 2499 // LS_get_set: 2500 // 2501 // int getAndSet(Object o, long offset, int newValue) 2502 // long getAndSet(Object o, long offset, long newValue) 2503 // Object getAndSet(Object o, long offset, Object newValue) 2504 // 2505 bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadStoreKind kind, const AccessKind access_kind) { 2506 // This basic scheme here is the same as inline_unsafe_access, but 2507 // differs in enough details that combining them would make the code 2508 // overly confusing. (This is a true fact! I originally combined 2509 // them, but even I was confused by it!) As much code/comments as 2510 // possible are retained from inline_unsafe_access though to make 2511 // the correspondences clearer. - dl 2512 2513 if (callee()->is_static()) return false; // caller must have the capability! 2514 2515 DecoratorSet decorators = C2_UNSAFE_ACCESS; 2516 decorators |= mo_decorator_for_access_kind(access_kind); 2517 2518 #ifndef PRODUCT 2519 BasicType rtype; 2520 { 2521 ResourceMark rm; 2522 // Check the signatures. 2523 ciSignature* sig = callee()->signature(); 2524 rtype = sig->return_type()->basic_type(); 2525 switch(kind) { 2526 case LS_get_add: 2527 case LS_get_set: { 2528 // Check the signatures. 2529 #ifdef ASSERT 2530 assert(rtype == type, "get and set must return the expected type"); 2531 assert(sig->count() == 3, "get and set has 3 arguments"); 2532 assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object"); 2533 assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long"); 2534 assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new value/delta"); 2535 assert(access_kind == Volatile, "mo is not passed to intrinsic nodes in current implementation"); 2536 #endif // ASSERT 2537 break; 2538 } 2539 case LS_cmp_swap: 2540 case LS_cmp_swap_weak: { 2541 // Check the signatures. 2542 #ifdef ASSERT 2543 assert(rtype == T_BOOLEAN, "CAS must return boolean"); 2544 assert(sig->count() == 4, "CAS has 4 arguments"); 2545 assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object"); 2546 assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long"); 2547 #endif // ASSERT 2548 break; 2549 } 2550 case LS_cmp_exchange: { 2551 // Check the signatures. 2552 #ifdef ASSERT 2553 assert(rtype == type, "CAS must return the expected type"); 2554 assert(sig->count() == 4, "CAS has 4 arguments"); 2555 assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object"); 2556 assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long"); 2557 #endif // ASSERT 2558 break; 2559 } 2560 default: 2561 ShouldNotReachHere(); 2562 } 2563 } 2564 #endif //PRODUCT 2565 2566 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". 2567 2568 // Get arguments: 2569 Node* receiver = NULL; 2570 Node* base = NULL; 2571 Node* offset = NULL; 2572 Node* oldval = NULL; 2573 Node* newval = NULL; 2574 switch(kind) { 2575 case LS_cmp_swap: 2576 case LS_cmp_swap_weak: 2577 case LS_cmp_exchange: { 2578 const bool two_slot_type = type2size[type] == 2; 2579 receiver = argument(0); // type: oop 2580 base = argument(1); // type: oop 2581 offset = argument(2); // type: long 2582 oldval = argument(4); // type: oop, int, or long 2583 newval = argument(two_slot_type ? 6 : 5); // type: oop, int, or long 2584 break; 2585 } 2586 case LS_get_add: 2587 case LS_get_set: { 2588 receiver = argument(0); // type: oop 2589 base = argument(1); // type: oop 2590 offset = argument(2); // type: long 2591 oldval = NULL; 2592 newval = argument(4); // type: oop, int, or long 2593 break; 2594 } 2595 default: 2596 ShouldNotReachHere(); 2597 } 2598 2599 // Build field offset expression. 2600 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset 2601 // to be plain byte offsets, which are also the same as those accepted 2602 // by oopDesc::field_addr. 2603 assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); 2604 // 32-bit machines ignore the high half of long offsets 2605 offset = ConvL2X(offset); 2606 // Save state and restore on bailout 2607 uint old_sp = sp(); 2608 SafePointNode* old_map = clone_map(); 2609 Node* adr = make_unsafe_address(base, offset,type, false); 2610 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); 2611 2612 Compile::AliasType* alias_type = C->alias_type(adr_type); 2613 BasicType bt = alias_type->basic_type(); 2614 if (bt != T_ILLEGAL && 2615 (is_reference_type(bt) != (type == T_OBJECT))) { 2616 // Don't intrinsify mismatched object accesses. 2617 set_map(old_map); 2618 set_sp(old_sp); 2619 return false; 2620 } 2621 2622 old_map->destruct(&_gvn); 2623 2624 // For CAS, unlike inline_unsafe_access, there seems no point in 2625 // trying to refine types. Just use the coarse types here. 2626 assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here"); 2627 const Type *value_type = Type::get_const_basic_type(type); 2628 2629 switch (kind) { 2630 case LS_get_set: 2631 case LS_cmp_exchange: { 2632 if (type == T_OBJECT) { 2633 const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type); 2634 if (tjp != NULL) { 2635 value_type = tjp; 2636 } 2637 } 2638 break; 2639 } 2640 case LS_cmp_swap: 2641 case LS_cmp_swap_weak: 2642 case LS_get_add: 2643 break; 2644 default: 2645 ShouldNotReachHere(); 2646 } 2647 2648 // Null check receiver. 2649 receiver = null_check(receiver); 2650 if (stopped()) { 2651 return true; 2652 } 2653 2654 int alias_idx = C->get_alias_index(adr_type); 2655 2656 if (is_reference_type(type)) { 2657 decorators |= IN_HEAP | ON_UNKNOWN_OOP_REF; 2658 2659 // Transformation of a value which could be NULL pointer (CastPP #NULL) 2660 // could be delayed during Parse (for example, in adjust_map_after_if()). 2661 // Execute transformation here to avoid barrier generation in such case. 2662 if (_gvn.type(newval) == TypePtr::NULL_PTR) 2663 newval = _gvn.makecon(TypePtr::NULL_PTR); 2664 2665 if (oldval != NULL && _gvn.type(oldval) == TypePtr::NULL_PTR) { 2666 // Refine the value to a null constant, when it is known to be null 2667 oldval = _gvn.makecon(TypePtr::NULL_PTR); 2668 } 2669 } 2670 2671 Node* result = NULL; 2672 switch (kind) { 2673 case LS_cmp_exchange: { 2674 result = access_atomic_cmpxchg_val_at(base, adr, adr_type, alias_idx, 2675 oldval, newval, value_type, type, decorators); 2676 break; 2677 } 2678 case LS_cmp_swap_weak: 2679 decorators |= C2_WEAK_CMPXCHG; 2680 case LS_cmp_swap: { 2681 result = access_atomic_cmpxchg_bool_at(base, adr, adr_type, alias_idx, 2682 oldval, newval, value_type, type, decorators); 2683 break; 2684 } 2685 case LS_get_set: { 2686 result = access_atomic_xchg_at(base, adr, adr_type, alias_idx, 2687 newval, value_type, type, decorators); 2688 break; 2689 } 2690 case LS_get_add: { 2691 result = access_atomic_add_at(base, adr, adr_type, alias_idx, 2692 newval, value_type, type, decorators); 2693 break; 2694 } 2695 default: 2696 ShouldNotReachHere(); 2697 } 2698 2699 assert(type2size[result->bottom_type()->basic_type()] == type2size[rtype], "result type should match"); 2700 set_result(result); 2701 return true; 2702 } 2703 2704 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) { 2705 // Regardless of form, don't allow previous ld/st to move down, 2706 // then issue acquire, release, or volatile mem_bar. 2707 insert_mem_bar(Op_MemBarCPUOrder); 2708 switch(id) { 2709 case vmIntrinsics::_loadFence: 2710 insert_mem_bar(Op_LoadFence); 2711 return true; 2712 case vmIntrinsics::_storeFence: 2713 insert_mem_bar(Op_StoreFence); 2714 return true; 2715 case vmIntrinsics::_storeStoreFence: 2716 insert_mem_bar(Op_StoreStoreFence); 2717 return true; 2718 case vmIntrinsics::_fullFence: 2719 insert_mem_bar(Op_MemBarVolatile); 2720 return true; 2721 default: 2722 fatal_unexpected_iid(id); 2723 return false; 2724 } 2725 } 2726 2727 bool LibraryCallKit::inline_onspinwait() { 2728 insert_mem_bar(Op_OnSpinWait); 2729 return true; 2730 } 2731 2732 bool LibraryCallKit::klass_needs_init_guard(Node* kls) { 2733 if (!kls->is_Con()) { 2734 return true; 2735 } 2736 const TypeInstKlassPtr* klsptr = kls->bottom_type()->isa_instklassptr(); 2737 if (klsptr == NULL) { 2738 return true; 2739 } 2740 ciInstanceKlass* ik = klsptr->instance_klass(); 2741 // don't need a guard for a klass that is already initialized 2742 return !ik->is_initialized(); 2743 } 2744 2745 //----------------------------inline_unsafe_writeback0------------------------- 2746 // public native void Unsafe.writeback0(long address) 2747 bool LibraryCallKit::inline_unsafe_writeback0() { 2748 if (!Matcher::has_match_rule(Op_CacheWB)) { 2749 return false; 2750 } 2751 #ifndef PRODUCT 2752 assert(Matcher::has_match_rule(Op_CacheWBPreSync), "found match rule for CacheWB but not CacheWBPreSync"); 2753 assert(Matcher::has_match_rule(Op_CacheWBPostSync), "found match rule for CacheWB but not CacheWBPostSync"); 2754 ciSignature* sig = callee()->signature(); 2755 assert(sig->type_at(0)->basic_type() == T_LONG, "Unsafe_writeback0 address is long!"); 2756 #endif 2757 null_check_receiver(); // null-check, then ignore 2758 Node *addr = argument(1); 2759 addr = new CastX2PNode(addr); 2760 addr = _gvn.transform(addr); 2761 Node *flush = new CacheWBNode(control(), memory(TypeRawPtr::BOTTOM), addr); 2762 flush = _gvn.transform(flush); 2763 set_memory(flush, TypeRawPtr::BOTTOM); 2764 return true; 2765 } 2766 2767 //----------------------------inline_unsafe_writeback0------------------------- 2768 // public native void Unsafe.writeback0(long address) 2769 bool LibraryCallKit::inline_unsafe_writebackSync0(bool is_pre) { 2770 if (is_pre && !Matcher::has_match_rule(Op_CacheWBPreSync)) { 2771 return false; 2772 } 2773 if (!is_pre && !Matcher::has_match_rule(Op_CacheWBPostSync)) { 2774 return false; 2775 } 2776 #ifndef PRODUCT 2777 assert(Matcher::has_match_rule(Op_CacheWB), 2778 (is_pre ? "found match rule for CacheWBPreSync but not CacheWB" 2779 : "found match rule for CacheWBPostSync but not CacheWB")); 2780 2781 #endif 2782 null_check_receiver(); // null-check, then ignore 2783 Node *sync; 2784 if (is_pre) { 2785 sync = new CacheWBPreSyncNode(control(), memory(TypeRawPtr::BOTTOM)); 2786 } else { 2787 sync = new CacheWBPostSyncNode(control(), memory(TypeRawPtr::BOTTOM)); 2788 } 2789 sync = _gvn.transform(sync); 2790 set_memory(sync, TypeRawPtr::BOTTOM); 2791 return true; 2792 } 2793 2794 //----------------------------inline_unsafe_allocate--------------------------- 2795 // public native Object Unsafe.allocateInstance(Class<?> cls); 2796 bool LibraryCallKit::inline_unsafe_allocate() { 2797 if (callee()->is_static()) return false; // caller must have the capability! 2798 2799 null_check_receiver(); // null-check, then ignore 2800 Node* cls = null_check(argument(1)); 2801 if (stopped()) return true; 2802 2803 Node* kls = load_klass_from_mirror(cls, false, NULL, 0); 2804 kls = null_check(kls); 2805 if (stopped()) return true; // argument was like int.class 2806 2807 Node* test = NULL; 2808 if (LibraryCallKit::klass_needs_init_guard(kls)) { 2809 // Note: The argument might still be an illegal value like 2810 // Serializable.class or Object[].class. The runtime will handle it. 2811 // But we must make an explicit check for initialization. 2812 Node* insp = basic_plus_adr(kls, in_bytes(InstanceKlass::init_state_offset())); 2813 // Use T_BOOLEAN for InstanceKlass::_init_state so the compiler 2814 // can generate code to load it as unsigned byte. 2815 Node* inst = make_load(NULL, insp, TypeInt::UBYTE, T_BOOLEAN, MemNode::unordered); 2816 Node* bits = intcon(InstanceKlass::fully_initialized); 2817 test = _gvn.transform(new SubINode(inst, bits)); 2818 // The 'test' is non-zero if we need to take a slow path. 2819 } 2820 2821 Node* obj = new_instance(kls, test); 2822 set_result(obj); 2823 return true; 2824 } 2825 2826 //------------------------inline_native_time_funcs-------------- 2827 // inline code for System.currentTimeMillis() and System.nanoTime() 2828 // these have the same type and signature 2829 bool LibraryCallKit::inline_native_time_funcs(address funcAddr, const char* funcName) { 2830 const TypeFunc* tf = OptoRuntime::void_long_Type(); 2831 const TypePtr* no_memory_effects = NULL; 2832 Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects); 2833 Node* value = _gvn.transform(new ProjNode(time, TypeFunc::Parms+0)); 2834 #ifdef ASSERT 2835 Node* value_top = _gvn.transform(new ProjNode(time, TypeFunc::Parms+1)); 2836 assert(value_top == top(), "second value must be top"); 2837 #endif 2838 set_result(value); 2839 return true; 2840 } 2841 2842 #ifdef JFR_HAVE_INTRINSICS 2843 2844 /** 2845 * if oop->klass != null 2846 * // normal class 2847 * epoch = _epoch_state ? 2 : 1 2848 * if oop->klass->trace_id & ((epoch << META_SHIFT) | epoch)) != epoch { 2849 * ... // enter slow path when the klass is first recorded or the epoch of JFR shifts 2850 * } 2851 * id = oop->klass->trace_id >> TRACE_ID_SHIFT // normal class path 2852 * else 2853 * // primitive class 2854 * if oop->array_klass != null 2855 * id = (oop->array_klass->trace_id >> TRACE_ID_SHIFT) + 1 // primitive class path 2856 * else 2857 * id = LAST_TYPE_ID + 1 // void class path 2858 * if (!signaled) 2859 * signaled = true 2860 */ 2861 bool LibraryCallKit::inline_native_classID() { 2862 Node* cls = argument(0); 2863 2864 IdealKit ideal(this); 2865 #define __ ideal. 2866 IdealVariable result(ideal); __ declarations_done(); 2867 Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), 2868 basic_plus_adr(cls, java_lang_Class::klass_offset()), 2869 TypeRawPtr::BOTTOM, TypeInstKlassPtr::OBJECT_OR_NULL)); 2870 2871 2872 __ if_then(kls, BoolTest::ne, null()); { 2873 Node* kls_trace_id_addr = basic_plus_adr(kls, in_bytes(KLASS_TRACE_ID_OFFSET)); 2874 Node* kls_trace_id_raw = ideal.load(ideal.ctrl(), kls_trace_id_addr,TypeLong::LONG, T_LONG, Compile::AliasIdxRaw); 2875 2876 Node* epoch_address = makecon(TypeRawPtr::make(JfrIntrinsicSupport::epoch_address())); 2877 Node* epoch = ideal.load(ideal.ctrl(), epoch_address, TypeInt::BOOL, T_BOOLEAN, Compile::AliasIdxRaw); 2878 epoch = _gvn.transform(new LShiftLNode(longcon(1), epoch)); 2879 Node* mask = _gvn.transform(new LShiftLNode(epoch, intcon(META_SHIFT))); 2880 mask = _gvn.transform(new OrLNode(mask, epoch)); 2881 Node* kls_trace_id_raw_and_mask = _gvn.transform(new AndLNode(kls_trace_id_raw, mask)); 2882 2883 float unlikely = PROB_UNLIKELY(0.999); 2884 __ if_then(kls_trace_id_raw_and_mask, BoolTest::ne, epoch, unlikely); { 2885 sync_kit(ideal); 2886 make_runtime_call(RC_LEAF, 2887 OptoRuntime::class_id_load_barrier_Type(), 2888 CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::load_barrier), 2889 "class id load barrier", 2890 TypePtr::BOTTOM, 2891 kls); 2892 ideal.sync_kit(this); 2893 } __ end_if(); 2894 2895 ideal.set(result, _gvn.transform(new URShiftLNode(kls_trace_id_raw, ideal.ConI(TRACE_ID_SHIFT)))); 2896 } __ else_(); { 2897 Node* array_kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), 2898 basic_plus_adr(cls, java_lang_Class::array_klass_offset()), 2899 TypeRawPtr::BOTTOM, TypeInstKlassPtr::OBJECT_OR_NULL)); 2900 __ if_then(array_kls, BoolTest::ne, null()); { 2901 Node* array_kls_trace_id_addr = basic_plus_adr(array_kls, in_bytes(KLASS_TRACE_ID_OFFSET)); 2902 Node* array_kls_trace_id_raw = ideal.load(ideal.ctrl(), array_kls_trace_id_addr, TypeLong::LONG, T_LONG, Compile::AliasIdxRaw); 2903 Node* array_kls_trace_id = _gvn.transform(new URShiftLNode(array_kls_trace_id_raw, ideal.ConI(TRACE_ID_SHIFT))); 2904 ideal.set(result, _gvn.transform(new AddLNode(array_kls_trace_id, longcon(1)))); 2905 } __ else_(); { 2906 // void class case 2907 ideal.set(result, _gvn.transform(longcon(LAST_TYPE_ID + 1))); 2908 } __ end_if(); 2909 2910 Node* signaled_flag_address = makecon(TypeRawPtr::make(JfrIntrinsicSupport::signal_address())); 2911 Node* signaled = ideal.load(ideal.ctrl(), signaled_flag_address, TypeInt::BOOL, T_BOOLEAN, Compile::AliasIdxRaw, true, MemNode::acquire); 2912 __ if_then(signaled, BoolTest::ne, ideal.ConI(1)); { 2913 ideal.store(ideal.ctrl(), signaled_flag_address, ideal.ConI(1), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::release, true); 2914 } __ end_if(); 2915 } __ end_if(); 2916 2917 final_sync(ideal); 2918 set_result(ideal.value(result)); 2919 #undef __ 2920 return true; 2921 } 2922 2923 /* 2924 * The intrinsic is a model of this pseudo-code: 2925 * 2926 * JfrThreadLocal* const tl = Thread::jfr_thread_local() 2927 * jobject h_event_writer = tl->java_event_writer(); 2928 * if (h_event_writer == NULL) { 2929 * return NULL; 2930 * } 2931 * oop threadObj = Thread::threadObj(); 2932 * oop vthread = java_lang_Thread::vthread(threadObj); 2933 * traceid tid; 2934 * bool excluded; 2935 * if (vthread != threadObj) { // i.e. current thread is virtual 2936 * tid = java_lang_Thread::tid(vthread); 2937 * u2 vthread_epoch_raw = java_lang_Thread::jfr_epoch(vthread); 2938 * excluded = vthread_epoch_raw & excluded_mask; 2939 * if (!excluded) { 2940 * traceid current_epoch = JfrTraceIdEpoch::current_generation(); 2941 * u2 vthread_epoch = vthread_epoch_raw & epoch_mask; 2942 * if (vthread_epoch != current_epoch) { 2943 * write_checkpoint(); 2944 * } 2945 * } 2946 * } else { 2947 * tid = java_lang_Thread::tid(threadObj); 2948 * u2 thread_epoch_raw = java_lang_Thread::jfr_epoch(threadObj); 2949 * excluded = thread_epoch_raw & excluded_mask; 2950 * } 2951 * oop event_writer = JNIHandles::resolve_non_null(h_event_writer); 2952 * traceid tid_in_event_writer = getField(event_writer, "threadID"); 2953 * if (tid_in_event_writer != tid) { 2954 * setField(event_writer, "threadID", tid); 2955 * setField(event_writer, "excluded", excluded); 2956 * } 2957 * return event_writer 2958 */ 2959 bool LibraryCallKit::inline_native_getEventWriter() { 2960 enum { _true_path = 1, _false_path = 2, PATH_LIMIT }; 2961 2962 // Save input memory and i_o state. 2963 Node* input_memory_state = reset_memory(); 2964 set_all_memory(input_memory_state); 2965 Node* input_io_state = i_o(); 2966 2967 Node* excluded_mask = _gvn.intcon(32768); 2968 Node* epoch_mask = _gvn.intcon(32767); 2969 2970 // TLS 2971 Node* tls_ptr = _gvn.transform(new ThreadLocalNode()); 2972 2973 // Load the address of java event writer jobject handle from the jfr_thread_local structure. 2974 Node* jobj_ptr = basic_plus_adr(top(), tls_ptr, in_bytes(THREAD_LOCAL_WRITER_OFFSET_JFR)); 2975 2976 // Load the eventwriter jobject handle. 2977 Node* jobj = make_load(control(), jobj_ptr, TypeRawPtr::BOTTOM, T_ADDRESS, MemNode::unordered); 2978 2979 // Null check the jobject handle. 2980 Node* jobj_cmp_null = _gvn.transform(new CmpPNode(jobj, null())); 2981 Node* test_jobj_not_equal_null = _gvn.transform(new BoolNode(jobj_cmp_null, BoolTest::ne)); 2982 IfNode* iff_jobj_not_equal_null = create_and_map_if(control(), test_jobj_not_equal_null, PROB_MAX, COUNT_UNKNOWN); 2983 2984 // False path, jobj is null. 2985 Node* jobj_is_null = _gvn.transform(new IfFalseNode(iff_jobj_not_equal_null)); 2986 2987 // True path, jobj is not null. 2988 Node* jobj_is_not_null = _gvn.transform(new IfTrueNode(iff_jobj_not_equal_null)); 2989 2990 set_control(jobj_is_not_null); 2991 2992 // Load the threadObj for the CarrierThread. 2993 Node* threadObj = generate_current_thread(tls_ptr); 2994 2995 // Load the vthread. 2996 Node* vthread = generate_virtual_thread(tls_ptr); 2997 2998 // If vthread != threadObj, this is a virtual thread. 2999 Node* vthread_cmp_threadObj = _gvn.transform(new CmpPNode(vthread, threadObj)); 3000 Node* test_vthread_not_equal_threadObj = _gvn.transform(new BoolNode(vthread_cmp_threadObj, BoolTest::ne)); 3001 IfNode* iff_vthread_not_equal_threadObj = 3002 create_and_map_if(jobj_is_not_null, test_vthread_not_equal_threadObj, PROB_FAIR, COUNT_UNKNOWN); 3003 3004 // False branch, fallback to threadObj. 3005 Node* vthread_equal_threadObj = _gvn.transform(new IfFalseNode(iff_vthread_not_equal_threadObj)); 3006 set_control(vthread_equal_threadObj); 3007 3008 // Load the tid field from the vthread object. 3009 Node* thread_obj_tid = load_field_from_object(threadObj, "tid", "J"); 3010 3011 // Load the raw epoch value from the threadObj. 3012 Node* threadObj_epoch_offset = basic_plus_adr(threadObj, java_lang_Thread::jfr_epoch_offset()); 3013 Node* threadObj_epoch_raw = access_load_at(threadObj, threadObj_epoch_offset, TypeRawPtr::BOTTOM, TypeInt::CHAR, T_CHAR, 3014 IN_HEAP | MO_UNORDERED | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD); 3015 3016 // Mask off the excluded information from the epoch. 3017 Node * threadObj_is_excluded = _gvn.transform(new AndINode(threadObj_epoch_raw, excluded_mask)); 3018 3019 // True branch, this is a virtual thread. 3020 Node* vthread_not_equal_threadObj = _gvn.transform(new IfTrueNode(iff_vthread_not_equal_threadObj)); 3021 set_control(vthread_not_equal_threadObj); 3022 3023 // Load the tid field from the vthread object. 3024 Node* vthread_tid = load_field_from_object(vthread, "tid", "J"); 3025 3026 // Load the raw epoch value from the vthread. 3027 Node* vthread_epoch_offset = basic_plus_adr(vthread, java_lang_Thread::jfr_epoch_offset()); 3028 Node* vthread_epoch_raw = access_load_at(vthread, vthread_epoch_offset, TypeRawPtr::BOTTOM, TypeInt::CHAR, T_CHAR, 3029 IN_HEAP | MO_UNORDERED | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD); 3030 3031 // Mask off the excluded information from the epoch. 3032 Node * vthread_is_excluded = _gvn.transform(new AndINode(vthread_epoch_raw, _gvn.transform(excluded_mask))); 3033 3034 // Branch on excluded to conditionalize updating the epoch for the virtual thread. 3035 Node* is_excluded_cmp = _gvn.transform(new CmpINode(vthread_is_excluded, _gvn.transform(excluded_mask))); 3036 Node* test_not_excluded = _gvn.transform(new BoolNode(is_excluded_cmp, BoolTest::ne)); 3037 IfNode* iff_not_excluded = create_and_map_if(control(), test_not_excluded, PROB_MAX, COUNT_UNKNOWN); 3038 3039 // False branch, vthread is excluded, no need to write epoch info. 3040 Node* excluded = _gvn.transform(new IfFalseNode(iff_not_excluded)); 3041 3042 // True branch, vthread is included, update epoch info. 3043 Node* included = _gvn.transform(new IfTrueNode(iff_not_excluded)); 3044 set_control(included); 3045 3046 // Get epoch value. 3047 Node* epoch = _gvn.transform(new AndINode(vthread_epoch_raw, _gvn.transform(epoch_mask))); 3048 3049 // Load the current epoch generation. The value is unsigned 16-bit, so we type it as T_CHAR. 3050 Node* epoch_generation_address = makecon(TypeRawPtr::make(JfrIntrinsicSupport::epoch_generation_address())); 3051 Node* current_epoch_generation = make_load(control(), epoch_generation_address, TypeInt::CHAR, T_CHAR, MemNode::unordered); 3052 3053 // Compare the epoch in the vthread to the current epoch generation. 3054 Node* const epoch_cmp = _gvn.transform(new CmpUNode(current_epoch_generation, epoch)); 3055 Node* test_epoch_not_equal = _gvn.transform(new BoolNode(epoch_cmp, BoolTest::ne)); 3056 IfNode* iff_epoch_not_equal = create_and_map_if(control(), test_epoch_not_equal, PROB_FAIR, COUNT_UNKNOWN); 3057 3058 // False path, epoch is equal, checkpoint information is valid. 3059 Node* epoch_is_equal = _gvn.transform(new IfFalseNode(iff_epoch_not_equal)); 3060 3061 // True path, epoch is not equal, write a checkpoint for the vthread. 3062 Node* epoch_is_not_equal = _gvn.transform(new IfTrueNode(iff_epoch_not_equal)); 3063 3064 set_control(epoch_is_not_equal); 3065 3066 // Make a runtime call, which can safepoint, to write a checkpoint for the vthread for this epoch. 3067 // The call also updates the native thread local thread id and the vthread with the current epoch. 3068 Node* call_write_checkpoint = make_runtime_call(RC_NO_LEAF, 3069 OptoRuntime::jfr_write_checkpoint_Type(), 3070 StubRoutines::jfr_write_checkpoint(), 3071 "write_checkpoint", TypePtr::BOTTOM); 3072 Node* call_write_checkpoint_control = _gvn.transform(new ProjNode(call_write_checkpoint, TypeFunc::Control)); 3073 3074 // vthread epoch != current epoch 3075 RegionNode* epoch_compare_rgn = new RegionNode(PATH_LIMIT); 3076 record_for_igvn(epoch_compare_rgn); 3077 PhiNode* epoch_compare_mem = new PhiNode(epoch_compare_rgn, Type::MEMORY, TypePtr::BOTTOM); 3078 record_for_igvn(epoch_compare_mem); 3079 PhiNode* epoch_compare_io = new PhiNode(epoch_compare_rgn, Type::ABIO); 3080 record_for_igvn(epoch_compare_io); 3081 3082 // Update control and phi nodes. 3083 epoch_compare_rgn->init_req(_true_path, call_write_checkpoint_control); 3084 epoch_compare_rgn->init_req(_false_path, epoch_is_equal); 3085 epoch_compare_mem->init_req(_true_path, _gvn.transform(reset_memory())); 3086 epoch_compare_mem->init_req(_false_path, input_memory_state); 3087 epoch_compare_io->init_req(_true_path, i_o()); 3088 epoch_compare_io->init_req(_false_path, input_io_state); 3089 3090 // excluded != true 3091 RegionNode* exclude_compare_rgn = new RegionNode(PATH_LIMIT); 3092 record_for_igvn(exclude_compare_rgn); 3093 PhiNode* exclude_compare_mem = new PhiNode(exclude_compare_rgn, Type::MEMORY, TypePtr::BOTTOM); 3094 record_for_igvn(exclude_compare_mem); 3095 PhiNode* exclude_compare_io = new PhiNode(exclude_compare_rgn, Type::ABIO); 3096 record_for_igvn(exclude_compare_io); 3097 3098 // Update control and phi nodes. 3099 exclude_compare_rgn->init_req(_true_path, _gvn.transform(epoch_compare_rgn)); 3100 exclude_compare_rgn->init_req(_false_path, excluded); 3101 exclude_compare_mem->init_req(_true_path, _gvn.transform(epoch_compare_mem)); 3102 exclude_compare_mem->init_req(_false_path, input_memory_state); 3103 exclude_compare_io->init_req(_true_path, _gvn.transform(epoch_compare_io)); 3104 exclude_compare_io->init_req(_false_path, input_io_state); 3105 3106 // vthread != threadObj 3107 RegionNode* vthread_compare_rgn = new RegionNode(PATH_LIMIT); 3108 record_for_igvn(vthread_compare_rgn); 3109 PhiNode* vthread_compare_mem = new PhiNode(vthread_compare_rgn, Type::MEMORY, TypePtr::BOTTOM); 3110 PhiNode* vthread_compare_io = new PhiNode(vthread_compare_rgn, Type::ABIO); 3111 record_for_igvn(vthread_compare_io); 3112 PhiNode* tid = new PhiNode(vthread_compare_rgn, TypeLong::LONG); 3113 record_for_igvn(tid); 3114 PhiNode* exclusion = new PhiNode(vthread_compare_rgn, TypeInt::BOOL); 3115 record_for_igvn(exclusion); 3116 3117 // Update control and phi nodes. 3118 vthread_compare_rgn->init_req(_true_path, _gvn.transform(exclude_compare_rgn)); 3119 vthread_compare_rgn->init_req(_false_path, vthread_equal_threadObj); 3120 vthread_compare_mem->init_req(_true_path, _gvn.transform(exclude_compare_mem)); 3121 vthread_compare_mem->init_req(_false_path, input_memory_state); 3122 vthread_compare_io->init_req(_true_path, _gvn.transform(exclude_compare_io)); 3123 vthread_compare_io->init_req(_false_path, input_io_state); 3124 tid->init_req(_true_path, _gvn.transform(vthread_tid)); 3125 tid->init_req(_false_path, _gvn.transform(thread_obj_tid)); 3126 exclusion->init_req(_true_path, _gvn.transform(vthread_is_excluded)); 3127 exclusion->init_req(_false_path, _gvn.transform(threadObj_is_excluded)); 3128 3129 // Update branch state. 3130 set_control(_gvn.transform(vthread_compare_rgn)); 3131 set_all_memory(_gvn.transform(vthread_compare_mem)); 3132 set_i_o(_gvn.transform(vthread_compare_io)); 3133 3134 // Load the event writer oop by dereferencing the jobject handle. 3135 ciKlass* klass_EventWriter = env()->find_system_klass(ciSymbol::make("jdk/jfr/internal/event/EventWriter")); 3136 assert(klass_EventWriter->is_loaded(), "invariant"); 3137 ciInstanceKlass* const instklass_EventWriter = klass_EventWriter->as_instance_klass(); 3138 const TypeKlassPtr* const aklass = TypeKlassPtr::make(instklass_EventWriter); 3139 const TypeOopPtr* const xtype = aklass->as_instance_type(); 3140 Node* jobj_untagged = _gvn.transform(new AddPNode(top(), jobj, _gvn.MakeConX(-JNIHandles::TypeTag::global))); 3141 Node* event_writer = access_load(jobj_untagged, xtype, T_OBJECT, IN_NATIVE | C2_CONTROL_DEPENDENT_LOAD); 3142 3143 // Load the current thread id from the event writer object. 3144 Node* const event_writer_tid = load_field_from_object(event_writer, "threadID", "J"); 3145 // Get the field offset to, conditionally, store an updated tid value later. 3146 Node* const event_writer_tid_field = field_address_from_object(event_writer, "threadID", "J", false); 3147 const TypePtr* event_writer_tid_field_type = _gvn.type(event_writer_tid_field)->isa_ptr(); 3148 // Get the field offset to, conditionally, store an updated exclusion value later. 3149 Node* const event_writer_excluded_field = field_address_from_object(event_writer, "excluded", "Z", false); 3150 const TypePtr* event_writer_excluded_field_type = _gvn.type(event_writer_excluded_field)->isa_ptr(); 3151 3152 RegionNode* event_writer_tid_compare_rgn = new RegionNode(PATH_LIMIT); 3153 record_for_igvn(event_writer_tid_compare_rgn); 3154 PhiNode* event_writer_tid_compare_mem = new PhiNode(event_writer_tid_compare_rgn, Type::MEMORY, TypePtr::BOTTOM); 3155 record_for_igvn(event_writer_tid_compare_mem); 3156 PhiNode* event_writer_tid_compare_io = new PhiNode(event_writer_tid_compare_rgn, Type::ABIO); 3157 record_for_igvn(event_writer_tid_compare_io); 3158 3159 // Compare the current tid from the thread object to what is currently stored in the event writer object. 3160 Node* const tid_cmp = _gvn.transform(new CmpLNode(event_writer_tid, _gvn.transform(tid))); 3161 Node* test_tid_not_equal = _gvn.transform(new BoolNode(tid_cmp, BoolTest::ne)); 3162 IfNode* iff_tid_not_equal = create_and_map_if(_gvn.transform(vthread_compare_rgn), test_tid_not_equal, PROB_FAIR, COUNT_UNKNOWN); 3163 3164 // False path, tids are the same. 3165 Node* tid_is_equal = _gvn.transform(new IfFalseNode(iff_tid_not_equal)); 3166 3167 // True path, tid is not equal, need to update the tid in the event writer. 3168 Node* tid_is_not_equal = _gvn.transform(new IfTrueNode(iff_tid_not_equal)); 3169 record_for_igvn(tid_is_not_equal); 3170 3171 // Store the exclusion state to the event writer. 3172 store_to_memory(tid_is_not_equal, event_writer_excluded_field, _gvn.transform(exclusion), T_BOOLEAN, event_writer_excluded_field_type, MemNode::unordered); 3173 3174 // Store the tid to the event writer. 3175 store_to_memory(tid_is_not_equal, event_writer_tid_field, tid, T_LONG, event_writer_tid_field_type, MemNode::unordered); 3176 3177 // Update control and phi nodes. 3178 event_writer_tid_compare_rgn->init_req(_true_path, tid_is_not_equal); 3179 event_writer_tid_compare_rgn->init_req(_false_path, tid_is_equal); 3180 event_writer_tid_compare_mem->init_req(_true_path, _gvn.transform(reset_memory())); 3181 event_writer_tid_compare_mem->init_req(_false_path, _gvn.transform(vthread_compare_mem)); 3182 event_writer_tid_compare_io->init_req(_true_path, _gvn.transform(i_o())); 3183 event_writer_tid_compare_io->init_req(_false_path, _gvn.transform(vthread_compare_io)); 3184 3185 // Result of top level CFG, Memory, IO and Value. 3186 RegionNode* result_rgn = new RegionNode(PATH_LIMIT); 3187 PhiNode* result_mem = new PhiNode(result_rgn, Type::MEMORY, TypePtr::BOTTOM); 3188 PhiNode* result_io = new PhiNode(result_rgn, Type::ABIO); 3189 PhiNode* result_value = new PhiNode(result_rgn, TypeInstPtr::BOTTOM); 3190 3191 // Result control. 3192 result_rgn->init_req(_true_path, _gvn.transform(event_writer_tid_compare_rgn)); 3193 result_rgn->init_req(_false_path, jobj_is_null); 3194 3195 // Result memory. 3196 result_mem->init_req(_true_path, _gvn.transform(event_writer_tid_compare_mem)); 3197 result_mem->init_req(_false_path, _gvn.transform(input_memory_state)); 3198 3199 // Result IO. 3200 result_io->init_req(_true_path, _gvn.transform(event_writer_tid_compare_io)); 3201 result_io->init_req(_false_path, _gvn.transform(input_io_state)); 3202 3203 // Result value. 3204 result_value->init_req(_true_path, _gvn.transform(event_writer)); // return event writer oop 3205 result_value->init_req(_false_path, null()); // return NULL 3206 3207 // Set output state. 3208 set_control(_gvn.transform(result_rgn)); 3209 set_all_memory(_gvn.transform(result_mem)); 3210 set_i_o(_gvn.transform(result_io)); 3211 set_result(result_rgn, result_value); 3212 return true; 3213 } 3214 3215 /* 3216 * The intrinsic is a model of this pseudo-code: 3217 * 3218 * JfrThreadLocal* const tl = thread->jfr_thread_local(); 3219 * if (carrierThread != thread) { // is virtual thread 3220 * const u2 vthread_epoch_raw = java_lang_Thread::jfr_epoch(thread); 3221 * bool excluded = vthread_epoch_raw & excluded_mask; 3222 * Atomic::store(&tl->_contextual_tid, java_lang_Thread::tid(thread)); 3223 * Atomic::store(&tl->_contextual_thread_excluded, is_excluded); 3224 * if (!excluded) { 3225 * const u2 vthread_epoch = vthread_epoch_raw & epoch_mask; 3226 * Atomic::store(&tl->_vthread_epoch, vthread_epoch); 3227 * } 3228 * Atomic::release_store(&tl->_vthread, true); 3229 * return; 3230 * } 3231 * Atomic::release_store(&tl->_vthread, false); 3232 */ 3233 void LibraryCallKit::extend_setCurrentThread(Node* jt, Node* thread) { 3234 enum { _true_path = 1, _false_path = 2, PATH_LIMIT }; 3235 3236 Node* input_memory_state = reset_memory(); 3237 set_all_memory(input_memory_state); 3238 3239 Node* excluded_mask = _gvn.intcon(32768); 3240 Node* epoch_mask = _gvn.intcon(32767); 3241 3242 Node* const carrierThread = generate_current_thread(jt); 3243 // If thread != carrierThread, this is a virtual thread. 3244 Node* thread_cmp_carrierThread = _gvn.transform(new CmpPNode(thread, carrierThread)); 3245 Node* test_thread_not_equal_carrierThread = _gvn.transform(new BoolNode(thread_cmp_carrierThread, BoolTest::ne)); 3246 IfNode* iff_thread_not_equal_carrierThread = 3247 create_and_map_if(control(), test_thread_not_equal_carrierThread, PROB_FAIR, COUNT_UNKNOWN); 3248 3249 Node* vthread_offset = basic_plus_adr(jt, in_bytes(THREAD_LOCAL_OFFSET_JFR + VTHREAD_OFFSET_JFR)); 3250 3251 // False branch, is carrierThread. 3252 Node* thread_equal_carrierThread = _gvn.transform(new IfFalseNode(iff_thread_not_equal_carrierThread)); 3253 // Store release 3254 Node* vthread_false_memory = store_to_memory(thread_equal_carrierThread, vthread_offset, _gvn.intcon(0), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::release, true); 3255 3256 set_all_memory(input_memory_state); 3257 3258 // True branch, is virtual thread. 3259 Node* thread_not_equal_carrierThread = _gvn.transform(new IfTrueNode(iff_thread_not_equal_carrierThread)); 3260 set_control(thread_not_equal_carrierThread); 3261 3262 // Load the raw epoch value from the vthread. 3263 Node* epoch_offset = basic_plus_adr(thread, java_lang_Thread::jfr_epoch_offset()); 3264 Node* epoch_raw = access_load_at(thread, epoch_offset, TypeRawPtr::BOTTOM, TypeInt::CHAR, T_CHAR, 3265 IN_HEAP | MO_UNORDERED | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD); 3266 3267 // Mask off the excluded information from the epoch. 3268 Node * const is_excluded = _gvn.transform(new AndINode(epoch_raw, _gvn.transform(excluded_mask))); 3269 3270 // Load the tid field from the thread. 3271 Node* tid = load_field_from_object(thread, "tid", "J"); 3272 3273 // Store the vthread tid to the jfr thread local. 3274 Node* thread_id_offset = basic_plus_adr(jt, in_bytes(THREAD_LOCAL_OFFSET_JFR + VTHREAD_ID_OFFSET_JFR)); 3275 Node* tid_memory = store_to_memory(control(), thread_id_offset, tid, T_LONG, Compile::AliasIdxRaw, MemNode::unordered, true); 3276 3277 // Branch is_excluded to conditionalize updating the epoch . 3278 Node* excluded_cmp = _gvn.transform(new CmpINode(is_excluded, _gvn.transform(excluded_mask))); 3279 Node* test_excluded = _gvn.transform(new BoolNode(excluded_cmp, BoolTest::eq)); 3280 IfNode* iff_excluded = create_and_map_if(control(), test_excluded, PROB_MIN, COUNT_UNKNOWN); 3281 3282 // True branch, vthread is excluded, no need to write epoch info. 3283 Node* excluded = _gvn.transform(new IfTrueNode(iff_excluded)); 3284 set_control(excluded); 3285 Node* vthread_is_excluded = _gvn.intcon(1); 3286 3287 // False branch, vthread is included, update epoch info. 3288 Node* included = _gvn.transform(new IfFalseNode(iff_excluded)); 3289 set_control(included); 3290 Node* vthread_is_included = _gvn.intcon(0); 3291 3292 // Get epoch value. 3293 Node* epoch = _gvn.transform(new AndINode(epoch_raw, _gvn.transform(epoch_mask))); 3294 3295 // Store the vthread epoch to the jfr thread local. 3296 Node* vthread_epoch_offset = basic_plus_adr(jt, in_bytes(THREAD_LOCAL_OFFSET_JFR + VTHREAD_EPOCH_OFFSET_JFR)); 3297 Node* included_memory = store_to_memory(control(), vthread_epoch_offset, epoch, T_CHAR, Compile::AliasIdxRaw, MemNode::unordered, true); 3298 3299 RegionNode* excluded_rgn = new RegionNode(PATH_LIMIT); 3300 record_for_igvn(excluded_rgn); 3301 PhiNode* excluded_mem = new PhiNode(excluded_rgn, Type::MEMORY, TypePtr::BOTTOM); 3302 record_for_igvn(excluded_mem); 3303 PhiNode* exclusion = new PhiNode(excluded_rgn, TypeInt::BOOL); 3304 record_for_igvn(exclusion); 3305 3306 // Merge the excluded control and memory. 3307 excluded_rgn->init_req(_true_path, excluded); 3308 excluded_rgn->init_req(_false_path, included); 3309 excluded_mem->init_req(_true_path, tid_memory); 3310 excluded_mem->init_req(_false_path, included_memory); 3311 exclusion->init_req(_true_path, _gvn.transform(vthread_is_excluded)); 3312 exclusion->init_req(_false_path, _gvn.transform(vthread_is_included)); 3313 3314 // Set intermediate state. 3315 set_control(_gvn.transform(excluded_rgn)); 3316 set_all_memory(excluded_mem); 3317 3318 // Store the vthread exclusion state to the jfr thread local. 3319 Node* thread_local_excluded_offset = basic_plus_adr(jt, in_bytes(THREAD_LOCAL_OFFSET_JFR + VTHREAD_EXCLUDED_OFFSET_JFR)); 3320 store_to_memory(control(), thread_local_excluded_offset, _gvn.transform(exclusion), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::unordered, true); 3321 3322 // Store release 3323 Node * vthread_true_memory = store_to_memory(control(), vthread_offset, _gvn.intcon(1), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::release, true); 3324 3325 RegionNode* thread_compare_rgn = new RegionNode(PATH_LIMIT); 3326 record_for_igvn(thread_compare_rgn); 3327 PhiNode* thread_compare_mem = new PhiNode(thread_compare_rgn, Type::MEMORY, TypePtr::BOTTOM); 3328 record_for_igvn(thread_compare_mem); 3329 PhiNode* vthread = new PhiNode(thread_compare_rgn, TypeInt::BOOL); 3330 record_for_igvn(vthread); 3331 3332 // Merge the thread_compare control and memory. 3333 thread_compare_rgn->init_req(_true_path, control()); 3334 thread_compare_rgn->init_req(_false_path, thread_equal_carrierThread); 3335 thread_compare_mem->init_req(_true_path, vthread_true_memory); 3336 thread_compare_mem->init_req(_false_path, vthread_false_memory); 3337 3338 // Set output state. 3339 set_control(_gvn.transform(thread_compare_rgn)); 3340 set_all_memory(_gvn.transform(thread_compare_mem)); 3341 } 3342 3343 #endif // JFR_HAVE_INTRINSICS 3344 3345 //------------------------inline_native_currentCarrierThread------------------ 3346 bool LibraryCallKit::inline_native_currentCarrierThread() { 3347 Node* junk = NULL; 3348 set_result(generate_current_thread(junk)); 3349 return true; 3350 } 3351 3352 //------------------------inline_native_currentThread------------------ 3353 bool LibraryCallKit::inline_native_currentThread() { 3354 Node* junk = NULL; 3355 set_result(generate_virtual_thread(junk)); 3356 return true; 3357 } 3358 3359 //------------------------inline_native_setVthread------------------ 3360 bool LibraryCallKit::inline_native_setCurrentThread() { 3361 assert(C->method()->changes_current_thread(), 3362 "method changes current Thread but is not annotated ChangesCurrentThread"); 3363 Node* arr = argument(1); 3364 Node* thread = _gvn.transform(new ThreadLocalNode()); 3365 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::vthread_offset())); 3366 Node* thread_obj_handle 3367 = make_load(NULL, p, p->bottom_type()->is_ptr(), T_OBJECT, MemNode::unordered); 3368 thread_obj_handle = _gvn.transform(thread_obj_handle); 3369 const TypePtr *adr_type = _gvn.type(thread_obj_handle)->isa_ptr(); 3370 access_store_at(NULL, thread_obj_handle, adr_type, arr, _gvn.type(arr), T_OBJECT, IN_NATIVE | MO_UNORDERED); 3371 JFR_ONLY(extend_setCurrentThread(thread, arr);) 3372 return true; 3373 } 3374 3375 Node* LibraryCallKit::scopedValueCache_helper() { 3376 ciKlass *objects_klass = ciObjArrayKlass::make(env()->Object_klass()); 3377 const TypeOopPtr *etype = TypeOopPtr::make_from_klass(env()->Object_klass()); 3378 3379 bool xk = etype->klass_is_exact(); 3380 3381 Node* thread = _gvn.transform(new ThreadLocalNode()); 3382 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::scopedValueCache_offset())); 3383 // We cannot use immutable_memory() because we might flip onto a 3384 // different carrier thread, at which point we'll need to use that 3385 // carrier thread's cache. 3386 // return _gvn.transform(LoadNode::make(_gvn, NULL, immutable_memory(), p, p->bottom_type()->is_ptr(), 3387 // TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered)); 3388 return make_load(NULL, p, p->bottom_type()->is_ptr(), T_ADDRESS, MemNode::unordered); 3389 } 3390 3391 //------------------------inline_native_scopedValueCache------------------ 3392 bool LibraryCallKit::inline_native_scopedValueCache() { 3393 ciKlass *objects_klass = ciObjArrayKlass::make(env()->Object_klass()); 3394 const TypeOopPtr *etype = TypeOopPtr::make_from_klass(env()->Object_klass()); 3395 const TypeAry* arr0 = TypeAry::make(etype, TypeInt::POS); 3396 3397 // Because we create the scopedValue cache lazily we have to make the 3398 // type of the result BotPTR. 3399 bool xk = etype->klass_is_exact(); 3400 const Type* objects_type = TypeAryPtr::make(TypePtr::BotPTR, arr0, objects_klass, xk, 0); 3401 Node* cache_obj_handle = scopedValueCache_helper(); 3402 set_result(access_load(cache_obj_handle, objects_type, T_OBJECT, IN_NATIVE)); 3403 3404 return true; 3405 } 3406 3407 //------------------------inline_native_setScopedValueCache------------------ 3408 bool LibraryCallKit::inline_native_setScopedValueCache() { 3409 Node* arr = argument(0); 3410 Node* cache_obj_handle = scopedValueCache_helper(); 3411 3412 const TypePtr *adr_type = _gvn.type(cache_obj_handle)->isa_ptr(); 3413 store_to_memory(control(), cache_obj_handle, arr, T_OBJECT, adr_type, 3414 MemNode::unordered); 3415 3416 return true; 3417 } 3418 3419 //---------------------------load_mirror_from_klass---------------------------- 3420 // Given a klass oop, load its java mirror (a java.lang.Class oop). 3421 Node* LibraryCallKit::load_mirror_from_klass(Node* klass) { 3422 Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset())); 3423 Node* load = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered); 3424 // mirror = ((OopHandle)mirror)->resolve(); 3425 return access_load(load, TypeInstPtr::MIRROR, T_OBJECT, IN_NATIVE); 3426 } 3427 3428 //-----------------------load_klass_from_mirror_common------------------------- 3429 // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop. 3430 // Test the klass oop for null (signifying a primitive Class like Integer.TYPE), 3431 // and branch to the given path on the region. 3432 // If never_see_null, take an uncommon trap on null, so we can optimistically 3433 // compile for the non-null case. 3434 // If the region is NULL, force never_see_null = true. 3435 Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror, 3436 bool never_see_null, 3437 RegionNode* region, 3438 int null_path, 3439 int offset) { 3440 if (region == NULL) never_see_null = true; 3441 Node* p = basic_plus_adr(mirror, offset); 3442 const TypeKlassPtr* kls_type = TypeInstKlassPtr::OBJECT_OR_NULL; 3443 Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type)); 3444 Node* null_ctl = top(); 3445 kls = null_check_oop(kls, &null_ctl, never_see_null); 3446 if (region != NULL) { 3447 // Set region->in(null_path) if the mirror is a primitive (e.g, int.class). 3448 region->init_req(null_path, null_ctl); 3449 } else { 3450 assert(null_ctl == top(), "no loose ends"); 3451 } 3452 return kls; 3453 } 3454 3455 //--------------------(inline_native_Class_query helpers)--------------------- 3456 // Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE_FAST, JVM_ACC_HAS_FINALIZER. 3457 // Fall through if (mods & mask) == bits, take the guard otherwise. 3458 Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) { 3459 // Branch around if the given klass has the given modifier bit set. 3460 // Like generate_guard, adds a new path onto the region. 3461 Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset())); 3462 Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT, MemNode::unordered); 3463 Node* mask = intcon(modifier_mask); 3464 Node* bits = intcon(modifier_bits); 3465 Node* mbit = _gvn.transform(new AndINode(mods, mask)); 3466 Node* cmp = _gvn.transform(new CmpINode(mbit, bits)); 3467 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne)); 3468 return generate_fair_guard(bol, region); 3469 } 3470 Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) { 3471 return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region); 3472 } 3473 Node* LibraryCallKit::generate_hidden_class_guard(Node* kls, RegionNode* region) { 3474 return generate_access_flags_guard(kls, JVM_ACC_IS_HIDDEN_CLASS, 0, region); 3475 } 3476 3477 //-------------------------inline_native_Class_query------------------- 3478 bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) { 3479 const Type* return_type = TypeInt::BOOL; 3480 Node* prim_return_value = top(); // what happens if it's a primitive class? 3481 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); 3482 bool expect_prim = false; // most of these guys expect to work on refs 3483 3484 enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT }; 3485 3486 Node* mirror = argument(0); 3487 Node* obj = top(); 3488 3489 switch (id) { 3490 case vmIntrinsics::_isInstance: 3491 // nothing is an instance of a primitive type 3492 prim_return_value = intcon(0); 3493 obj = argument(1); 3494 break; 3495 case vmIntrinsics::_getModifiers: 3496 prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC); 3497 assert(is_power_of_2((int)JVM_ACC_WRITTEN_FLAGS+1), "change next line"); 3498 return_type = TypeInt::make(0, JVM_ACC_WRITTEN_FLAGS, Type::WidenMin); 3499 break; 3500 case vmIntrinsics::_isInterface: 3501 prim_return_value = intcon(0); 3502 break; 3503 case vmIntrinsics::_isArray: 3504 prim_return_value = intcon(0); 3505 expect_prim = true; // cf. ObjectStreamClass.getClassSignature 3506 break; 3507 case vmIntrinsics::_isPrimitive: 3508 prim_return_value = intcon(1); 3509 expect_prim = true; // obviously 3510 break; 3511 case vmIntrinsics::_isHidden: 3512 prim_return_value = intcon(0); 3513 break; 3514 case vmIntrinsics::_getSuperclass: 3515 prim_return_value = null(); 3516 return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR); 3517 break; 3518 case vmIntrinsics::_getClassAccessFlags: 3519 prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC); 3520 return_type = TypeInt::INT; // not bool! 6297094 3521 break; 3522 default: 3523 fatal_unexpected_iid(id); 3524 break; 3525 } 3526 3527 const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr(); 3528 if (mirror_con == NULL) return false; // cannot happen? 3529 3530 #ifndef PRODUCT 3531 if (C->print_intrinsics() || C->print_inlining()) { 3532 ciType* k = mirror_con->java_mirror_type(); 3533 if (k) { 3534 tty->print("Inlining %s on constant Class ", vmIntrinsics::name_at(intrinsic_id())); 3535 k->print_name(); 3536 tty->cr(); 3537 } 3538 } 3539 #endif 3540 3541 // Null-check the mirror, and the mirror's klass ptr (in case it is a primitive). 3542 RegionNode* region = new RegionNode(PATH_LIMIT); 3543 record_for_igvn(region); 3544 PhiNode* phi = new PhiNode(region, return_type); 3545 3546 // The mirror will never be null of Reflection.getClassAccessFlags, however 3547 // it may be null for Class.isInstance or Class.getModifiers. Throw a NPE 3548 // if it is. See bug 4774291. 3549 3550 // For Reflection.getClassAccessFlags(), the null check occurs in 3551 // the wrong place; see inline_unsafe_access(), above, for a similar 3552 // situation. 3553 mirror = null_check(mirror); 3554 // If mirror or obj is dead, only null-path is taken. 3555 if (stopped()) return true; 3556 3557 if (expect_prim) never_see_null = false; // expect nulls (meaning prims) 3558 3559 // Now load the mirror's klass metaobject, and null-check it. 3560 // Side-effects region with the control path if the klass is null. 3561 Node* kls = load_klass_from_mirror(mirror, never_see_null, region, _prim_path); 3562 // If kls is null, we have a primitive mirror. 3563 phi->init_req(_prim_path, prim_return_value); 3564 if (stopped()) { set_result(region, phi); return true; } 3565 bool safe_for_replace = (region->in(_prim_path) == top()); 3566 3567 Node* p; // handy temp 3568 Node* null_ctl; 3569 3570 // Now that we have the non-null klass, we can perform the real query. 3571 // For constant classes, the query will constant-fold in LoadNode::Value. 3572 Node* query_value = top(); 3573 switch (id) { 3574 case vmIntrinsics::_isInstance: 3575 // nothing is an instance of a primitive type 3576 query_value = gen_instanceof(obj, kls, safe_for_replace); 3577 break; 3578 3579 case vmIntrinsics::_getModifiers: 3580 p = basic_plus_adr(kls, in_bytes(Klass::modifier_flags_offset())); 3581 query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered); 3582 break; 3583 3584 case vmIntrinsics::_isInterface: 3585 // (To verify this code sequence, check the asserts in JVM_IsInterface.) 3586 if (generate_interface_guard(kls, region) != NULL) 3587 // A guard was added. If the guard is taken, it was an interface. 3588 phi->add_req(intcon(1)); 3589 // If we fall through, it's a plain class. 3590 query_value = intcon(0); 3591 break; 3592 3593 case vmIntrinsics::_isArray: 3594 // (To verify this code sequence, check the asserts in JVM_IsArrayClass.) 3595 if (generate_array_guard(kls, region) != NULL) 3596 // A guard was added. If the guard is taken, it was an array. 3597 phi->add_req(intcon(1)); 3598 // If we fall through, it's a plain class. 3599 query_value = intcon(0); 3600 break; 3601 3602 case vmIntrinsics::_isPrimitive: 3603 query_value = intcon(0); // "normal" path produces false 3604 break; 3605 3606 case vmIntrinsics::_isHidden: 3607 // (To verify this code sequence, check the asserts in JVM_IsHiddenClass.) 3608 if (generate_hidden_class_guard(kls, region) != NULL) 3609 // A guard was added. If the guard is taken, it was an hidden class. 3610 phi->add_req(intcon(1)); 3611 // If we fall through, it's a plain class. 3612 query_value = intcon(0); 3613 break; 3614 3615 3616 case vmIntrinsics::_getSuperclass: 3617 // The rules here are somewhat unfortunate, but we can still do better 3618 // with random logic than with a JNI call. 3619 // Interfaces store null or Object as _super, but must report null. 3620 // Arrays store an intermediate super as _super, but must report Object. 3621 // Other types can report the actual _super. 3622 // (To verify this code sequence, check the asserts in JVM_IsInterface.) 3623 if (generate_interface_guard(kls, region) != NULL) 3624 // A guard was added. If the guard is taken, it was an interface. 3625 phi->add_req(null()); 3626 if (generate_array_guard(kls, region) != NULL) 3627 // A guard was added. If the guard is taken, it was an array. 3628 phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror()))); 3629 // If we fall through, it's a plain class. Get its _super. 3630 p = basic_plus_adr(kls, in_bytes(Klass::super_offset())); 3631 kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeInstKlassPtr::OBJECT_OR_NULL)); 3632 null_ctl = top(); 3633 kls = null_check_oop(kls, &null_ctl); 3634 if (null_ctl != top()) { 3635 // If the guard is taken, Object.superClass is null (both klass and mirror). 3636 region->add_req(null_ctl); 3637 phi ->add_req(null()); 3638 } 3639 if (!stopped()) { 3640 query_value = load_mirror_from_klass(kls); 3641 } 3642 break; 3643 3644 case vmIntrinsics::_getClassAccessFlags: 3645 p = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset())); 3646 query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered); 3647 break; 3648 3649 default: 3650 fatal_unexpected_iid(id); 3651 break; 3652 } 3653 3654 // Fall-through is the normal case of a query to a real class. 3655 phi->init_req(1, query_value); 3656 region->init_req(1, control()); 3657 3658 C->set_has_split_ifs(true); // Has chance for split-if optimization 3659 set_result(region, phi); 3660 return true; 3661 } 3662 3663 //-------------------------inline_Class_cast------------------- 3664 bool LibraryCallKit::inline_Class_cast() { 3665 Node* mirror = argument(0); // Class 3666 Node* obj = argument(1); 3667 const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr(); 3668 if (mirror_con == NULL) { 3669 return false; // dead path (mirror->is_top()). 3670 } 3671 if (obj == NULL || obj->is_top()) { 3672 return false; // dead path 3673 } 3674 const TypeOopPtr* tp = _gvn.type(obj)->isa_oopptr(); 3675 3676 // First, see if Class.cast() can be folded statically. 3677 // java_mirror_type() returns non-null for compile-time Class constants. 3678 ciType* tm = mirror_con->java_mirror_type(); 3679 if (tm != NULL && tm->is_klass() && 3680 tp != NULL) { 3681 if (!tp->is_loaded()) { 3682 // Don't use intrinsic when class is not loaded. 3683 return false; 3684 } else { 3685 int static_res = C->static_subtype_check(TypeKlassPtr::make(tm->as_klass(), Type::trust_interfaces), tp->as_klass_type()); 3686 if (static_res == Compile::SSC_always_true) { 3687 // isInstance() is true - fold the code. 3688 set_result(obj); 3689 return true; 3690 } else if (static_res == Compile::SSC_always_false) { 3691 // Don't use intrinsic, have to throw ClassCastException. 3692 // If the reference is null, the non-intrinsic bytecode will 3693 // be optimized appropriately. 3694 return false; 3695 } 3696 } 3697 } 3698 3699 // Bailout intrinsic and do normal inlining if exception path is frequent. 3700 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 3701 return false; 3702 } 3703 3704 // Generate dynamic checks. 3705 // Class.cast() is java implementation of _checkcast bytecode. 3706 // Do checkcast (Parse::do_checkcast()) optimizations here. 3707 3708 mirror = null_check(mirror); 3709 // If mirror is dead, only null-path is taken. 3710 if (stopped()) { 3711 return true; 3712 } 3713 3714 // Not-subtype or the mirror's klass ptr is NULL (in case it is a primitive). 3715 enum { _bad_type_path = 1, _prim_path = 2, PATH_LIMIT }; 3716 RegionNode* region = new RegionNode(PATH_LIMIT); 3717 record_for_igvn(region); 3718 3719 // Now load the mirror's klass metaobject, and null-check it. 3720 // If kls is null, we have a primitive mirror and 3721 // nothing is an instance of a primitive type. 3722 Node* kls = load_klass_from_mirror(mirror, false, region, _prim_path); 3723 3724 Node* res = top(); 3725 if (!stopped()) { 3726 Node* bad_type_ctrl = top(); 3727 // Do checkcast optimizations. 3728 res = gen_checkcast(obj, kls, &bad_type_ctrl); 3729 region->init_req(_bad_type_path, bad_type_ctrl); 3730 } 3731 if (region->in(_prim_path) != top() || 3732 region->in(_bad_type_path) != top()) { 3733 // Let Interpreter throw ClassCastException. 3734 PreserveJVMState pjvms(this); 3735 set_control(_gvn.transform(region)); 3736 uncommon_trap(Deoptimization::Reason_intrinsic, 3737 Deoptimization::Action_maybe_recompile); 3738 } 3739 if (!stopped()) { 3740 set_result(res); 3741 } 3742 return true; 3743 } 3744 3745 3746 //--------------------------inline_native_subtype_check------------------------ 3747 // This intrinsic takes the JNI calls out of the heart of 3748 // UnsafeFieldAccessorImpl.set, which improves Field.set, readObject, etc. 3749 bool LibraryCallKit::inline_native_subtype_check() { 3750 // Pull both arguments off the stack. 3751 Node* args[2]; // two java.lang.Class mirrors: superc, subc 3752 args[0] = argument(0); 3753 args[1] = argument(1); 3754 Node* klasses[2]; // corresponding Klasses: superk, subk 3755 klasses[0] = klasses[1] = top(); 3756 3757 enum { 3758 // A full decision tree on {superc is prim, subc is prim}: 3759 _prim_0_path = 1, // {P,N} => false 3760 // {P,P} & superc!=subc => false 3761 _prim_same_path, // {P,P} & superc==subc => true 3762 _prim_1_path, // {N,P} => false 3763 _ref_subtype_path, // {N,N} & subtype check wins => true 3764 _both_ref_path, // {N,N} & subtype check loses => false 3765 PATH_LIMIT 3766 }; 3767 3768 RegionNode* region = new RegionNode(PATH_LIMIT); 3769 Node* phi = new PhiNode(region, TypeInt::BOOL); 3770 record_for_igvn(region); 3771 3772 const TypePtr* adr_type = TypeRawPtr::BOTTOM; // memory type of loads 3773 const TypeKlassPtr* kls_type = TypeInstKlassPtr::OBJECT_OR_NULL; 3774 int class_klass_offset = java_lang_Class::klass_offset(); 3775 3776 // First null-check both mirrors and load each mirror's klass metaobject. 3777 int which_arg; 3778 for (which_arg = 0; which_arg <= 1; which_arg++) { 3779 Node* arg = args[which_arg]; 3780 arg = null_check(arg); 3781 if (stopped()) break; 3782 args[which_arg] = arg; 3783 3784 Node* p = basic_plus_adr(arg, class_klass_offset); 3785 Node* kls = LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, adr_type, kls_type); 3786 klasses[which_arg] = _gvn.transform(kls); 3787 } 3788 3789 // Having loaded both klasses, test each for null. 3790 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); 3791 for (which_arg = 0; which_arg <= 1; which_arg++) { 3792 Node* kls = klasses[which_arg]; 3793 Node* null_ctl = top(); 3794 kls = null_check_oop(kls, &null_ctl, never_see_null); 3795 int prim_path = (which_arg == 0 ? _prim_0_path : _prim_1_path); 3796 region->init_req(prim_path, null_ctl); 3797 if (stopped()) break; 3798 klasses[which_arg] = kls; 3799 } 3800 3801 if (!stopped()) { 3802 // now we have two reference types, in klasses[0..1] 3803 Node* subk = klasses[1]; // the argument to isAssignableFrom 3804 Node* superk = klasses[0]; // the receiver 3805 region->set_req(_both_ref_path, gen_subtype_check(subk, superk)); 3806 // now we have a successful reference subtype check 3807 region->set_req(_ref_subtype_path, control()); 3808 } 3809 3810 // If both operands are primitive (both klasses null), then 3811 // we must return true when they are identical primitives. 3812 // It is convenient to test this after the first null klass check. 3813 set_control(region->in(_prim_0_path)); // go back to first null check 3814 if (!stopped()) { 3815 // Since superc is primitive, make a guard for the superc==subc case. 3816 Node* cmp_eq = _gvn.transform(new CmpPNode(args[0], args[1])); 3817 Node* bol_eq = _gvn.transform(new BoolNode(cmp_eq, BoolTest::eq)); 3818 generate_guard(bol_eq, region, PROB_FAIR); 3819 if (region->req() == PATH_LIMIT+1) { 3820 // A guard was added. If the added guard is taken, superc==subc. 3821 region->swap_edges(PATH_LIMIT, _prim_same_path); 3822 region->del_req(PATH_LIMIT); 3823 } 3824 region->set_req(_prim_0_path, control()); // Not equal after all. 3825 } 3826 3827 // these are the only paths that produce 'true': 3828 phi->set_req(_prim_same_path, intcon(1)); 3829 phi->set_req(_ref_subtype_path, intcon(1)); 3830 3831 // pull together the cases: 3832 assert(region->req() == PATH_LIMIT, "sane region"); 3833 for (uint i = 1; i < region->req(); i++) { 3834 Node* ctl = region->in(i); 3835 if (ctl == NULL || ctl == top()) { 3836 region->set_req(i, top()); 3837 phi ->set_req(i, top()); 3838 } else if (phi->in(i) == NULL) { 3839 phi->set_req(i, intcon(0)); // all other paths produce 'false' 3840 } 3841 } 3842 3843 set_control(_gvn.transform(region)); 3844 set_result(_gvn.transform(phi)); 3845 return true; 3846 } 3847 3848 //---------------------generate_array_guard_common------------------------ 3849 Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region, 3850 bool obj_array, bool not_array) { 3851 3852 if (stopped()) { 3853 return NULL; 3854 } 3855 3856 // If obj_array/non_array==false/false: 3857 // Branch around if the given klass is in fact an array (either obj or prim). 3858 // If obj_array/non_array==false/true: 3859 // Branch around if the given klass is not an array klass of any kind. 3860 // If obj_array/non_array==true/true: 3861 // Branch around if the kls is not an oop array (kls is int[], String, etc.) 3862 // If obj_array/non_array==true/false: 3863 // Branch around if the kls is an oop array (Object[] or subtype) 3864 // 3865 // Like generate_guard, adds a new path onto the region. 3866 jint layout_con = 0; 3867 Node* layout_val = get_layout_helper(kls, layout_con); 3868 if (layout_val == NULL) { 3869 bool query = (obj_array 3870 ? Klass::layout_helper_is_objArray(layout_con) 3871 : Klass::layout_helper_is_array(layout_con)); 3872 if (query == not_array) { 3873 return NULL; // never a branch 3874 } else { // always a branch 3875 Node* always_branch = control(); 3876 if (region != NULL) 3877 region->add_req(always_branch); 3878 set_control(top()); 3879 return always_branch; 3880 } 3881 } 3882 // Now test the correct condition. 3883 jint nval = (obj_array 3884 ? (jint)(Klass::_lh_array_tag_type_value 3885 << Klass::_lh_array_tag_shift) 3886 : Klass::_lh_neutral_value); 3887 Node* cmp = _gvn.transform(new CmpINode(layout_val, intcon(nval))); 3888 BoolTest::mask btest = BoolTest::lt; // correct for testing is_[obj]array 3889 // invert the test if we are looking for a non-array 3890 if (not_array) btest = BoolTest(btest).negate(); 3891 Node* bol = _gvn.transform(new BoolNode(cmp, btest)); 3892 return generate_fair_guard(bol, region); 3893 } 3894 3895 3896 //-----------------------inline_native_newArray-------------------------- 3897 // private static native Object java.lang.reflect.newArray(Class<?> componentType, int length); 3898 // private native Object Unsafe.allocateUninitializedArray0(Class<?> cls, int size); 3899 bool LibraryCallKit::inline_unsafe_newArray(bool uninitialized) { 3900 Node* mirror; 3901 Node* count_val; 3902 if (uninitialized) { 3903 mirror = argument(1); 3904 count_val = argument(2); 3905 } else { 3906 mirror = argument(0); 3907 count_val = argument(1); 3908 } 3909 3910 mirror = null_check(mirror); 3911 // If mirror or obj is dead, only null-path is taken. 3912 if (stopped()) return true; 3913 3914 enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT }; 3915 RegionNode* result_reg = new RegionNode(PATH_LIMIT); 3916 PhiNode* result_val = new PhiNode(result_reg, TypeInstPtr::NOTNULL); 3917 PhiNode* result_io = new PhiNode(result_reg, Type::ABIO); 3918 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM); 3919 3920 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); 3921 Node* klass_node = load_array_klass_from_mirror(mirror, never_see_null, 3922 result_reg, _slow_path); 3923 Node* normal_ctl = control(); 3924 Node* no_array_ctl = result_reg->in(_slow_path); 3925 3926 // Generate code for the slow case. We make a call to newArray(). 3927 set_control(no_array_ctl); 3928 if (!stopped()) { 3929 // Either the input type is void.class, or else the 3930 // array klass has not yet been cached. Either the 3931 // ensuing call will throw an exception, or else it 3932 // will cache the array klass for next time. 3933 PreserveJVMState pjvms(this); 3934 CallJavaNode* slow_call = NULL; 3935 if (uninitialized) { 3936 // Generate optimized virtual call (holder class 'Unsafe' is final) 3937 slow_call = generate_method_call(vmIntrinsics::_allocateUninitializedArray, false, false); 3938 } else { 3939 slow_call = generate_method_call_static(vmIntrinsics::_newArray); 3940 } 3941 Node* slow_result = set_results_for_java_call(slow_call); 3942 // this->control() comes from set_results_for_java_call 3943 result_reg->set_req(_slow_path, control()); 3944 result_val->set_req(_slow_path, slow_result); 3945 result_io ->set_req(_slow_path, i_o()); 3946 result_mem->set_req(_slow_path, reset_memory()); 3947 } 3948 3949 set_control(normal_ctl); 3950 if (!stopped()) { 3951 // Normal case: The array type has been cached in the java.lang.Class. 3952 // The following call works fine even if the array type is polymorphic. 3953 // It could be a dynamic mix of int[], boolean[], Object[], etc. 3954 Node* obj = new_array(klass_node, count_val, 0); // no arguments to push 3955 result_reg->init_req(_normal_path, control()); 3956 result_val->init_req(_normal_path, obj); 3957 result_io ->init_req(_normal_path, i_o()); 3958 result_mem->init_req(_normal_path, reset_memory()); 3959 3960 if (uninitialized) { 3961 // Mark the allocation so that zeroing is skipped 3962 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(obj, &_gvn); 3963 alloc->maybe_set_complete(&_gvn); 3964 } 3965 } 3966 3967 // Return the combined state. 3968 set_i_o( _gvn.transform(result_io) ); 3969 set_all_memory( _gvn.transform(result_mem)); 3970 3971 C->set_has_split_ifs(true); // Has chance for split-if optimization 3972 set_result(result_reg, result_val); 3973 return true; 3974 } 3975 3976 //----------------------inline_native_getLength-------------------------- 3977 // public static native int java.lang.reflect.Array.getLength(Object array); 3978 bool LibraryCallKit::inline_native_getLength() { 3979 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; 3980 3981 Node* array = null_check(argument(0)); 3982 // If array is dead, only null-path is taken. 3983 if (stopped()) return true; 3984 3985 // Deoptimize if it is a non-array. 3986 Node* non_array = generate_non_array_guard(load_object_klass(array), NULL); 3987 3988 if (non_array != NULL) { 3989 PreserveJVMState pjvms(this); 3990 set_control(non_array); 3991 uncommon_trap(Deoptimization::Reason_intrinsic, 3992 Deoptimization::Action_maybe_recompile); 3993 } 3994 3995 // If control is dead, only non-array-path is taken. 3996 if (stopped()) return true; 3997 3998 // The works fine even if the array type is polymorphic. 3999 // It could be a dynamic mix of int[], boolean[], Object[], etc. 4000 Node* result = load_array_length(array); 4001 4002 C->set_has_split_ifs(true); // Has chance for split-if optimization 4003 set_result(result); 4004 return true; 4005 } 4006 4007 //------------------------inline_array_copyOf---------------------------- 4008 // public static <T,U> T[] java.util.Arrays.copyOf( U[] original, int newLength, Class<? extends T[]> newType); 4009 // public static <T,U> T[] java.util.Arrays.copyOfRange(U[] original, int from, int to, Class<? extends T[]> newType); 4010 bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) { 4011 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; 4012 4013 // Get the arguments. 4014 Node* original = argument(0); 4015 Node* start = is_copyOfRange? argument(1): intcon(0); 4016 Node* end = is_copyOfRange? argument(2): argument(1); 4017 Node* array_type_mirror = is_copyOfRange? argument(3): argument(2); 4018 4019 Node* newcopy = NULL; 4020 4021 // Set the original stack and the reexecute bit for the interpreter to reexecute 4022 // the bytecode that invokes Arrays.copyOf if deoptimization happens. 4023 { PreserveReexecuteState preexecs(this); 4024 jvms()->set_should_reexecute(true); 4025 4026 array_type_mirror = null_check(array_type_mirror); 4027 original = null_check(original); 4028 4029 // Check if a null path was taken unconditionally. 4030 if (stopped()) return true; 4031 4032 Node* orig_length = load_array_length(original); 4033 4034 Node* klass_node = load_klass_from_mirror(array_type_mirror, false, NULL, 0); 4035 klass_node = null_check(klass_node); 4036 4037 RegionNode* bailout = new RegionNode(1); 4038 record_for_igvn(bailout); 4039 4040 // Despite the generic type of Arrays.copyOf, the mirror might be int, int[], etc. 4041 // Bail out if that is so. 4042 Node* not_objArray = generate_non_objArray_guard(klass_node, bailout); 4043 if (not_objArray != NULL) { 4044 // Improve the klass node's type from the new optimistic assumption: 4045 ciKlass* ak = ciArrayKlass::make(env()->Object_klass()); 4046 const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/); 4047 Node* cast = new CastPPNode(klass_node, akls); 4048 cast->init_req(0, control()); 4049 klass_node = _gvn.transform(cast); 4050 } 4051 4052 // Bail out if either start or end is negative. 4053 generate_negative_guard(start, bailout, &start); 4054 generate_negative_guard(end, bailout, &end); 4055 4056 Node* length = end; 4057 if (_gvn.type(start) != TypeInt::ZERO) { 4058 length = _gvn.transform(new SubINode(end, start)); 4059 } 4060 4061 // Bail out if length is negative. 4062 // Without this the new_array would throw 4063 // NegativeArraySizeException but IllegalArgumentException is what 4064 // should be thrown 4065 generate_negative_guard(length, bailout, &length); 4066 4067 if (bailout->req() > 1) { 4068 PreserveJVMState pjvms(this); 4069 set_control(_gvn.transform(bailout)); 4070 uncommon_trap(Deoptimization::Reason_intrinsic, 4071 Deoptimization::Action_maybe_recompile); 4072 } 4073 4074 if (!stopped()) { 4075 // How many elements will we copy from the original? 4076 // The answer is MinI(orig_length - start, length). 4077 Node* orig_tail = _gvn.transform(new SubINode(orig_length, start)); 4078 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length); 4079 4080 // Generate a direct call to the right arraycopy function(s). 4081 // We know the copy is disjoint but we might not know if the 4082 // oop stores need checking. 4083 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class). 4084 // This will fail a store-check if x contains any non-nulls. 4085 4086 // ArrayCopyNode:Ideal may transform the ArrayCopyNode to 4087 // loads/stores but it is legal only if we're sure the 4088 // Arrays.copyOf would succeed. So we need all input arguments 4089 // to the copyOf to be validated, including that the copy to the 4090 // new array won't trigger an ArrayStoreException. That subtype 4091 // check can be optimized if we know something on the type of 4092 // the input array from type speculation. 4093 if (_gvn.type(klass_node)->singleton()) { 4094 const TypeKlassPtr* subk = _gvn.type(load_object_klass(original))->is_klassptr(); 4095 const TypeKlassPtr* superk = _gvn.type(klass_node)->is_klassptr(); 4096 4097 int test = C->static_subtype_check(superk, subk); 4098 if (test != Compile::SSC_always_true && test != Compile::SSC_always_false) { 4099 const TypeOopPtr* t_original = _gvn.type(original)->is_oopptr(); 4100 if (t_original->speculative_type() != NULL) { 4101 original = maybe_cast_profiled_obj(original, t_original->speculative_type(), true); 4102 } 4103 } 4104 } 4105 4106 bool validated = false; 4107 // Reason_class_check rather than Reason_intrinsic because we 4108 // want to intrinsify even if this traps. 4109 if (!too_many_traps(Deoptimization::Reason_class_check)) { 4110 Node* not_subtype_ctrl = gen_subtype_check(original, klass_node); 4111 4112 if (not_subtype_ctrl != top()) { 4113 PreserveJVMState pjvms(this); 4114 set_control(not_subtype_ctrl); 4115 uncommon_trap(Deoptimization::Reason_class_check, 4116 Deoptimization::Action_make_not_entrant); 4117 assert(stopped(), "Should be stopped"); 4118 } 4119 validated = true; 4120 } 4121 4122 if (!stopped()) { 4123 newcopy = new_array(klass_node, length, 0); // no arguments to push 4124 4125 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, original, start, newcopy, intcon(0), moved, true, false, 4126 load_object_klass(original), klass_node); 4127 if (!is_copyOfRange) { 4128 ac->set_copyof(validated); 4129 } else { 4130 ac->set_copyofrange(validated); 4131 } 4132 Node* n = _gvn.transform(ac); 4133 if (n == ac) { 4134 ac->connect_outputs(this); 4135 } else { 4136 assert(validated, "shouldn't transform if all arguments not validated"); 4137 set_all_memory(n); 4138 } 4139 } 4140 } 4141 } // original reexecute is set back here 4142 4143 C->set_has_split_ifs(true); // Has chance for split-if optimization 4144 if (!stopped()) { 4145 set_result(newcopy); 4146 } 4147 return true; 4148 } 4149 4150 4151 //----------------------generate_virtual_guard--------------------------- 4152 // Helper for hashCode and clone. Peeks inside the vtable to avoid a call. 4153 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass, 4154 RegionNode* slow_region) { 4155 ciMethod* method = callee(); 4156 int vtable_index = method->vtable_index(); 4157 assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index, 4158 "bad index %d", vtable_index); 4159 // Get the Method* out of the appropriate vtable entry. 4160 int entry_offset = in_bytes(Klass::vtable_start_offset()) + 4161 vtable_index*vtableEntry::size_in_bytes() + 4162 vtableEntry::method_offset_in_bytes(); 4163 Node* entry_addr = basic_plus_adr(obj_klass, entry_offset); 4164 Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS, MemNode::unordered); 4165 4166 // Compare the target method with the expected method (e.g., Object.hashCode). 4167 const TypePtr* native_call_addr = TypeMetadataPtr::make(method); 4168 4169 Node* native_call = makecon(native_call_addr); 4170 Node* chk_native = _gvn.transform(new CmpPNode(target_call, native_call)); 4171 Node* test_native = _gvn.transform(new BoolNode(chk_native, BoolTest::ne)); 4172 4173 return generate_slow_guard(test_native, slow_region); 4174 } 4175 4176 //-----------------------generate_method_call---------------------------- 4177 // Use generate_method_call to make a slow-call to the real 4178 // method if the fast path fails. An alternative would be to 4179 // use a stub like OptoRuntime::slow_arraycopy_Java. 4180 // This only works for expanding the current library call, 4181 // not another intrinsic. (E.g., don't use this for making an 4182 // arraycopy call inside of the copyOf intrinsic.) 4183 CallJavaNode* 4184 LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) { 4185 // When compiling the intrinsic method itself, do not use this technique. 4186 guarantee(callee() != C->method(), "cannot make slow-call to self"); 4187 4188 ciMethod* method = callee(); 4189 // ensure the JVMS we have will be correct for this call 4190 guarantee(method_id == method->intrinsic_id(), "must match"); 4191 4192 const TypeFunc* tf = TypeFunc::make(method); 4193 CallJavaNode* slow_call; 4194 if (is_static) { 4195 assert(!is_virtual, ""); 4196 slow_call = new CallStaticJavaNode(C, tf, 4197 SharedRuntime::get_resolve_static_call_stub(), method); 4198 } else if (is_virtual) { 4199 null_check_receiver(); 4200 int vtable_index = Method::invalid_vtable_index; 4201 if (UseInlineCaches) { 4202 // Suppress the vtable call 4203 } else { 4204 // hashCode and clone are not a miranda methods, 4205 // so the vtable index is fixed. 4206 // No need to use the linkResolver to get it. 4207 vtable_index = method->vtable_index(); 4208 assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index, 4209 "bad index %d", vtable_index); 4210 } 4211 slow_call = new CallDynamicJavaNode(tf, 4212 SharedRuntime::get_resolve_virtual_call_stub(), 4213 method, vtable_index); 4214 } else { // neither virtual nor static: opt_virtual 4215 null_check_receiver(); 4216 slow_call = new CallStaticJavaNode(C, tf, 4217 SharedRuntime::get_resolve_opt_virtual_call_stub(), method); 4218 slow_call->set_optimized_virtual(true); 4219 } 4220 if (CallGenerator::is_inlined_method_handle_intrinsic(this->method(), bci(), callee())) { 4221 // To be able to issue a direct call (optimized virtual or virtual) 4222 // and skip a call to MH.linkTo*/invokeBasic adapter, additional information 4223 // about the method being invoked should be attached to the call site to 4224 // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C). 4225 slow_call->set_override_symbolic_info(true); 4226 } 4227 set_arguments_for_java_call(slow_call); 4228 set_edges_for_java_call(slow_call); 4229 return slow_call; 4230 } 4231 4232 4233 /** 4234 * Build special case code for calls to hashCode on an object. This call may 4235 * be virtual (invokevirtual) or bound (invokespecial). For each case we generate 4236 * slightly different code. 4237 */ 4238 bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) { 4239 assert(is_static == callee()->is_static(), "correct intrinsic selection"); 4240 assert(!(is_virtual && is_static), "either virtual, special, or static"); 4241 4242 enum { _slow_path = 1, _fast_path, _null_path, PATH_LIMIT }; 4243 4244 RegionNode* result_reg = new RegionNode(PATH_LIMIT); 4245 PhiNode* result_val = new PhiNode(result_reg, TypeInt::INT); 4246 PhiNode* result_io = new PhiNode(result_reg, Type::ABIO); 4247 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM); 4248 Node* obj = NULL; 4249 if (!is_static) { 4250 // Check for hashing null object 4251 obj = null_check_receiver(); 4252 if (stopped()) return true; // unconditionally null 4253 result_reg->init_req(_null_path, top()); 4254 result_val->init_req(_null_path, top()); 4255 } else { 4256 // Do a null check, and return zero if null. 4257 // System.identityHashCode(null) == 0 4258 obj = argument(0); 4259 Node* null_ctl = top(); 4260 obj = null_check_oop(obj, &null_ctl); 4261 result_reg->init_req(_null_path, null_ctl); 4262 result_val->init_req(_null_path, _gvn.intcon(0)); 4263 } 4264 4265 // Unconditionally null? Then return right away. 4266 if (stopped()) { 4267 set_control( result_reg->in(_null_path)); 4268 if (!stopped()) 4269 set_result(result_val->in(_null_path)); 4270 return true; 4271 } 4272 4273 // We only go to the fast case code if we pass a number of guards. The 4274 // paths which do not pass are accumulated in the slow_region. 4275 RegionNode* slow_region = new RegionNode(1); 4276 record_for_igvn(slow_region); 4277 4278 // If this is a virtual call, we generate a funny guard. We pull out 4279 // the vtable entry corresponding to hashCode() from the target object. 4280 // If the target method which we are calling happens to be the native 4281 // Object hashCode() method, we pass the guard. We do not need this 4282 // guard for non-virtual calls -- the caller is known to be the native 4283 // Object hashCode(). 4284 if (is_virtual) { 4285 // After null check, get the object's klass. 4286 Node* obj_klass = load_object_klass(obj); 4287 generate_virtual_guard(obj_klass, slow_region); 4288 } 4289 4290 // Get the header out of the object, use LoadMarkNode when available 4291 Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes()); 4292 // The control of the load must be NULL. Otherwise, the load can move before 4293 // the null check after castPP removal. 4294 Node* no_ctrl = NULL; 4295 Node* header = make_load(no_ctrl, header_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered); 4296 4297 // Test the header to see if it is unlocked. 4298 Node *lock_mask = _gvn.MakeConX(markWord::lock_mask_in_place); 4299 Node *lmasked_header = _gvn.transform(new AndXNode(header, lock_mask)); 4300 Node *unlocked_val = _gvn.MakeConX(markWord::unlocked_value); 4301 Node *chk_unlocked = _gvn.transform(new CmpXNode( lmasked_header, unlocked_val)); 4302 Node *test_unlocked = _gvn.transform(new BoolNode( chk_unlocked, BoolTest::ne)); 4303 4304 generate_slow_guard(test_unlocked, slow_region); 4305 4306 // Get the hash value and check to see that it has been properly assigned. 4307 // We depend on hash_mask being at most 32 bits and avoid the use of 4308 // hash_mask_in_place because it could be larger than 32 bits in a 64-bit 4309 // vm: see markWord.hpp. 4310 Node *hash_mask = _gvn.intcon(markWord::hash_mask); 4311 Node *hash_shift = _gvn.intcon(markWord::hash_shift); 4312 Node *hshifted_header= _gvn.transform(new URShiftXNode(header, hash_shift)); 4313 // This hack lets the hash bits live anywhere in the mark object now, as long 4314 // as the shift drops the relevant bits into the low 32 bits. Note that 4315 // Java spec says that HashCode is an int so there's no point in capturing 4316 // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build). 4317 hshifted_header = ConvX2I(hshifted_header); 4318 Node *hash_val = _gvn.transform(new AndINode(hshifted_header, hash_mask)); 4319 4320 Node *no_hash_val = _gvn.intcon(markWord::no_hash); 4321 Node *chk_assigned = _gvn.transform(new CmpINode( hash_val, no_hash_val)); 4322 Node *test_assigned = _gvn.transform(new BoolNode( chk_assigned, BoolTest::eq)); 4323 4324 generate_slow_guard(test_assigned, slow_region); 4325 4326 Node* init_mem = reset_memory(); 4327 // fill in the rest of the null path: 4328 result_io ->init_req(_null_path, i_o()); 4329 result_mem->init_req(_null_path, init_mem); 4330 4331 result_val->init_req(_fast_path, hash_val); 4332 result_reg->init_req(_fast_path, control()); 4333 result_io ->init_req(_fast_path, i_o()); 4334 result_mem->init_req(_fast_path, init_mem); 4335 4336 // Generate code for the slow case. We make a call to hashCode(). 4337 set_control(_gvn.transform(slow_region)); 4338 if (!stopped()) { 4339 // No need for PreserveJVMState, because we're using up the present state. 4340 set_all_memory(init_mem); 4341 vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode; 4342 CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static); 4343 Node* slow_result = set_results_for_java_call(slow_call); 4344 // this->control() comes from set_results_for_java_call 4345 result_reg->init_req(_slow_path, control()); 4346 result_val->init_req(_slow_path, slow_result); 4347 result_io ->set_req(_slow_path, i_o()); 4348 result_mem ->set_req(_slow_path, reset_memory()); 4349 } 4350 4351 // Return the combined state. 4352 set_i_o( _gvn.transform(result_io) ); 4353 set_all_memory( _gvn.transform(result_mem)); 4354 4355 set_result(result_reg, result_val); 4356 return true; 4357 } 4358 4359 //---------------------------inline_native_getClass---------------------------- 4360 // public final native Class<?> java.lang.Object.getClass(); 4361 // 4362 // Build special case code for calls to getClass on an object. 4363 bool LibraryCallKit::inline_native_getClass() { 4364 Node* obj = null_check_receiver(); 4365 if (stopped()) return true; 4366 set_result(load_mirror_from_klass(load_object_klass(obj))); 4367 return true; 4368 } 4369 4370 //-----------------inline_native_Reflection_getCallerClass--------------------- 4371 // public static native Class<?> sun.reflect.Reflection.getCallerClass(); 4372 // 4373 // In the presence of deep enough inlining, getCallerClass() becomes a no-op. 4374 // 4375 // NOTE: This code must perform the same logic as JVM_GetCallerClass 4376 // in that it must skip particular security frames and checks for 4377 // caller sensitive methods. 4378 bool LibraryCallKit::inline_native_Reflection_getCallerClass() { 4379 #ifndef PRODUCT 4380 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 4381 tty->print_cr("Attempting to inline sun.reflect.Reflection.getCallerClass"); 4382 } 4383 #endif 4384 4385 if (!jvms()->has_method()) { 4386 #ifndef PRODUCT 4387 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 4388 tty->print_cr(" Bailing out because intrinsic was inlined at top level"); 4389 } 4390 #endif 4391 return false; 4392 } 4393 4394 // Walk back up the JVM state to find the caller at the required 4395 // depth. 4396 JVMState* caller_jvms = jvms(); 4397 4398 // Cf. JVM_GetCallerClass 4399 // NOTE: Start the loop at depth 1 because the current JVM state does 4400 // not include the Reflection.getCallerClass() frame. 4401 for (int n = 1; caller_jvms != NULL; caller_jvms = caller_jvms->caller(), n++) { 4402 ciMethod* m = caller_jvms->method(); 4403 switch (n) { 4404 case 0: 4405 fatal("current JVM state does not include the Reflection.getCallerClass frame"); 4406 break; 4407 case 1: 4408 // Frame 0 and 1 must be caller sensitive (see JVM_GetCallerClass). 4409 if (!m->caller_sensitive()) { 4410 #ifndef PRODUCT 4411 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 4412 tty->print_cr(" Bailing out: CallerSensitive annotation expected at frame %d", n); 4413 } 4414 #endif 4415 return false; // bail-out; let JVM_GetCallerClass do the work 4416 } 4417 break; 4418 default: 4419 if (!m->is_ignored_by_security_stack_walk()) { 4420 // We have reached the desired frame; return the holder class. 4421 // Acquire method holder as java.lang.Class and push as constant. 4422 ciInstanceKlass* caller_klass = caller_jvms->method()->holder(); 4423 ciInstance* caller_mirror = caller_klass->java_mirror(); 4424 set_result(makecon(TypeInstPtr::make(caller_mirror))); 4425 4426 #ifndef PRODUCT 4427 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 4428 tty->print_cr(" Succeeded: caller = %d) %s.%s, JVMS depth = %d", n, caller_klass->name()->as_utf8(), caller_jvms->method()->name()->as_utf8(), jvms()->depth()); 4429 tty->print_cr(" JVM state at this point:"); 4430 for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) { 4431 ciMethod* m = jvms()->of_depth(i)->method(); 4432 tty->print_cr(" %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8()); 4433 } 4434 } 4435 #endif 4436 return true; 4437 } 4438 break; 4439 } 4440 } 4441 4442 #ifndef PRODUCT 4443 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 4444 tty->print_cr(" Bailing out because caller depth exceeded inlining depth = %d", jvms()->depth()); 4445 tty->print_cr(" JVM state at this point:"); 4446 for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) { 4447 ciMethod* m = jvms()->of_depth(i)->method(); 4448 tty->print_cr(" %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8()); 4449 } 4450 } 4451 #endif 4452 4453 return false; // bail-out; let JVM_GetCallerClass do the work 4454 } 4455 4456 bool LibraryCallKit::inline_fp_conversions(vmIntrinsics::ID id) { 4457 Node* arg = argument(0); 4458 Node* result = NULL; 4459 4460 switch (id) { 4461 case vmIntrinsics::_floatToRawIntBits: result = new MoveF2INode(arg); break; 4462 case vmIntrinsics::_intBitsToFloat: result = new MoveI2FNode(arg); break; 4463 case vmIntrinsics::_doubleToRawLongBits: result = new MoveD2LNode(arg); break; 4464 case vmIntrinsics::_longBitsToDouble: result = new MoveL2DNode(arg); break; 4465 case vmIntrinsics::_floatToFloat16: result = new ConvF2HFNode(arg); break; 4466 case vmIntrinsics::_float16ToFloat: result = new ConvHF2FNode(arg); break; 4467 4468 case vmIntrinsics::_doubleToLongBits: { 4469 // two paths (plus control) merge in a wood 4470 RegionNode *r = new RegionNode(3); 4471 Node *phi = new PhiNode(r, TypeLong::LONG); 4472 4473 Node *cmpisnan = _gvn.transform(new CmpDNode(arg, arg)); 4474 // Build the boolean node 4475 Node *bolisnan = _gvn.transform(new BoolNode(cmpisnan, BoolTest::ne)); 4476 4477 // Branch either way. 4478 // NaN case is less traveled, which makes all the difference. 4479 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); 4480 Node *opt_isnan = _gvn.transform(ifisnan); 4481 assert( opt_isnan->is_If(), "Expect an IfNode"); 4482 IfNode *opt_ifisnan = (IfNode*)opt_isnan; 4483 Node *iftrue = _gvn.transform(new IfTrueNode(opt_ifisnan)); 4484 4485 set_control(iftrue); 4486 4487 static const jlong nan_bits = CONST64(0x7ff8000000000000); 4488 Node *slow_result = longcon(nan_bits); // return NaN 4489 phi->init_req(1, _gvn.transform( slow_result )); 4490 r->init_req(1, iftrue); 4491 4492 // Else fall through 4493 Node *iffalse = _gvn.transform(new IfFalseNode(opt_ifisnan)); 4494 set_control(iffalse); 4495 4496 phi->init_req(2, _gvn.transform(new MoveD2LNode(arg))); 4497 r->init_req(2, iffalse); 4498 4499 // Post merge 4500 set_control(_gvn.transform(r)); 4501 record_for_igvn(r); 4502 4503 C->set_has_split_ifs(true); // Has chance for split-if optimization 4504 result = phi; 4505 assert(result->bottom_type()->isa_long(), "must be"); 4506 break; 4507 } 4508 4509 case vmIntrinsics::_floatToIntBits: { 4510 // two paths (plus control) merge in a wood 4511 RegionNode *r = new RegionNode(3); 4512 Node *phi = new PhiNode(r, TypeInt::INT); 4513 4514 Node *cmpisnan = _gvn.transform(new CmpFNode(arg, arg)); 4515 // Build the boolean node 4516 Node *bolisnan = _gvn.transform(new BoolNode(cmpisnan, BoolTest::ne)); 4517 4518 // Branch either way. 4519 // NaN case is less traveled, which makes all the difference. 4520 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); 4521 Node *opt_isnan = _gvn.transform(ifisnan); 4522 assert( opt_isnan->is_If(), "Expect an IfNode"); 4523 IfNode *opt_ifisnan = (IfNode*)opt_isnan; 4524 Node *iftrue = _gvn.transform(new IfTrueNode(opt_ifisnan)); 4525 4526 set_control(iftrue); 4527 4528 static const jint nan_bits = 0x7fc00000; 4529 Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN 4530 phi->init_req(1, _gvn.transform( slow_result )); 4531 r->init_req(1, iftrue); 4532 4533 // Else fall through 4534 Node *iffalse = _gvn.transform(new IfFalseNode(opt_ifisnan)); 4535 set_control(iffalse); 4536 4537 phi->init_req(2, _gvn.transform(new MoveF2INode(arg))); 4538 r->init_req(2, iffalse); 4539 4540 // Post merge 4541 set_control(_gvn.transform(r)); 4542 record_for_igvn(r); 4543 4544 C->set_has_split_ifs(true); // Has chance for split-if optimization 4545 result = phi; 4546 assert(result->bottom_type()->isa_int(), "must be"); 4547 break; 4548 } 4549 4550 default: 4551 fatal_unexpected_iid(id); 4552 break; 4553 } 4554 set_result(_gvn.transform(result)); 4555 return true; 4556 } 4557 4558 bool LibraryCallKit::inline_fp_range_check(vmIntrinsics::ID id) { 4559 Node* arg = argument(0); 4560 Node* result = NULL; 4561 4562 switch (id) { 4563 case vmIntrinsics::_floatIsInfinite: 4564 result = new IsInfiniteFNode(arg); 4565 break; 4566 case vmIntrinsics::_floatIsFinite: 4567 result = new IsFiniteFNode(arg); 4568 break; 4569 case vmIntrinsics::_doubleIsInfinite: 4570 result = new IsInfiniteDNode(arg); 4571 break; 4572 case vmIntrinsics::_doubleIsFinite: 4573 result = new IsFiniteDNode(arg); 4574 break; 4575 default: 4576 fatal_unexpected_iid(id); 4577 break; 4578 } 4579 set_result(_gvn.transform(result)); 4580 return true; 4581 } 4582 4583 //----------------------inline_unsafe_copyMemory------------------------- 4584 // public native void Unsafe.copyMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes); 4585 4586 static bool has_wide_mem(PhaseGVN& gvn, Node* addr, Node* base) { 4587 const TypeAryPtr* addr_t = gvn.type(addr)->isa_aryptr(); 4588 const Type* base_t = gvn.type(base); 4589 4590 bool in_native = (base_t == TypePtr::NULL_PTR); 4591 bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_t); 4592 bool is_mixed = !in_heap && !in_native; 4593 4594 if (is_mixed) { 4595 return true; // mixed accesses can touch both on-heap and off-heap memory 4596 } 4597 if (in_heap) { 4598 bool is_prim_array = (addr_t != NULL) && (addr_t->elem() != Type::BOTTOM); 4599 if (!is_prim_array) { 4600 // Though Unsafe.copyMemory() ensures at runtime for on-heap accesses that base is a primitive array, 4601 // there's not enough type information available to determine proper memory slice for it. 4602 return true; 4603 } 4604 } 4605 return false; 4606 } 4607 4608 bool LibraryCallKit::inline_unsafe_copyMemory() { 4609 if (callee()->is_static()) return false; // caller must have the capability! 4610 null_check_receiver(); // null-check receiver 4611 if (stopped()) return true; 4612 4613 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". 4614 4615 Node* src_base = argument(1); // type: oop 4616 Node* src_off = ConvL2X(argument(2)); // type: long 4617 Node* dst_base = argument(4); // type: oop 4618 Node* dst_off = ConvL2X(argument(5)); // type: long 4619 Node* size = ConvL2X(argument(7)); // type: long 4620 4621 assert(Unsafe_field_offset_to_byte_offset(11) == 11, 4622 "fieldOffset must be byte-scaled"); 4623 4624 Node* src_addr = make_unsafe_address(src_base, src_off); 4625 Node* dst_addr = make_unsafe_address(dst_base, dst_off); 4626 4627 Node* thread = _gvn.transform(new ThreadLocalNode()); 4628 Node* doing_unsafe_access_addr = basic_plus_adr(top(), thread, in_bytes(JavaThread::doing_unsafe_access_offset())); 4629 BasicType doing_unsafe_access_bt = T_BYTE; 4630 assert((sizeof(bool) * CHAR_BIT) == 8, "not implemented"); 4631 4632 // update volatile field 4633 store_to_memory(control(), doing_unsafe_access_addr, intcon(1), doing_unsafe_access_bt, Compile::AliasIdxRaw, MemNode::unordered); 4634 4635 int flags = RC_LEAF | RC_NO_FP; 4636 4637 const TypePtr* dst_type = TypePtr::BOTTOM; 4638 4639 // Adjust memory effects of the runtime call based on input values. 4640 if (!has_wide_mem(_gvn, src_addr, src_base) && 4641 !has_wide_mem(_gvn, dst_addr, dst_base)) { 4642 dst_type = _gvn.type(dst_addr)->is_ptr(); // narrow out memory 4643 4644 const TypePtr* src_type = _gvn.type(src_addr)->is_ptr(); 4645 if (C->get_alias_index(src_type) == C->get_alias_index(dst_type)) { 4646 flags |= RC_NARROW_MEM; // narrow in memory 4647 } 4648 } 4649 4650 // Call it. Note that the length argument is not scaled. 4651 make_runtime_call(flags, 4652 OptoRuntime::fast_arraycopy_Type(), 4653 StubRoutines::unsafe_arraycopy(), 4654 "unsafe_arraycopy", 4655 dst_type, 4656 src_addr, dst_addr, size XTOP); 4657 4658 store_to_memory(control(), doing_unsafe_access_addr, intcon(0), doing_unsafe_access_bt, Compile::AliasIdxRaw, MemNode::unordered); 4659 4660 return true; 4661 } 4662 4663 #undef XTOP 4664 4665 //------------------------clone_coping----------------------------------- 4666 // Helper function for inline_native_clone. 4667 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array) { 4668 assert(obj_size != NULL, ""); 4669 Node* raw_obj = alloc_obj->in(1); 4670 assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); 4671 4672 AllocateNode* alloc = NULL; 4673 if (ReduceBulkZeroing) { 4674 // We will be completely responsible for initializing this object - 4675 // mark Initialize node as complete. 4676 alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); 4677 // The object was just allocated - there should be no any stores! 4678 guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), ""); 4679 // Mark as complete_with_arraycopy so that on AllocateNode 4680 // expansion, we know this AllocateNode is initialized by an array 4681 // copy and a StoreStore barrier exists after the array copy. 4682 alloc->initialization()->set_complete_with_arraycopy(); 4683 } 4684 4685 Node* size = _gvn.transform(obj_size); 4686 access_clone(obj, alloc_obj, size, is_array); 4687 4688 // Do not let reads from the cloned object float above the arraycopy. 4689 if (alloc != NULL) { 4690 // Do not let stores that initialize this object be reordered with 4691 // a subsequent store that would make this object accessible by 4692 // other threads. 4693 // Record what AllocateNode this StoreStore protects so that 4694 // escape analysis can go from the MemBarStoreStoreNode to the 4695 // AllocateNode and eliminate the MemBarStoreStoreNode if possible 4696 // based on the escape status of the AllocateNode. 4697 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 4698 } else { 4699 insert_mem_bar(Op_MemBarCPUOrder); 4700 } 4701 } 4702 4703 //------------------------inline_native_clone---------------------------- 4704 // protected native Object java.lang.Object.clone(); 4705 // 4706 // Here are the simple edge cases: 4707 // null receiver => normal trap 4708 // virtual and clone was overridden => slow path to out-of-line clone 4709 // not cloneable or finalizer => slow path to out-of-line Object.clone 4710 // 4711 // The general case has two steps, allocation and copying. 4712 // Allocation has two cases, and uses GraphKit::new_instance or new_array. 4713 // 4714 // Copying also has two cases, oop arrays and everything else. 4715 // Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy). 4716 // Everything else uses the tight inline loop supplied by CopyArrayNode. 4717 // 4718 // These steps fold up nicely if and when the cloned object's klass 4719 // can be sharply typed as an object array, a type array, or an instance. 4720 // 4721 bool LibraryCallKit::inline_native_clone(bool is_virtual) { 4722 PhiNode* result_val; 4723 4724 // Set the reexecute bit for the interpreter to reexecute 4725 // the bytecode that invokes Object.clone if deoptimization happens. 4726 { PreserveReexecuteState preexecs(this); 4727 jvms()->set_should_reexecute(true); 4728 4729 Node* obj = null_check_receiver(); 4730 if (stopped()) return true; 4731 4732 const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr(); 4733 4734 // If we are going to clone an instance, we need its exact type to 4735 // know the number and types of fields to convert the clone to 4736 // loads/stores. Maybe a speculative type can help us. 4737 if (!obj_type->klass_is_exact() && 4738 obj_type->speculative_type() != NULL && 4739 obj_type->speculative_type()->is_instance_klass()) { 4740 ciInstanceKlass* spec_ik = obj_type->speculative_type()->as_instance_klass(); 4741 if (spec_ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem && 4742 !spec_ik->has_injected_fields()) { 4743 if (!obj_type->isa_instptr() || 4744 obj_type->is_instptr()->instance_klass()->has_subklass()) { 4745 obj = maybe_cast_profiled_obj(obj, obj_type->speculative_type(), false); 4746 } 4747 } 4748 } 4749 4750 // Conservatively insert a memory barrier on all memory slices. 4751 // Do not let writes into the original float below the clone. 4752 insert_mem_bar(Op_MemBarCPUOrder); 4753 4754 // paths into result_reg: 4755 enum { 4756 _slow_path = 1, // out-of-line call to clone method (virtual or not) 4757 _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy 4758 _array_path, // plain array allocation, plus arrayof_long_arraycopy 4759 _instance_path, // plain instance allocation, plus arrayof_long_arraycopy 4760 PATH_LIMIT 4761 }; 4762 RegionNode* result_reg = new RegionNode(PATH_LIMIT); 4763 result_val = new PhiNode(result_reg, TypeInstPtr::NOTNULL); 4764 PhiNode* result_i_o = new PhiNode(result_reg, Type::ABIO); 4765 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM); 4766 record_for_igvn(result_reg); 4767 4768 Node* obj_klass = load_object_klass(obj); 4769 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL); 4770 if (array_ctl != NULL) { 4771 // It's an array. 4772 PreserveJVMState pjvms(this); 4773 set_control(array_ctl); 4774 Node* obj_length = load_array_length(obj); 4775 Node* obj_size = NULL; 4776 Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size, /*deoptimize_on_exception=*/true); 4777 4778 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 4779 if (bs->array_copy_requires_gc_barriers(true, T_OBJECT, true, false, BarrierSetC2::Parsing)) { 4780 // If it is an oop array, it requires very special treatment, 4781 // because gc barriers are required when accessing the array. 4782 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL); 4783 if (is_obja != NULL) { 4784 PreserveJVMState pjvms2(this); 4785 set_control(is_obja); 4786 // Generate a direct call to the right arraycopy function(s). 4787 // Clones are always tightly coupled. 4788 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, true, false); 4789 ac->set_clone_oop_array(); 4790 Node* n = _gvn.transform(ac); 4791 assert(n == ac, "cannot disappear"); 4792 ac->connect_outputs(this, /*deoptimize_on_exception=*/true); 4793 4794 result_reg->init_req(_objArray_path, control()); 4795 result_val->init_req(_objArray_path, alloc_obj); 4796 result_i_o ->set_req(_objArray_path, i_o()); 4797 result_mem ->set_req(_objArray_path, reset_memory()); 4798 } 4799 } 4800 // Otherwise, there are no barriers to worry about. 4801 // (We can dispense with card marks if we know the allocation 4802 // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks 4803 // causes the non-eden paths to take compensating steps to 4804 // simulate a fresh allocation, so that no further 4805 // card marks are required in compiled code to initialize 4806 // the object.) 4807 4808 if (!stopped()) { 4809 copy_to_clone(obj, alloc_obj, obj_size, true); 4810 4811 // Present the results of the copy. 4812 result_reg->init_req(_array_path, control()); 4813 result_val->init_req(_array_path, alloc_obj); 4814 result_i_o ->set_req(_array_path, i_o()); 4815 result_mem ->set_req(_array_path, reset_memory()); 4816 } 4817 } 4818 4819 // We only go to the instance fast case code if we pass a number of guards. 4820 // The paths which do not pass are accumulated in the slow_region. 4821 RegionNode* slow_region = new RegionNode(1); 4822 record_for_igvn(slow_region); 4823 if (!stopped()) { 4824 // It's an instance (we did array above). Make the slow-path tests. 4825 // If this is a virtual call, we generate a funny guard. We grab 4826 // the vtable entry corresponding to clone() from the target object. 4827 // If the target method which we are calling happens to be the 4828 // Object clone() method, we pass the guard. We do not need this 4829 // guard for non-virtual calls; the caller is known to be the native 4830 // Object clone(). 4831 if (is_virtual) { 4832 generate_virtual_guard(obj_klass, slow_region); 4833 } 4834 4835 // The object must be easily cloneable and must not have a finalizer. 4836 // Both of these conditions may be checked in a single test. 4837 // We could optimize the test further, but we don't care. 4838 generate_access_flags_guard(obj_klass, 4839 // Test both conditions: 4840 JVM_ACC_IS_CLONEABLE_FAST | JVM_ACC_HAS_FINALIZER, 4841 // Must be cloneable but not finalizer: 4842 JVM_ACC_IS_CLONEABLE_FAST, 4843 slow_region); 4844 } 4845 4846 if (!stopped()) { 4847 // It's an instance, and it passed the slow-path tests. 4848 PreserveJVMState pjvms(this); 4849 Node* obj_size = NULL; 4850 // Need to deoptimize on exception from allocation since Object.clone intrinsic 4851 // is reexecuted if deoptimization occurs and there could be problems when merging 4852 // exception state between multiple Object.clone versions (reexecute=true vs reexecute=false). 4853 Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true); 4854 4855 copy_to_clone(obj, alloc_obj, obj_size, false); 4856 4857 // Present the results of the slow call. 4858 result_reg->init_req(_instance_path, control()); 4859 result_val->init_req(_instance_path, alloc_obj); 4860 result_i_o ->set_req(_instance_path, i_o()); 4861 result_mem ->set_req(_instance_path, reset_memory()); 4862 } 4863 4864 // Generate code for the slow case. We make a call to clone(). 4865 set_control(_gvn.transform(slow_region)); 4866 if (!stopped()) { 4867 PreserveJVMState pjvms(this); 4868 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual); 4869 // We need to deoptimize on exception (see comment above) 4870 Node* slow_result = set_results_for_java_call(slow_call, false, /* deoptimize */ true); 4871 // this->control() comes from set_results_for_java_call 4872 result_reg->init_req(_slow_path, control()); 4873 result_val->init_req(_slow_path, slow_result); 4874 result_i_o ->set_req(_slow_path, i_o()); 4875 result_mem ->set_req(_slow_path, reset_memory()); 4876 } 4877 4878 // Return the combined state. 4879 set_control( _gvn.transform(result_reg)); 4880 set_i_o( _gvn.transform(result_i_o)); 4881 set_all_memory( _gvn.transform(result_mem)); 4882 } // original reexecute is set back here 4883 4884 set_result(_gvn.transform(result_val)); 4885 return true; 4886 } 4887 4888 // If we have a tightly coupled allocation, the arraycopy may take care 4889 // of the array initialization. If one of the guards we insert between 4890 // the allocation and the arraycopy causes a deoptimization, an 4891 // uninitialized array will escape the compiled method. To prevent that 4892 // we set the JVM state for uncommon traps between the allocation and 4893 // the arraycopy to the state before the allocation so, in case of 4894 // deoptimization, we'll reexecute the allocation and the 4895 // initialization. 4896 JVMState* LibraryCallKit::arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp) { 4897 if (alloc != NULL) { 4898 ciMethod* trap_method = alloc->jvms()->method(); 4899 int trap_bci = alloc->jvms()->bci(); 4900 4901 if (!C->too_many_traps(trap_method, trap_bci, Deoptimization::Reason_intrinsic) && 4902 !C->too_many_traps(trap_method, trap_bci, Deoptimization::Reason_null_check)) { 4903 // Make sure there's no store between the allocation and the 4904 // arraycopy otherwise visible side effects could be rexecuted 4905 // in case of deoptimization and cause incorrect execution. 4906 bool no_interfering_store = true; 4907 Node* mem = alloc->in(TypeFunc::Memory); 4908 if (mem->is_MergeMem()) { 4909 for (MergeMemStream mms(merged_memory(), mem->as_MergeMem()); mms.next_non_empty2(); ) { 4910 Node* n = mms.memory(); 4911 if (n != mms.memory2() && !(n->is_Proj() && n->in(0) == alloc->initialization())) { 4912 assert(n->is_Store(), "what else?"); 4913 no_interfering_store = false; 4914 break; 4915 } 4916 } 4917 } else { 4918 for (MergeMemStream mms(merged_memory()); mms.next_non_empty(); ) { 4919 Node* n = mms.memory(); 4920 if (n != mem && !(n->is_Proj() && n->in(0) == alloc->initialization())) { 4921 assert(n->is_Store(), "what else?"); 4922 no_interfering_store = false; 4923 break; 4924 } 4925 } 4926 } 4927 4928 if (no_interfering_store) { 4929 SafePointNode* sfpt = create_safepoint_with_state_before_array_allocation(alloc); 4930 4931 JVMState* saved_jvms = jvms(); 4932 saved_reexecute_sp = _reexecute_sp; 4933 4934 set_jvms(sfpt->jvms()); 4935 _reexecute_sp = jvms()->sp(); 4936 4937 return saved_jvms; 4938 } 4939 } 4940 } 4941 return NULL; 4942 } 4943 4944 // Clone the JVMState of the array allocation and create a new safepoint with it. Re-push the array length to the stack 4945 // such that uncommon traps can be emitted to re-execute the array allocation in the interpreter. 4946 SafePointNode* LibraryCallKit::create_safepoint_with_state_before_array_allocation(const AllocateArrayNode* alloc) const { 4947 JVMState* old_jvms = alloc->jvms()->clone_shallow(C); 4948 uint size = alloc->req(); 4949 SafePointNode* sfpt = new SafePointNode(size, old_jvms); 4950 old_jvms->set_map(sfpt); 4951 for (uint i = 0; i < size; i++) { 4952 sfpt->init_req(i, alloc->in(i)); 4953 } 4954 // re-push array length for deoptimization 4955 sfpt->ins_req(old_jvms->stkoff() + old_jvms->sp(), alloc->in(AllocateNode::ALength)); 4956 old_jvms->set_sp(old_jvms->sp()+1); 4957 old_jvms->set_monoff(old_jvms->monoff()+1); 4958 old_jvms->set_scloff(old_jvms->scloff()+1); 4959 old_jvms->set_endoff(old_jvms->endoff()+1); 4960 old_jvms->set_should_reexecute(true); 4961 4962 sfpt->set_i_o(map()->i_o()); 4963 sfpt->set_memory(map()->memory()); 4964 sfpt->set_control(map()->control()); 4965 return sfpt; 4966 } 4967 4968 // In case of a deoptimization, we restart execution at the 4969 // allocation, allocating a new array. We would leave an uninitialized 4970 // array in the heap that GCs wouldn't expect. Move the allocation 4971 // after the traps so we don't allocate the array if we 4972 // deoptimize. This is possible because tightly_coupled_allocation() 4973 // guarantees there's no observer of the allocated array at this point 4974 // and the control flow is simple enough. 4975 void LibraryCallKit::arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms_before_guards, 4976 int saved_reexecute_sp, uint new_idx) { 4977 if (saved_jvms_before_guards != NULL && !stopped()) { 4978 replace_unrelated_uncommon_traps_with_alloc_state(alloc, saved_jvms_before_guards); 4979 4980 assert(alloc != NULL, "only with a tightly coupled allocation"); 4981 // restore JVM state to the state at the arraycopy 4982 saved_jvms_before_guards->map()->set_control(map()->control()); 4983 assert(saved_jvms_before_guards->map()->memory() == map()->memory(), "memory state changed?"); 4984 assert(saved_jvms_before_guards->map()->i_o() == map()->i_o(), "IO state changed?"); 4985 // If we've improved the types of some nodes (null check) while 4986 // emitting the guards, propagate them to the current state 4987 map()->replaced_nodes().apply(saved_jvms_before_guards->map(), new_idx); 4988 set_jvms(saved_jvms_before_guards); 4989 _reexecute_sp = saved_reexecute_sp; 4990 4991 // Remove the allocation from above the guards 4992 CallProjections callprojs; 4993 alloc->extract_projections(&callprojs, true); 4994 InitializeNode* init = alloc->initialization(); 4995 Node* alloc_mem = alloc->in(TypeFunc::Memory); 4996 C->gvn_replace_by(callprojs.fallthrough_ioproj, alloc->in(TypeFunc::I_O)); 4997 C->gvn_replace_by(init->proj_out(TypeFunc::Memory), alloc_mem); 4998 4999 // The CastIINode created in GraphKit::new_array (in AllocateArrayNode::make_ideal_length) must stay below 5000 // the allocation (i.e. is only valid if the allocation succeeds): 5001 // 1) replace CastIINode with AllocateArrayNode's length here 5002 // 2) Create CastIINode again once allocation has moved (see below) at the end of this method 5003 // 5004 // Multiple identical CastIINodes might exist here. Each GraphKit::load_array_length() call will generate 5005 // new separate CastIINode (arraycopy guard checks or any array length use between array allocation and ararycopy) 5006 Node* init_control = init->proj_out(TypeFunc::Control); 5007 Node* alloc_length = alloc->Ideal_length(); 5008 #ifdef ASSERT 5009 Node* prev_cast = NULL; 5010 #endif 5011 for (uint i = 0; i < init_control->outcnt(); i++) { 5012 Node* init_out = init_control->raw_out(i); 5013 if (init_out->is_CastII() && init_out->in(TypeFunc::Control) == init_control && init_out->in(1) == alloc_length) { 5014 #ifdef ASSERT 5015 if (prev_cast == NULL) { 5016 prev_cast = init_out; 5017 } else { 5018 if (prev_cast->cmp(*init_out) == false) { 5019 prev_cast->dump(); 5020 init_out->dump(); 5021 assert(false, "not equal CastIINode"); 5022 } 5023 } 5024 #endif 5025 C->gvn_replace_by(init_out, alloc_length); 5026 } 5027 } 5028 C->gvn_replace_by(init->proj_out(TypeFunc::Control), alloc->in(0)); 5029 5030 // move the allocation here (after the guards) 5031 _gvn.hash_delete(alloc); 5032 alloc->set_req(TypeFunc::Control, control()); 5033 alloc->set_req(TypeFunc::I_O, i_o()); 5034 Node *mem = reset_memory(); 5035 set_all_memory(mem); 5036 alloc->set_req(TypeFunc::Memory, mem); 5037 set_control(init->proj_out_or_null(TypeFunc::Control)); 5038 set_i_o(callprojs.fallthrough_ioproj); 5039 5040 // Update memory as done in GraphKit::set_output_for_allocation() 5041 const TypeInt* length_type = _gvn.find_int_type(alloc->in(AllocateNode::ALength)); 5042 const TypeOopPtr* ary_type = _gvn.type(alloc->in(AllocateNode::KlassNode))->is_klassptr()->as_instance_type(); 5043 if (ary_type->isa_aryptr() && length_type != NULL) { 5044 ary_type = ary_type->is_aryptr()->cast_to_size(length_type); 5045 } 5046 const TypePtr* telemref = ary_type->add_offset(Type::OffsetBot); 5047 int elemidx = C->get_alias_index(telemref); 5048 set_memory(init->proj_out_or_null(TypeFunc::Memory), Compile::AliasIdxRaw); 5049 set_memory(init->proj_out_or_null(TypeFunc::Memory), elemidx); 5050 5051 Node* allocx = _gvn.transform(alloc); 5052 assert(allocx == alloc, "where has the allocation gone?"); 5053 assert(dest->is_CheckCastPP(), "not an allocation result?"); 5054 5055 _gvn.hash_delete(dest); 5056 dest->set_req(0, control()); 5057 Node* destx = _gvn.transform(dest); 5058 assert(destx == dest, "where has the allocation result gone?"); 5059 5060 array_ideal_length(alloc, ary_type, true); 5061 } 5062 } 5063 5064 // Unrelated UCTs between the array allocation and the array copy, which are considered safe by tightly_coupled_allocation(), 5065 // need to be replaced by an UCT with a state before the array allocation (including the array length). This is necessary 5066 // because we could hit one of these UCTs (which are executed before the emitted array copy guards and the actual array 5067 // allocation which is moved down in arraycopy_move_allocation_here()). When later resuming execution in the interpreter, 5068 // we would have wrongly skipped the array allocation. To prevent this, we resume execution at the array allocation in 5069 // the interpreter similar to what we are doing for the newly emitted guards for the array copy. 5070 void LibraryCallKit::replace_unrelated_uncommon_traps_with_alloc_state(AllocateArrayNode* alloc, 5071 JVMState* saved_jvms_before_guards) { 5072 if (saved_jvms_before_guards->map()->control()->is_IfProj()) { 5073 // There is at least one unrelated uncommon trap which needs to be replaced. 5074 SafePointNode* sfpt = create_safepoint_with_state_before_array_allocation(alloc); 5075 5076 JVMState* saved_jvms = jvms(); 5077 const int saved_reexecute_sp = _reexecute_sp; 5078 set_jvms(sfpt->jvms()); 5079 _reexecute_sp = jvms()->sp(); 5080 5081 replace_unrelated_uncommon_traps_with_alloc_state(saved_jvms_before_guards); 5082 5083 // Restore state 5084 set_jvms(saved_jvms); 5085 _reexecute_sp = saved_reexecute_sp; 5086 } 5087 } 5088 5089 // Replace the unrelated uncommon traps with new uncommon trap nodes by reusing the action and reason. The new uncommon 5090 // traps will have the state of the array allocation. Let the old uncommon trap nodes die. 5091 void LibraryCallKit::replace_unrelated_uncommon_traps_with_alloc_state(JVMState* saved_jvms_before_guards) { 5092 Node* if_proj = saved_jvms_before_guards->map()->control(); // Start the search right before the newly emitted guards 5093 while (if_proj->is_IfProj()) { 5094 CallStaticJavaNode* uncommon_trap = get_uncommon_trap_from_success_proj(if_proj); 5095 if (uncommon_trap != nullptr) { 5096 create_new_uncommon_trap(uncommon_trap); 5097 } 5098 assert(if_proj->in(0)->is_If(), "must be If"); 5099 if_proj = if_proj->in(0)->in(0); 5100 } 5101 assert(if_proj->is_Proj() && if_proj->in(0)->is_Initialize(), 5102 "must have reached control projection of init node"); 5103 } 5104 5105 void LibraryCallKit::create_new_uncommon_trap(CallStaticJavaNode* uncommon_trap_call) { 5106 const int trap_request = uncommon_trap_call->uncommon_trap_request(); 5107 assert(trap_request != 0, "no valid UCT trap request"); 5108 PreserveJVMState pjvms(this); 5109 set_control(uncommon_trap_call->in(0)); 5110 uncommon_trap(Deoptimization::trap_request_reason(trap_request), 5111 Deoptimization::trap_request_action(trap_request)); 5112 assert(stopped(), "Should be stopped"); 5113 _gvn.hash_delete(uncommon_trap_call); 5114 uncommon_trap_call->set_req(0, top()); // not used anymore, kill it 5115 } 5116 5117 //------------------------------inline_arraycopy----------------------- 5118 // public static native void java.lang.System.arraycopy(Object src, int srcPos, 5119 // Object dest, int destPos, 5120 // int length); 5121 bool LibraryCallKit::inline_arraycopy() { 5122 // Get the arguments. 5123 Node* src = argument(0); // type: oop 5124 Node* src_offset = argument(1); // type: int 5125 Node* dest = argument(2); // type: oop 5126 Node* dest_offset = argument(3); // type: int 5127 Node* length = argument(4); // type: int 5128 5129 uint new_idx = C->unique(); 5130 5131 // Check for allocation before we add nodes that would confuse 5132 // tightly_coupled_allocation() 5133 AllocateArrayNode* alloc = tightly_coupled_allocation(dest); 5134 5135 int saved_reexecute_sp = -1; 5136 JVMState* saved_jvms_before_guards = arraycopy_restore_alloc_state(alloc, saved_reexecute_sp); 5137 // See arraycopy_restore_alloc_state() comment 5138 // if alloc == NULL we don't have to worry about a tightly coupled allocation so we can emit all needed guards 5139 // if saved_jvms_before_guards != NULL (then alloc != NULL) then we can handle guards and a tightly coupled allocation 5140 // if saved_jvms_before_guards == NULL and alloc != NULL, we can't emit any guards 5141 bool can_emit_guards = (alloc == NULL || saved_jvms_before_guards != NULL); 5142 5143 // The following tests must be performed 5144 // (1) src and dest are arrays. 5145 // (2) src and dest arrays must have elements of the same BasicType 5146 // (3) src and dest must not be null. 5147 // (4) src_offset must not be negative. 5148 // (5) dest_offset must not be negative. 5149 // (6) length must not be negative. 5150 // (7) src_offset + length must not exceed length of src. 5151 // (8) dest_offset + length must not exceed length of dest. 5152 // (9) each element of an oop array must be assignable 5153 5154 // (3) src and dest must not be null. 5155 // always do this here because we need the JVM state for uncommon traps 5156 Node* null_ctl = top(); 5157 src = saved_jvms_before_guards != NULL ? null_check_oop(src, &null_ctl, true, true) : null_check(src, T_ARRAY); 5158 assert(null_ctl->is_top(), "no null control here"); 5159 dest = null_check(dest, T_ARRAY); 5160 5161 if (!can_emit_guards) { 5162 // if saved_jvms_before_guards == NULL and alloc != NULL, we don't emit any 5163 // guards but the arraycopy node could still take advantage of a 5164 // tightly allocated allocation. tightly_coupled_allocation() is 5165 // called again to make sure it takes the null check above into 5166 // account: the null check is mandatory and if it caused an 5167 // uncommon trap to be emitted then the allocation can't be 5168 // considered tightly coupled in this context. 5169 alloc = tightly_coupled_allocation(dest); 5170 } 5171 5172 bool validated = false; 5173 5174 const Type* src_type = _gvn.type(src); 5175 const Type* dest_type = _gvn.type(dest); 5176 const TypeAryPtr* top_src = src_type->isa_aryptr(); 5177 const TypeAryPtr* top_dest = dest_type->isa_aryptr(); 5178 5179 // Do we have the type of src? 5180 bool has_src = (top_src != NULL && top_src->elem() != Type::BOTTOM); 5181 // Do we have the type of dest? 5182 bool has_dest = (top_dest != NULL && top_dest->elem() != Type::BOTTOM); 5183 // Is the type for src from speculation? 5184 bool src_spec = false; 5185 // Is the type for dest from speculation? 5186 bool dest_spec = false; 5187 5188 if ((!has_src || !has_dest) && can_emit_guards) { 5189 // We don't have sufficient type information, let's see if 5190 // speculative types can help. We need to have types for both src 5191 // and dest so that it pays off. 5192 5193 // Do we already have or could we have type information for src 5194 bool could_have_src = has_src; 5195 // Do we already have or could we have type information for dest 5196 bool could_have_dest = has_dest; 5197 5198 ciKlass* src_k = NULL; 5199 if (!has_src) { 5200 src_k = src_type->speculative_type_not_null(); 5201 if (src_k != NULL && src_k->is_array_klass()) { 5202 could_have_src = true; 5203 } 5204 } 5205 5206 ciKlass* dest_k = NULL; 5207 if (!has_dest) { 5208 dest_k = dest_type->speculative_type_not_null(); 5209 if (dest_k != NULL && dest_k->is_array_klass()) { 5210 could_have_dest = true; 5211 } 5212 } 5213 5214 if (could_have_src && could_have_dest) { 5215 // This is going to pay off so emit the required guards 5216 if (!has_src) { 5217 src = maybe_cast_profiled_obj(src, src_k, true); 5218 src_type = _gvn.type(src); 5219 top_src = src_type->isa_aryptr(); 5220 has_src = (top_src != NULL && top_src->elem() != Type::BOTTOM); 5221 src_spec = true; 5222 } 5223 if (!has_dest) { 5224 dest = maybe_cast_profiled_obj(dest, dest_k, true); 5225 dest_type = _gvn.type(dest); 5226 top_dest = dest_type->isa_aryptr(); 5227 has_dest = (top_dest != NULL && top_dest->elem() != Type::BOTTOM); 5228 dest_spec = true; 5229 } 5230 } 5231 } 5232 5233 if (has_src && has_dest && can_emit_guards) { 5234 BasicType src_elem = top_src->isa_aryptr()->elem()->array_element_basic_type(); 5235 BasicType dest_elem = top_dest->isa_aryptr()->elem()->array_element_basic_type(); 5236 if (is_reference_type(src_elem, true)) src_elem = T_OBJECT; 5237 if (is_reference_type(dest_elem, true)) dest_elem = T_OBJECT; 5238 5239 if (src_elem == dest_elem && src_elem == T_OBJECT) { 5240 // If both arrays are object arrays then having the exact types 5241 // for both will remove the need for a subtype check at runtime 5242 // before the call and may make it possible to pick a faster copy 5243 // routine (without a subtype check on every element) 5244 // Do we have the exact type of src? 5245 bool could_have_src = src_spec; 5246 // Do we have the exact type of dest? 5247 bool could_have_dest = dest_spec; 5248 ciKlass* src_k = NULL; 5249 ciKlass* dest_k = NULL; 5250 if (!src_spec) { 5251 src_k = src_type->speculative_type_not_null(); 5252 if (src_k != NULL && src_k->is_array_klass()) { 5253 could_have_src = true; 5254 } 5255 } 5256 if (!dest_spec) { 5257 dest_k = dest_type->speculative_type_not_null(); 5258 if (dest_k != NULL && dest_k->is_array_klass()) { 5259 could_have_dest = true; 5260 } 5261 } 5262 if (could_have_src && could_have_dest) { 5263 // If we can have both exact types, emit the missing guards 5264 if (could_have_src && !src_spec) { 5265 src = maybe_cast_profiled_obj(src, src_k, true); 5266 } 5267 if (could_have_dest && !dest_spec) { 5268 dest = maybe_cast_profiled_obj(dest, dest_k, true); 5269 } 5270 } 5271 } 5272 } 5273 5274 ciMethod* trap_method = method(); 5275 int trap_bci = bci(); 5276 if (saved_jvms_before_guards != NULL) { 5277 trap_method = alloc->jvms()->method(); 5278 trap_bci = alloc->jvms()->bci(); 5279 } 5280 5281 bool negative_length_guard_generated = false; 5282 5283 if (!C->too_many_traps(trap_method, trap_bci, Deoptimization::Reason_intrinsic) && 5284 can_emit_guards && 5285 !src->is_top() && !dest->is_top()) { 5286 // validate arguments: enables transformation the ArrayCopyNode 5287 validated = true; 5288 5289 RegionNode* slow_region = new RegionNode(1); 5290 record_for_igvn(slow_region); 5291 5292 // (1) src and dest are arrays. 5293 generate_non_array_guard(load_object_klass(src), slow_region); 5294 generate_non_array_guard(load_object_klass(dest), slow_region); 5295 5296 // (2) src and dest arrays must have elements of the same BasicType 5297 // done at macro expansion or at Ideal transformation time 5298 5299 // (4) src_offset must not be negative. 5300 generate_negative_guard(src_offset, slow_region); 5301 5302 // (5) dest_offset must not be negative. 5303 generate_negative_guard(dest_offset, slow_region); 5304 5305 // (7) src_offset + length must not exceed length of src. 5306 generate_limit_guard(src_offset, length, 5307 load_array_length(src), 5308 slow_region); 5309 5310 // (8) dest_offset + length must not exceed length of dest. 5311 generate_limit_guard(dest_offset, length, 5312 load_array_length(dest), 5313 slow_region); 5314 5315 // (6) length must not be negative. 5316 // This is also checked in generate_arraycopy() during macro expansion, but 5317 // we also have to check it here for the case where the ArrayCopyNode will 5318 // be eliminated by Escape Analysis. 5319 if (EliminateAllocations) { 5320 generate_negative_guard(length, slow_region); 5321 negative_length_guard_generated = true; 5322 } 5323 5324 // (9) each element of an oop array must be assignable 5325 Node* dest_klass = load_object_klass(dest); 5326 if (src != dest) { 5327 Node* not_subtype_ctrl = gen_subtype_check(src, dest_klass); 5328 5329 if (not_subtype_ctrl != top()) { 5330 PreserveJVMState pjvms(this); 5331 set_control(not_subtype_ctrl); 5332 uncommon_trap(Deoptimization::Reason_intrinsic, 5333 Deoptimization::Action_make_not_entrant); 5334 assert(stopped(), "Should be stopped"); 5335 } 5336 } 5337 { 5338 PreserveJVMState pjvms(this); 5339 set_control(_gvn.transform(slow_region)); 5340 uncommon_trap(Deoptimization::Reason_intrinsic, 5341 Deoptimization::Action_make_not_entrant); 5342 assert(stopped(), "Should be stopped"); 5343 } 5344 5345 const TypeKlassPtr* dest_klass_t = _gvn.type(dest_klass)->is_klassptr(); 5346 const Type *toop = dest_klass_t->cast_to_exactness(false)->as_instance_type(); 5347 src = _gvn.transform(new CheckCastPPNode(control(), src, toop)); 5348 arraycopy_move_allocation_here(alloc, dest, saved_jvms_before_guards, saved_reexecute_sp, new_idx); 5349 } 5350 5351 if (stopped()) { 5352 return true; 5353 } 5354 5355 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, src, src_offset, dest, dest_offset, length, alloc != NULL, negative_length_guard_generated, 5356 // Create LoadRange and LoadKlass nodes for use during macro expansion here 5357 // so the compiler has a chance to eliminate them: during macro expansion, 5358 // we have to set their control (CastPP nodes are eliminated). 5359 load_object_klass(src), load_object_klass(dest), 5360 load_array_length(src), load_array_length(dest)); 5361 5362 ac->set_arraycopy(validated); 5363 5364 Node* n = _gvn.transform(ac); 5365 if (n == ac) { 5366 ac->connect_outputs(this); 5367 } else { 5368 assert(validated, "shouldn't transform if all arguments not validated"); 5369 set_all_memory(n); 5370 } 5371 clear_upper_avx(); 5372 5373 5374 return true; 5375 } 5376 5377 5378 // Helper function which determines if an arraycopy immediately follows 5379 // an allocation, with no intervening tests or other escapes for the object. 5380 AllocateArrayNode* 5381 LibraryCallKit::tightly_coupled_allocation(Node* ptr) { 5382 if (stopped()) return NULL; // no fast path 5383 if (!C->do_aliasing()) return NULL; // no MergeMems around 5384 5385 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn); 5386 if (alloc == NULL) return NULL; 5387 5388 Node* rawmem = memory(Compile::AliasIdxRaw); 5389 // Is the allocation's memory state untouched? 5390 if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) { 5391 // Bail out if there have been raw-memory effects since the allocation. 5392 // (Example: There might have been a call or safepoint.) 5393 return NULL; 5394 } 5395 rawmem = rawmem->in(0)->as_Initialize()->memory(Compile::AliasIdxRaw); 5396 if (!(rawmem->is_Proj() && rawmem->in(0) == alloc)) { 5397 return NULL; 5398 } 5399 5400 // There must be no unexpected observers of this allocation. 5401 for (DUIterator_Fast imax, i = ptr->fast_outs(imax); i < imax; i++) { 5402 Node* obs = ptr->fast_out(i); 5403 if (obs != this->map()) { 5404 return NULL; 5405 } 5406 } 5407 5408 // This arraycopy must unconditionally follow the allocation of the ptr. 5409 Node* alloc_ctl = ptr->in(0); 5410 Node* ctl = control(); 5411 while (ctl != alloc_ctl) { 5412 // There may be guards which feed into the slow_region. 5413 // Any other control flow means that we might not get a chance 5414 // to finish initializing the allocated object. 5415 // Various low-level checks bottom out in uncommon traps. These 5416 // are considered safe since we've already checked above that 5417 // there is no unexpected observer of this allocation. 5418 if (get_uncommon_trap_from_success_proj(ctl) != nullptr) { 5419 assert(ctl->in(0)->is_If(), "must be If"); 5420 ctl = ctl->in(0)->in(0); 5421 } else { 5422 return nullptr; 5423 } 5424 } 5425 5426 // If we get this far, we have an allocation which immediately 5427 // precedes the arraycopy, and we can take over zeroing the new object. 5428 // The arraycopy will finish the initialization, and provide 5429 // a new control state to which we will anchor the destination pointer. 5430 5431 return alloc; 5432 } 5433 5434 CallStaticJavaNode* LibraryCallKit::get_uncommon_trap_from_success_proj(Node* node) { 5435 if (node->is_IfProj()) { 5436 Node* other_proj = node->as_IfProj()->other_if_proj(); 5437 for (DUIterator_Fast jmax, j = other_proj->fast_outs(jmax); j < jmax; j++) { 5438 Node* obs = other_proj->fast_out(j); 5439 if (obs->in(0) == other_proj && obs->is_CallStaticJava() && 5440 (obs->as_CallStaticJava()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) { 5441 return obs->as_CallStaticJava(); 5442 } 5443 } 5444 } 5445 return nullptr; 5446 } 5447 5448 //-------------inline_encodeISOArray----------------------------------- 5449 // encode char[] to byte[] in ISO_8859_1 or ASCII 5450 bool LibraryCallKit::inline_encodeISOArray(bool ascii) { 5451 assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters"); 5452 // no receiver since it is static method 5453 Node *src = argument(0); 5454 Node *src_offset = argument(1); 5455 Node *dst = argument(2); 5456 Node *dst_offset = argument(3); 5457 Node *length = argument(4); 5458 5459 src = must_be_not_null(src, true); 5460 dst = must_be_not_null(dst, true); 5461 5462 const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr(); 5463 const TypeAryPtr* dst_type = dst->Value(&_gvn)->isa_aryptr(); 5464 if (src_type == nullptr || src_type->elem() == Type::BOTTOM || 5465 dst_type == nullptr || dst_type->elem() == Type::BOTTOM) { 5466 // failed array check 5467 return false; 5468 } 5469 5470 // Figure out the size and type of the elements we will be copying. 5471 BasicType src_elem = src_type->elem()->array_element_basic_type(); 5472 BasicType dst_elem = dst_type->elem()->array_element_basic_type(); 5473 if (!((src_elem == T_CHAR) || (src_elem== T_BYTE)) || dst_elem != T_BYTE) { 5474 return false; 5475 } 5476 5477 Node* src_start = array_element_address(src, src_offset, T_CHAR); 5478 Node* dst_start = array_element_address(dst, dst_offset, dst_elem); 5479 // 'src_start' points to src array + scaled offset 5480 // 'dst_start' points to dst array + scaled offset 5481 5482 const TypeAryPtr* mtype = TypeAryPtr::BYTES; 5483 Node* enc = new EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length, ascii); 5484 enc = _gvn.transform(enc); 5485 Node* res_mem = _gvn.transform(new SCMemProjNode(enc)); 5486 set_memory(res_mem, mtype); 5487 set_result(enc); 5488 clear_upper_avx(); 5489 5490 return true; 5491 } 5492 5493 //-------------inline_multiplyToLen----------------------------------- 5494 bool LibraryCallKit::inline_multiplyToLen() { 5495 assert(UseMultiplyToLenIntrinsic, "not implemented on this platform"); 5496 5497 address stubAddr = StubRoutines::multiplyToLen(); 5498 if (stubAddr == NULL) { 5499 return false; // Intrinsic's stub is not implemented on this platform 5500 } 5501 const char* stubName = "multiplyToLen"; 5502 5503 assert(callee()->signature()->size() == 5, "multiplyToLen has 5 parameters"); 5504 5505 // no receiver because it is a static method 5506 Node* x = argument(0); 5507 Node* xlen = argument(1); 5508 Node* y = argument(2); 5509 Node* ylen = argument(3); 5510 Node* z = argument(4); 5511 5512 x = must_be_not_null(x, true); 5513 y = must_be_not_null(y, true); 5514 5515 const TypeAryPtr* x_type = x->Value(&_gvn)->isa_aryptr(); 5516 const TypeAryPtr* y_type = y->Value(&_gvn)->isa_aryptr(); 5517 if (x_type == nullptr || x_type->elem() == Type::BOTTOM || 5518 y_type == nullptr || y_type->elem() == Type::BOTTOM) { 5519 // failed array check 5520 return false; 5521 } 5522 5523 BasicType x_elem = x_type->elem()->array_element_basic_type(); 5524 BasicType y_elem = y_type->elem()->array_element_basic_type(); 5525 if (x_elem != T_INT || y_elem != T_INT) { 5526 return false; 5527 } 5528 5529 // Set the original stack and the reexecute bit for the interpreter to reexecute 5530 // the bytecode that invokes BigInteger.multiplyToLen() if deoptimization happens 5531 // on the return from z array allocation in runtime. 5532 { PreserveReexecuteState preexecs(this); 5533 jvms()->set_should_reexecute(true); 5534 5535 Node* x_start = array_element_address(x, intcon(0), x_elem); 5536 Node* y_start = array_element_address(y, intcon(0), y_elem); 5537 // 'x_start' points to x array + scaled xlen 5538 // 'y_start' points to y array + scaled ylen 5539 5540 // Allocate the result array 5541 Node* zlen = _gvn.transform(new AddINode(xlen, ylen)); 5542 ciKlass* klass = ciTypeArrayKlass::make(T_INT); 5543 Node* klass_node = makecon(TypeKlassPtr::make(klass)); 5544 5545 IdealKit ideal(this); 5546 5547 #define __ ideal. 5548 Node* one = __ ConI(1); 5549 Node* zero = __ ConI(0); 5550 IdealVariable need_alloc(ideal), z_alloc(ideal); __ declarations_done(); 5551 __ set(need_alloc, zero); 5552 __ set(z_alloc, z); 5553 __ if_then(z, BoolTest::eq, null()); { 5554 __ increment (need_alloc, one); 5555 } __ else_(); { 5556 // Update graphKit memory and control from IdealKit. 5557 sync_kit(ideal); 5558 Node *cast = new CastPPNode(z, TypePtr::NOTNULL); 5559 cast->init_req(0, control()); 5560 _gvn.set_type(cast, cast->bottom_type()); 5561 C->record_for_igvn(cast); 5562 5563 Node* zlen_arg = load_array_length(cast); 5564 // Update IdealKit memory and control from graphKit. 5565 __ sync_kit(this); 5566 __ if_then(zlen_arg, BoolTest::lt, zlen); { 5567 __ increment (need_alloc, one); 5568 } __ end_if(); 5569 } __ end_if(); 5570 5571 __ if_then(__ value(need_alloc), BoolTest::ne, zero); { 5572 // Update graphKit memory and control from IdealKit. 5573 sync_kit(ideal); 5574 Node * narr = new_array(klass_node, zlen, 1); 5575 // Update IdealKit memory and control from graphKit. 5576 __ sync_kit(this); 5577 __ set(z_alloc, narr); 5578 } __ end_if(); 5579 5580 sync_kit(ideal); 5581 z = __ value(z_alloc); 5582 // Can't use TypeAryPtr::INTS which uses Bottom offset. 5583 _gvn.set_type(z, TypeOopPtr::make_from_klass(klass)); 5584 // Final sync IdealKit and GraphKit. 5585 final_sync(ideal); 5586 #undef __ 5587 5588 Node* z_start = array_element_address(z, intcon(0), T_INT); 5589 5590 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, 5591 OptoRuntime::multiplyToLen_Type(), 5592 stubAddr, stubName, TypePtr::BOTTOM, 5593 x_start, xlen, y_start, ylen, z_start, zlen); 5594 } // original reexecute is set back here 5595 5596 C->set_has_split_ifs(true); // Has chance for split-if optimization 5597 set_result(z); 5598 return true; 5599 } 5600 5601 //-------------inline_squareToLen------------------------------------ 5602 bool LibraryCallKit::inline_squareToLen() { 5603 assert(UseSquareToLenIntrinsic, "not implemented on this platform"); 5604 5605 address stubAddr = StubRoutines::squareToLen(); 5606 if (stubAddr == NULL) { 5607 return false; // Intrinsic's stub is not implemented on this platform 5608 } 5609 const char* stubName = "squareToLen"; 5610 5611 assert(callee()->signature()->size() == 4, "implSquareToLen has 4 parameters"); 5612 5613 Node* x = argument(0); 5614 Node* len = argument(1); 5615 Node* z = argument(2); 5616 Node* zlen = argument(3); 5617 5618 x = must_be_not_null(x, true); 5619 z = must_be_not_null(z, true); 5620 5621 const TypeAryPtr* x_type = x->Value(&_gvn)->isa_aryptr(); 5622 const TypeAryPtr* z_type = z->Value(&_gvn)->isa_aryptr(); 5623 if (x_type == nullptr || x_type->elem() == Type::BOTTOM || 5624 z_type == nullptr || z_type->elem() == Type::BOTTOM) { 5625 // failed array check 5626 return false; 5627 } 5628 5629 BasicType x_elem = x_type->elem()->array_element_basic_type(); 5630 BasicType z_elem = z_type->elem()->array_element_basic_type(); 5631 if (x_elem != T_INT || z_elem != T_INT) { 5632 return false; 5633 } 5634 5635 5636 Node* x_start = array_element_address(x, intcon(0), x_elem); 5637 Node* z_start = array_element_address(z, intcon(0), z_elem); 5638 5639 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, 5640 OptoRuntime::squareToLen_Type(), 5641 stubAddr, stubName, TypePtr::BOTTOM, 5642 x_start, len, z_start, zlen); 5643 5644 set_result(z); 5645 return true; 5646 } 5647 5648 //-------------inline_mulAdd------------------------------------------ 5649 bool LibraryCallKit::inline_mulAdd() { 5650 assert(UseMulAddIntrinsic, "not implemented on this platform"); 5651 5652 address stubAddr = StubRoutines::mulAdd(); 5653 if (stubAddr == NULL) { 5654 return false; // Intrinsic's stub is not implemented on this platform 5655 } 5656 const char* stubName = "mulAdd"; 5657 5658 assert(callee()->signature()->size() == 5, "mulAdd has 5 parameters"); 5659 5660 Node* out = argument(0); 5661 Node* in = argument(1); 5662 Node* offset = argument(2); 5663 Node* len = argument(3); 5664 Node* k = argument(4); 5665 5666 in = must_be_not_null(in, true); 5667 out = must_be_not_null(out, true); 5668 5669 const TypeAryPtr* out_type = out->Value(&_gvn)->isa_aryptr(); 5670 const TypeAryPtr* in_type = in->Value(&_gvn)->isa_aryptr(); 5671 if (out_type == nullptr || out_type->elem() == Type::BOTTOM || 5672 in_type == nullptr || in_type->elem() == Type::BOTTOM) { 5673 // failed array check 5674 return false; 5675 } 5676 5677 BasicType out_elem = out_type->elem()->array_element_basic_type(); 5678 BasicType in_elem = in_type->elem()->array_element_basic_type(); 5679 if (out_elem != T_INT || in_elem != T_INT) { 5680 return false; 5681 } 5682 5683 Node* outlen = load_array_length(out); 5684 Node* new_offset = _gvn.transform(new SubINode(outlen, offset)); 5685 Node* out_start = array_element_address(out, intcon(0), out_elem); 5686 Node* in_start = array_element_address(in, intcon(0), in_elem); 5687 5688 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, 5689 OptoRuntime::mulAdd_Type(), 5690 stubAddr, stubName, TypePtr::BOTTOM, 5691 out_start,in_start, new_offset, len, k); 5692 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 5693 set_result(result); 5694 return true; 5695 } 5696 5697 //-------------inline_montgomeryMultiply----------------------------------- 5698 bool LibraryCallKit::inline_montgomeryMultiply() { 5699 address stubAddr = StubRoutines::montgomeryMultiply(); 5700 if (stubAddr == NULL) { 5701 return false; // Intrinsic's stub is not implemented on this platform 5702 } 5703 5704 assert(UseMontgomeryMultiplyIntrinsic, "not implemented on this platform"); 5705 const char* stubName = "montgomery_multiply"; 5706 5707 assert(callee()->signature()->size() == 7, "montgomeryMultiply has 7 parameters"); 5708 5709 Node* a = argument(0); 5710 Node* b = argument(1); 5711 Node* n = argument(2); 5712 Node* len = argument(3); 5713 Node* inv = argument(4); 5714 Node* m = argument(6); 5715 5716 const TypeAryPtr* a_type = a->Value(&_gvn)->isa_aryptr(); 5717 const TypeAryPtr* b_type = b->Value(&_gvn)->isa_aryptr(); 5718 const TypeAryPtr* n_type = n->Value(&_gvn)->isa_aryptr(); 5719 const TypeAryPtr* m_type = m->Value(&_gvn)->isa_aryptr(); 5720 if (a_type == nullptr || a_type->elem() == Type::BOTTOM || 5721 b_type == nullptr || b_type->elem() == Type::BOTTOM || 5722 n_type == nullptr || n_type->elem() == Type::BOTTOM || 5723 m_type == nullptr || m_type->elem() == Type::BOTTOM) { 5724 // failed array check 5725 return false; 5726 } 5727 5728 BasicType a_elem = a_type->elem()->array_element_basic_type(); 5729 BasicType b_elem = b_type->elem()->array_element_basic_type(); 5730 BasicType n_elem = n_type->elem()->array_element_basic_type(); 5731 BasicType m_elem = m_type->elem()->array_element_basic_type(); 5732 if (a_elem != T_INT || b_elem != T_INT || n_elem != T_INT || m_elem != T_INT) { 5733 return false; 5734 } 5735 5736 // Make the call 5737 { 5738 Node* a_start = array_element_address(a, intcon(0), a_elem); 5739 Node* b_start = array_element_address(b, intcon(0), b_elem); 5740 Node* n_start = array_element_address(n, intcon(0), n_elem); 5741 Node* m_start = array_element_address(m, intcon(0), m_elem); 5742 5743 Node* call = make_runtime_call(RC_LEAF, 5744 OptoRuntime::montgomeryMultiply_Type(), 5745 stubAddr, stubName, TypePtr::BOTTOM, 5746 a_start, b_start, n_start, len, inv, top(), 5747 m_start); 5748 set_result(m); 5749 } 5750 5751 return true; 5752 } 5753 5754 bool LibraryCallKit::inline_montgomerySquare() { 5755 address stubAddr = StubRoutines::montgomerySquare(); 5756 if (stubAddr == NULL) { 5757 return false; // Intrinsic's stub is not implemented on this platform 5758 } 5759 5760 assert(UseMontgomerySquareIntrinsic, "not implemented on this platform"); 5761 const char* stubName = "montgomery_square"; 5762 5763 assert(callee()->signature()->size() == 6, "montgomerySquare has 6 parameters"); 5764 5765 Node* a = argument(0); 5766 Node* n = argument(1); 5767 Node* len = argument(2); 5768 Node* inv = argument(3); 5769 Node* m = argument(5); 5770 5771 const TypeAryPtr* a_type = a->Value(&_gvn)->isa_aryptr(); 5772 const TypeAryPtr* n_type = n->Value(&_gvn)->isa_aryptr(); 5773 const TypeAryPtr* m_type = m->Value(&_gvn)->isa_aryptr(); 5774 if (a_type == nullptr || a_type->elem() == Type::BOTTOM || 5775 n_type == nullptr || n_type->elem() == Type::BOTTOM || 5776 m_type == nullptr || m_type->elem() == Type::BOTTOM) { 5777 // failed array check 5778 return false; 5779 } 5780 5781 BasicType a_elem = a_type->elem()->array_element_basic_type(); 5782 BasicType n_elem = n_type->elem()->array_element_basic_type(); 5783 BasicType m_elem = m_type->elem()->array_element_basic_type(); 5784 if (a_elem != T_INT || n_elem != T_INT || m_elem != T_INT) { 5785 return false; 5786 } 5787 5788 // Make the call 5789 { 5790 Node* a_start = array_element_address(a, intcon(0), a_elem); 5791 Node* n_start = array_element_address(n, intcon(0), n_elem); 5792 Node* m_start = array_element_address(m, intcon(0), m_elem); 5793 5794 Node* call = make_runtime_call(RC_LEAF, 5795 OptoRuntime::montgomerySquare_Type(), 5796 stubAddr, stubName, TypePtr::BOTTOM, 5797 a_start, n_start, len, inv, top(), 5798 m_start); 5799 set_result(m); 5800 } 5801 5802 return true; 5803 } 5804 5805 bool LibraryCallKit::inline_bigIntegerShift(bool isRightShift) { 5806 address stubAddr = NULL; 5807 const char* stubName = NULL; 5808 5809 stubAddr = isRightShift? StubRoutines::bigIntegerRightShift(): StubRoutines::bigIntegerLeftShift(); 5810 if (stubAddr == NULL) { 5811 return false; // Intrinsic's stub is not implemented on this platform 5812 } 5813 5814 stubName = isRightShift? "bigIntegerRightShiftWorker" : "bigIntegerLeftShiftWorker"; 5815 5816 assert(callee()->signature()->size() == 5, "expected 5 arguments"); 5817 5818 Node* newArr = argument(0); 5819 Node* oldArr = argument(1); 5820 Node* newIdx = argument(2); 5821 Node* shiftCount = argument(3); 5822 Node* numIter = argument(4); 5823 5824 const TypeAryPtr* newArr_type = newArr->Value(&_gvn)->isa_aryptr(); 5825 const TypeAryPtr* oldArr_type = oldArr->Value(&_gvn)->isa_aryptr(); 5826 if (newArr_type == nullptr || newArr_type->elem() == Type::BOTTOM || 5827 oldArr_type == nullptr || oldArr_type->elem() == Type::BOTTOM) { 5828 return false; 5829 } 5830 5831 BasicType newArr_elem = newArr_type->elem()->array_element_basic_type(); 5832 BasicType oldArr_elem = oldArr_type->elem()->array_element_basic_type(); 5833 if (newArr_elem != T_INT || oldArr_elem != T_INT) { 5834 return false; 5835 } 5836 5837 // Make the call 5838 { 5839 Node* newArr_start = array_element_address(newArr, intcon(0), newArr_elem); 5840 Node* oldArr_start = array_element_address(oldArr, intcon(0), oldArr_elem); 5841 5842 Node* call = make_runtime_call(RC_LEAF, 5843 OptoRuntime::bigIntegerShift_Type(), 5844 stubAddr, 5845 stubName, 5846 TypePtr::BOTTOM, 5847 newArr_start, 5848 oldArr_start, 5849 newIdx, 5850 shiftCount, 5851 numIter); 5852 } 5853 5854 return true; 5855 } 5856 5857 //-------------inline_vectorizedMismatch------------------------------ 5858 bool LibraryCallKit::inline_vectorizedMismatch() { 5859 assert(UseVectorizedMismatchIntrinsic, "not implemented on this platform"); 5860 5861 assert(callee()->signature()->size() == 8, "vectorizedMismatch has 6 parameters"); 5862 Node* obja = argument(0); // Object 5863 Node* aoffset = argument(1); // long 5864 Node* objb = argument(3); // Object 5865 Node* boffset = argument(4); // long 5866 Node* length = argument(6); // int 5867 Node* scale = argument(7); // int 5868 5869 const TypeAryPtr* obja_t = _gvn.type(obja)->isa_aryptr(); 5870 const TypeAryPtr* objb_t = _gvn.type(objb)->isa_aryptr(); 5871 if (obja_t == NULL || obja_t->elem() == Type::BOTTOM || 5872 objb_t == NULL || objb_t->elem() == Type::BOTTOM || 5873 scale == top()) { 5874 return false; // failed input validation 5875 } 5876 5877 Node* obja_adr = make_unsafe_address(obja, aoffset); 5878 Node* objb_adr = make_unsafe_address(objb, boffset); 5879 5880 // Partial inlining handling for inputs smaller than ArrayOperationPartialInlineSize bytes in size. 5881 // 5882 // inline_limit = ArrayOperationPartialInlineSize / element_size; 5883 // if (length <= inline_limit) { 5884 // inline_path: 5885 // vmask = VectorMaskGen length 5886 // vload1 = LoadVectorMasked obja, vmask 5887 // vload2 = LoadVectorMasked objb, vmask 5888 // result1 = VectorCmpMasked vload1, vload2, vmask 5889 // } else { 5890 // call_stub_path: 5891 // result2 = call vectorizedMismatch_stub(obja, objb, length, scale) 5892 // } 5893 // exit_block: 5894 // return Phi(result1, result2); 5895 // 5896 enum { inline_path = 1, // input is small enough to process it all at once 5897 stub_path = 2, // input is too large; call into the VM 5898 PATH_LIMIT = 3 5899 }; 5900 5901 Node* exit_block = new RegionNode(PATH_LIMIT); 5902 Node* result_phi = new PhiNode(exit_block, TypeInt::INT); 5903 Node* memory_phi = new PhiNode(exit_block, Type::MEMORY, TypePtr::BOTTOM); 5904 5905 Node* call_stub_path = control(); 5906 5907 BasicType elem_bt = T_ILLEGAL; 5908 5909 const TypeInt* scale_t = _gvn.type(scale)->is_int(); 5910 if (scale_t->is_con()) { 5911 switch (scale_t->get_con()) { 5912 case 0: elem_bt = T_BYTE; break; 5913 case 1: elem_bt = T_SHORT; break; 5914 case 2: elem_bt = T_INT; break; 5915 case 3: elem_bt = T_LONG; break; 5916 5917 default: elem_bt = T_ILLEGAL; break; // not supported 5918 } 5919 } 5920 5921 int inline_limit = 0; 5922 bool do_partial_inline = false; 5923 5924 if (elem_bt != T_ILLEGAL && ArrayOperationPartialInlineSize > 0) { 5925 inline_limit = ArrayOperationPartialInlineSize / type2aelembytes(elem_bt); 5926 do_partial_inline = inline_limit >= 16; 5927 } 5928 5929 if (do_partial_inline) { 5930 assert(elem_bt != T_ILLEGAL, "sanity"); 5931 5932 if (Matcher::match_rule_supported_vector(Op_VectorMaskGen, inline_limit, elem_bt) && 5933 Matcher::match_rule_supported_vector(Op_LoadVectorMasked, inline_limit, elem_bt) && 5934 Matcher::match_rule_supported_vector(Op_VectorCmpMasked, inline_limit, elem_bt)) { 5935 5936 const TypeVect* vt = TypeVect::make(elem_bt, inline_limit); 5937 Node* cmp_length = _gvn.transform(new CmpINode(length, intcon(inline_limit))); 5938 Node* bol_gt = _gvn.transform(new BoolNode(cmp_length, BoolTest::gt)); 5939 5940 call_stub_path = generate_guard(bol_gt, NULL, PROB_MIN); 5941 5942 if (!stopped()) { 5943 Node* casted_length = _gvn.transform(new CastIINode(control(), length, TypeInt::make(0, inline_limit, Type::WidenMin))); 5944 5945 const TypePtr* obja_adr_t = _gvn.type(obja_adr)->isa_ptr(); 5946 const TypePtr* objb_adr_t = _gvn.type(objb_adr)->isa_ptr(); 5947 Node* obja_adr_mem = memory(C->get_alias_index(obja_adr_t)); 5948 Node* objb_adr_mem = memory(C->get_alias_index(objb_adr_t)); 5949 5950 Node* vmask = _gvn.transform(VectorMaskGenNode::make(ConvI2X(casted_length), elem_bt)); 5951 Node* vload_obja = _gvn.transform(new LoadVectorMaskedNode(control(), obja_adr_mem, obja_adr, obja_adr_t, vt, vmask)); 5952 Node* vload_objb = _gvn.transform(new LoadVectorMaskedNode(control(), objb_adr_mem, objb_adr, objb_adr_t, vt, vmask)); 5953 Node* result = _gvn.transform(new VectorCmpMaskedNode(vload_obja, vload_objb, vmask, TypeInt::INT)); 5954 5955 exit_block->init_req(inline_path, control()); 5956 memory_phi->init_req(inline_path, map()->memory()); 5957 result_phi->init_req(inline_path, result); 5958 5959 C->set_max_vector_size(MAX2((uint)ArrayOperationPartialInlineSize, C->max_vector_size())); 5960 clear_upper_avx(); 5961 } 5962 } 5963 } 5964 5965 if (call_stub_path != NULL) { 5966 set_control(call_stub_path); 5967 5968 Node* call = make_runtime_call(RC_LEAF, 5969 OptoRuntime::vectorizedMismatch_Type(), 5970 StubRoutines::vectorizedMismatch(), "vectorizedMismatch", TypePtr::BOTTOM, 5971 obja_adr, objb_adr, length, scale); 5972 5973 exit_block->init_req(stub_path, control()); 5974 memory_phi->init_req(stub_path, map()->memory()); 5975 result_phi->init_req(stub_path, _gvn.transform(new ProjNode(call, TypeFunc::Parms))); 5976 } 5977 5978 exit_block = _gvn.transform(exit_block); 5979 memory_phi = _gvn.transform(memory_phi); 5980 result_phi = _gvn.transform(result_phi); 5981 5982 set_control(exit_block); 5983 set_all_memory(memory_phi); 5984 set_result(result_phi); 5985 5986 return true; 5987 } 5988 5989 //------------------------------inline_vectorizedHashcode---------------------------- 5990 bool LibraryCallKit::inline_vectorizedHashCode() { 5991 assert(UseVectorizedHashCodeIntrinsic, "not implemented on this platform"); 5992 5993 assert(callee()->signature()->size() == 5, "vectorizedHashCode has 5 parameters"); 5994 Node* array = argument(0); 5995 Node* offset = argument(1); 5996 Node* length = argument(2); 5997 Node* initialValue = argument(3); 5998 Node* basic_type = argument(4); 5999 6000 if (basic_type == top()) { 6001 return false; // failed input validation 6002 } 6003 6004 const TypeInt* basic_type_t = _gvn.type(basic_type)->is_int(); 6005 if (!basic_type_t->is_con()) { 6006 return false; // Only intrinsify if mode argument is constant 6007 } 6008 6009 array = must_be_not_null(array, true); 6010 6011 BasicType bt = (BasicType)basic_type_t->get_con(); 6012 6013 // Resolve address of first element 6014 Node* array_start = array_element_address(array, offset, bt); 6015 6016 set_result(_gvn.transform(new VectorizedHashCodeNode(control(), memory(TypeAryPtr::get_array_body_type(bt)), 6017 array_start, length, initialValue, basic_type))); 6018 clear_upper_avx(); 6019 6020 return true; 6021 } 6022 6023 /** 6024 * Calculate CRC32 for byte. 6025 * int java.util.zip.CRC32.update(int crc, int b) 6026 */ 6027 bool LibraryCallKit::inline_updateCRC32() { 6028 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support"); 6029 assert(callee()->signature()->size() == 2, "update has 2 parameters"); 6030 // no receiver since it is static method 6031 Node* crc = argument(0); // type: int 6032 Node* b = argument(1); // type: int 6033 6034 /* 6035 * int c = ~ crc; 6036 * b = timesXtoThe32[(b ^ c) & 0xFF]; 6037 * b = b ^ (c >>> 8); 6038 * crc = ~b; 6039 */ 6040 6041 Node* M1 = intcon(-1); 6042 crc = _gvn.transform(new XorINode(crc, M1)); 6043 Node* result = _gvn.transform(new XorINode(crc, b)); 6044 result = _gvn.transform(new AndINode(result, intcon(0xFF))); 6045 6046 Node* base = makecon(TypeRawPtr::make(StubRoutines::crc_table_addr())); 6047 Node* offset = _gvn.transform(new LShiftINode(result, intcon(0x2))); 6048 Node* adr = basic_plus_adr(top(), base, ConvI2X(offset)); 6049 result = make_load(control(), adr, TypeInt::INT, T_INT, MemNode::unordered); 6050 6051 crc = _gvn.transform(new URShiftINode(crc, intcon(8))); 6052 result = _gvn.transform(new XorINode(crc, result)); 6053 result = _gvn.transform(new XorINode(result, M1)); 6054 set_result(result); 6055 return true; 6056 } 6057 6058 /** 6059 * Calculate CRC32 for byte[] array. 6060 * int java.util.zip.CRC32.updateBytes(int crc, byte[] buf, int off, int len) 6061 */ 6062 bool LibraryCallKit::inline_updateBytesCRC32() { 6063 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support"); 6064 assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters"); 6065 // no receiver since it is static method 6066 Node* crc = argument(0); // type: int 6067 Node* src = argument(1); // type: oop 6068 Node* offset = argument(2); // type: int 6069 Node* length = argument(3); // type: int 6070 6071 const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr(); 6072 if (src_type == nullptr || src_type->elem() == Type::BOTTOM) { 6073 // failed array check 6074 return false; 6075 } 6076 6077 // Figure out the size and type of the elements we will be copying. 6078 BasicType src_elem = src_type->elem()->array_element_basic_type(); 6079 if (src_elem != T_BYTE) { 6080 return false; 6081 } 6082 6083 // 'src_start' points to src array + scaled offset 6084 src = must_be_not_null(src, true); 6085 Node* src_start = array_element_address(src, offset, src_elem); 6086 6087 // We assume that range check is done by caller. 6088 // TODO: generate range check (offset+length < src.length) in debug VM. 6089 6090 // Call the stub. 6091 address stubAddr = StubRoutines::updateBytesCRC32(); 6092 const char *stubName = "updateBytesCRC32"; 6093 6094 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(), 6095 stubAddr, stubName, TypePtr::BOTTOM, 6096 crc, src_start, length); 6097 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 6098 set_result(result); 6099 return true; 6100 } 6101 6102 /** 6103 * Calculate CRC32 for ByteBuffer. 6104 * int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len) 6105 */ 6106 bool LibraryCallKit::inline_updateByteBufferCRC32() { 6107 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support"); 6108 assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long"); 6109 // no receiver since it is static method 6110 Node* crc = argument(0); // type: int 6111 Node* src = argument(1); // type: long 6112 Node* offset = argument(3); // type: int 6113 Node* length = argument(4); // type: int 6114 6115 src = ConvL2X(src); // adjust Java long to machine word 6116 Node* base = _gvn.transform(new CastX2PNode(src)); 6117 offset = ConvI2X(offset); 6118 6119 // 'src_start' points to src array + scaled offset 6120 Node* src_start = basic_plus_adr(top(), base, offset); 6121 6122 // Call the stub. 6123 address stubAddr = StubRoutines::updateBytesCRC32(); 6124 const char *stubName = "updateBytesCRC32"; 6125 6126 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(), 6127 stubAddr, stubName, TypePtr::BOTTOM, 6128 crc, src_start, length); 6129 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 6130 set_result(result); 6131 return true; 6132 } 6133 6134 //------------------------------get_table_from_crc32c_class----------------------- 6135 Node * LibraryCallKit::get_table_from_crc32c_class(ciInstanceKlass *crc32c_class) { 6136 Node* table = load_field_from_object(NULL, "byteTable", "[I", /*decorators*/ IN_HEAP, /*is_static*/ true, crc32c_class); 6137 assert (table != NULL, "wrong version of java.util.zip.CRC32C"); 6138 6139 return table; 6140 } 6141 6142 //------------------------------inline_updateBytesCRC32C----------------------- 6143 // 6144 // Calculate CRC32C for byte[] array. 6145 // int java.util.zip.CRC32C.updateBytes(int crc, byte[] buf, int off, int end) 6146 // 6147 bool LibraryCallKit::inline_updateBytesCRC32C() { 6148 assert(UseCRC32CIntrinsics, "need CRC32C instruction support"); 6149 assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters"); 6150 assert(callee()->holder()->is_loaded(), "CRC32C class must be loaded"); 6151 // no receiver since it is a static method 6152 Node* crc = argument(0); // type: int 6153 Node* src = argument(1); // type: oop 6154 Node* offset = argument(2); // type: int 6155 Node* end = argument(3); // type: int 6156 6157 Node* length = _gvn.transform(new SubINode(end, offset)); 6158 6159 const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr(); 6160 if (src_type == nullptr || src_type->elem() == Type::BOTTOM) { 6161 // failed array check 6162 return false; 6163 } 6164 6165 // Figure out the size and type of the elements we will be copying. 6166 BasicType src_elem = src_type->elem()->array_element_basic_type(); 6167 if (src_elem != T_BYTE) { 6168 return false; 6169 } 6170 6171 // 'src_start' points to src array + scaled offset 6172 src = must_be_not_null(src, true); 6173 Node* src_start = array_element_address(src, offset, src_elem); 6174 6175 // static final int[] byteTable in class CRC32C 6176 Node* table = get_table_from_crc32c_class(callee()->holder()); 6177 table = must_be_not_null(table, true); 6178 Node* table_start = array_element_address(table, intcon(0), T_INT); 6179 6180 // We assume that range check is done by caller. 6181 // TODO: generate range check (offset+length < src.length) in debug VM. 6182 6183 // Call the stub. 6184 address stubAddr = StubRoutines::updateBytesCRC32C(); 6185 const char *stubName = "updateBytesCRC32C"; 6186 6187 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesCRC32C_Type(), 6188 stubAddr, stubName, TypePtr::BOTTOM, 6189 crc, src_start, length, table_start); 6190 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 6191 set_result(result); 6192 return true; 6193 } 6194 6195 //------------------------------inline_updateDirectByteBufferCRC32C----------------------- 6196 // 6197 // Calculate CRC32C for DirectByteBuffer. 6198 // int java.util.zip.CRC32C.updateDirectByteBuffer(int crc, long buf, int off, int end) 6199 // 6200 bool LibraryCallKit::inline_updateDirectByteBufferCRC32C() { 6201 assert(UseCRC32CIntrinsics, "need CRC32C instruction support"); 6202 assert(callee()->signature()->size() == 5, "updateDirectByteBuffer has 4 parameters and one is long"); 6203 assert(callee()->holder()->is_loaded(), "CRC32C class must be loaded"); 6204 // no receiver since it is a static method 6205 Node* crc = argument(0); // type: int 6206 Node* src = argument(1); // type: long 6207 Node* offset = argument(3); // type: int 6208 Node* end = argument(4); // type: int 6209 6210 Node* length = _gvn.transform(new SubINode(end, offset)); 6211 6212 src = ConvL2X(src); // adjust Java long to machine word 6213 Node* base = _gvn.transform(new CastX2PNode(src)); 6214 offset = ConvI2X(offset); 6215 6216 // 'src_start' points to src array + scaled offset 6217 Node* src_start = basic_plus_adr(top(), base, offset); 6218 6219 // static final int[] byteTable in class CRC32C 6220 Node* table = get_table_from_crc32c_class(callee()->holder()); 6221 table = must_be_not_null(table, true); 6222 Node* table_start = array_element_address(table, intcon(0), T_INT); 6223 6224 // Call the stub. 6225 address stubAddr = StubRoutines::updateBytesCRC32C(); 6226 const char *stubName = "updateBytesCRC32C"; 6227 6228 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesCRC32C_Type(), 6229 stubAddr, stubName, TypePtr::BOTTOM, 6230 crc, src_start, length, table_start); 6231 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 6232 set_result(result); 6233 return true; 6234 } 6235 6236 //------------------------------inline_updateBytesAdler32---------------------- 6237 // 6238 // Calculate Adler32 checksum for byte[] array. 6239 // int java.util.zip.Adler32.updateBytes(int crc, byte[] buf, int off, int len) 6240 // 6241 bool LibraryCallKit::inline_updateBytesAdler32() { 6242 assert(UseAdler32Intrinsics, "Adler32 Intrinsic support need"); // check if we actually need to check this flag or check a different one 6243 assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters"); 6244 assert(callee()->holder()->is_loaded(), "Adler32 class must be loaded"); 6245 // no receiver since it is static method 6246 Node* crc = argument(0); // type: int 6247 Node* src = argument(1); // type: oop 6248 Node* offset = argument(2); // type: int 6249 Node* length = argument(3); // type: int 6250 6251 const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr(); 6252 if (src_type == nullptr || src_type->elem() == Type::BOTTOM) { 6253 // failed array check 6254 return false; 6255 } 6256 6257 // Figure out the size and type of the elements we will be copying. 6258 BasicType src_elem = src_type->elem()->array_element_basic_type(); 6259 if (src_elem != T_BYTE) { 6260 return false; 6261 } 6262 6263 // 'src_start' points to src array + scaled offset 6264 Node* src_start = array_element_address(src, offset, src_elem); 6265 6266 // We assume that range check is done by caller. 6267 // TODO: generate range check (offset+length < src.length) in debug VM. 6268 6269 // Call the stub. 6270 address stubAddr = StubRoutines::updateBytesAdler32(); 6271 const char *stubName = "updateBytesAdler32"; 6272 6273 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesAdler32_Type(), 6274 stubAddr, stubName, TypePtr::BOTTOM, 6275 crc, src_start, length); 6276 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 6277 set_result(result); 6278 return true; 6279 } 6280 6281 //------------------------------inline_updateByteBufferAdler32--------------- 6282 // 6283 // Calculate Adler32 checksum for DirectByteBuffer. 6284 // int java.util.zip.Adler32.updateByteBuffer(int crc, long buf, int off, int len) 6285 // 6286 bool LibraryCallKit::inline_updateByteBufferAdler32() { 6287 assert(UseAdler32Intrinsics, "Adler32 Intrinsic support need"); // check if we actually need to check this flag or check a different one 6288 assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long"); 6289 assert(callee()->holder()->is_loaded(), "Adler32 class must be loaded"); 6290 // no receiver since it is static method 6291 Node* crc = argument(0); // type: int 6292 Node* src = argument(1); // type: long 6293 Node* offset = argument(3); // type: int 6294 Node* length = argument(4); // type: int 6295 6296 src = ConvL2X(src); // adjust Java long to machine word 6297 Node* base = _gvn.transform(new CastX2PNode(src)); 6298 offset = ConvI2X(offset); 6299 6300 // 'src_start' points to src array + scaled offset 6301 Node* src_start = basic_plus_adr(top(), base, offset); 6302 6303 // Call the stub. 6304 address stubAddr = StubRoutines::updateBytesAdler32(); 6305 const char *stubName = "updateBytesAdler32"; 6306 6307 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesAdler32_Type(), 6308 stubAddr, stubName, TypePtr::BOTTOM, 6309 crc, src_start, length); 6310 6311 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 6312 set_result(result); 6313 return true; 6314 } 6315 6316 //----------------------------inline_reference_get---------------------------- 6317 // public T java.lang.ref.Reference.get(); 6318 bool LibraryCallKit::inline_reference_get() { 6319 const int referent_offset = java_lang_ref_Reference::referent_offset(); 6320 6321 // Get the argument: 6322 Node* reference_obj = null_check_receiver(); 6323 if (stopped()) return true; 6324 6325 DecoratorSet decorators = IN_HEAP | ON_WEAK_OOP_REF; 6326 Node* result = load_field_from_object(reference_obj, "referent", "Ljava/lang/Object;", 6327 decorators, /*is_static*/ false, NULL); 6328 if (result == NULL) return false; 6329 6330 // Add memory barrier to prevent commoning reads from this field 6331 // across safepoint since GC can change its value. 6332 insert_mem_bar(Op_MemBarCPUOrder); 6333 6334 set_result(result); 6335 return true; 6336 } 6337 6338 //----------------------------inline_reference_refersTo0---------------------------- 6339 // bool java.lang.ref.Reference.refersTo0(); 6340 // bool java.lang.ref.PhantomReference.refersTo0(); 6341 bool LibraryCallKit::inline_reference_refersTo0(bool is_phantom) { 6342 // Get arguments: 6343 Node* reference_obj = null_check_receiver(); 6344 Node* other_obj = argument(1); 6345 if (stopped()) return true; 6346 6347 DecoratorSet decorators = IN_HEAP | AS_NO_KEEPALIVE; 6348 decorators |= (is_phantom ? ON_PHANTOM_OOP_REF : ON_WEAK_OOP_REF); 6349 Node* referent = load_field_from_object(reference_obj, "referent", "Ljava/lang/Object;", 6350 decorators, /*is_static*/ false, NULL); 6351 if (referent == NULL) return false; 6352 6353 // Add memory barrier to prevent commoning reads from this field 6354 // across safepoint since GC can change its value. 6355 insert_mem_bar(Op_MemBarCPUOrder); 6356 6357 Node* cmp = _gvn.transform(new CmpPNode(referent, other_obj)); 6358 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq)); 6359 IfNode* if_node = create_and_map_if(control(), bol, PROB_FAIR, COUNT_UNKNOWN); 6360 6361 RegionNode* region = new RegionNode(3); 6362 PhiNode* phi = new PhiNode(region, TypeInt::BOOL); 6363 6364 Node* if_true = _gvn.transform(new IfTrueNode(if_node)); 6365 region->init_req(1, if_true); 6366 phi->init_req(1, intcon(1)); 6367 6368 Node* if_false = _gvn.transform(new IfFalseNode(if_node)); 6369 region->init_req(2, if_false); 6370 phi->init_req(2, intcon(0)); 6371 6372 set_control(_gvn.transform(region)); 6373 record_for_igvn(region); 6374 set_result(_gvn.transform(phi)); 6375 return true; 6376 } 6377 6378 6379 Node* LibraryCallKit::load_field_from_object(Node* fromObj, const char* fieldName, const char* fieldTypeString, 6380 DecoratorSet decorators, bool is_static, 6381 ciInstanceKlass* fromKls) { 6382 if (fromKls == NULL) { 6383 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr(); 6384 assert(tinst != NULL, "obj is null"); 6385 assert(tinst->is_loaded(), "obj is not loaded"); 6386 fromKls = tinst->instance_klass(); 6387 } else { 6388 assert(is_static, "only for static field access"); 6389 } 6390 ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName), 6391 ciSymbol::make(fieldTypeString), 6392 is_static); 6393 6394 assert(field != NULL, "undefined field %s %s %s", fieldTypeString, fromKls->name()->as_utf8(), fieldName); 6395 if (field == NULL) return (Node *) NULL; 6396 6397 if (is_static) { 6398 const TypeInstPtr* tip = TypeInstPtr::make(fromKls->java_mirror()); 6399 fromObj = makecon(tip); 6400 } 6401 6402 // Next code copied from Parse::do_get_xxx(): 6403 6404 // Compute address and memory type. 6405 int offset = field->offset_in_bytes(); 6406 bool is_vol = field->is_volatile(); 6407 ciType* field_klass = field->type(); 6408 assert(field_klass->is_loaded(), "should be loaded"); 6409 const TypePtr* adr_type = C->alias_type(field)->adr_type(); 6410 Node *adr = basic_plus_adr(fromObj, fromObj, offset); 6411 BasicType bt = field->layout_type(); 6412 6413 // Build the resultant type of the load 6414 const Type *type; 6415 if (bt == T_OBJECT) { 6416 type = TypeOopPtr::make_from_klass(field_klass->as_klass()); 6417 } else { 6418 type = Type::get_const_basic_type(bt); 6419 } 6420 6421 if (is_vol) { 6422 decorators |= MO_SEQ_CST; 6423 } 6424 6425 return access_load_at(fromObj, adr, adr_type, type, bt, decorators); 6426 } 6427 6428 Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, 6429 bool is_exact /* true */, bool is_static /* false */, 6430 ciInstanceKlass * fromKls /* NULL */) { 6431 if (fromKls == NULL) { 6432 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr(); 6433 assert(tinst != NULL, "obj is null"); 6434 assert(tinst->is_loaded(), "obj is not loaded"); 6435 assert(!is_exact || tinst->klass_is_exact(), "klass not exact"); 6436 fromKls = tinst->instance_klass(); 6437 } 6438 else { 6439 assert(is_static, "only for static field access"); 6440 } 6441 ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName), 6442 ciSymbol::make(fieldTypeString), 6443 is_static); 6444 6445 assert(field != NULL, "undefined field"); 6446 assert(!field->is_volatile(), "not defined for volatile fields"); 6447 6448 if (is_static) { 6449 const TypeInstPtr* tip = TypeInstPtr::make(fromKls->java_mirror()); 6450 fromObj = makecon(tip); 6451 } 6452 6453 // Next code copied from Parse::do_get_xxx(): 6454 6455 // Compute address and memory type. 6456 int offset = field->offset_in_bytes(); 6457 Node *adr = basic_plus_adr(fromObj, fromObj, offset); 6458 6459 return adr; 6460 } 6461 6462 //------------------------------inline_aescrypt_Block----------------------- 6463 bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) { 6464 address stubAddr = NULL; 6465 const char *stubName; 6466 assert(UseAES, "need AES instruction support"); 6467 6468 switch(id) { 6469 case vmIntrinsics::_aescrypt_encryptBlock: 6470 stubAddr = StubRoutines::aescrypt_encryptBlock(); 6471 stubName = "aescrypt_encryptBlock"; 6472 break; 6473 case vmIntrinsics::_aescrypt_decryptBlock: 6474 stubAddr = StubRoutines::aescrypt_decryptBlock(); 6475 stubName = "aescrypt_decryptBlock"; 6476 break; 6477 default: 6478 break; 6479 } 6480 if (stubAddr == NULL) return false; 6481 6482 Node* aescrypt_object = argument(0); 6483 Node* src = argument(1); 6484 Node* src_offset = argument(2); 6485 Node* dest = argument(3); 6486 Node* dest_offset = argument(4); 6487 6488 src = must_be_not_null(src, true); 6489 dest = must_be_not_null(dest, true); 6490 6491 // (1) src and dest are arrays. 6492 const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr(); 6493 const TypeAryPtr* dest_type = dest->Value(&_gvn)->isa_aryptr(); 6494 assert( src_type != nullptr && src_type->elem() != Type::BOTTOM && 6495 dest_type != nullptr && dest_type->elem() != Type::BOTTOM, "args are strange"); 6496 6497 // for the quick and dirty code we will skip all the checks. 6498 // we are just trying to get the call to be generated. 6499 Node* src_start = src; 6500 Node* dest_start = dest; 6501 if (src_offset != NULL || dest_offset != NULL) { 6502 assert(src_offset != NULL && dest_offset != NULL, ""); 6503 src_start = array_element_address(src, src_offset, T_BYTE); 6504 dest_start = array_element_address(dest, dest_offset, T_BYTE); 6505 } 6506 6507 // now need to get the start of its expanded key array 6508 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java 6509 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); 6510 if (k_start == NULL) return false; 6511 6512 // Call the stub. 6513 make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(), 6514 stubAddr, stubName, TypePtr::BOTTOM, 6515 src_start, dest_start, k_start); 6516 6517 return true; 6518 } 6519 6520 //------------------------------inline_cipherBlockChaining_AESCrypt----------------------- 6521 bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) { 6522 address stubAddr = NULL; 6523 const char *stubName = NULL; 6524 6525 assert(UseAES, "need AES instruction support"); 6526 6527 switch(id) { 6528 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt: 6529 stubAddr = StubRoutines::cipherBlockChaining_encryptAESCrypt(); 6530 stubName = "cipherBlockChaining_encryptAESCrypt"; 6531 break; 6532 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt: 6533 stubAddr = StubRoutines::cipherBlockChaining_decryptAESCrypt(); 6534 stubName = "cipherBlockChaining_decryptAESCrypt"; 6535 break; 6536 default: 6537 break; 6538 } 6539 if (stubAddr == NULL) return false; 6540 6541 Node* cipherBlockChaining_object = argument(0); 6542 Node* src = argument(1); 6543 Node* src_offset = argument(2); 6544 Node* len = argument(3); 6545 Node* dest = argument(4); 6546 Node* dest_offset = argument(5); 6547 6548 src = must_be_not_null(src, false); 6549 dest = must_be_not_null(dest, false); 6550 6551 // (1) src and dest are arrays. 6552 const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr(); 6553 const TypeAryPtr* dest_type = dest->Value(&_gvn)->isa_aryptr(); 6554 assert( src_type != nullptr && src_type->elem() != Type::BOTTOM && 6555 dest_type != nullptr && dest_type->elem() != Type::BOTTOM, "args are strange"); 6556 6557 // checks are the responsibility of the caller 6558 Node* src_start = src; 6559 Node* dest_start = dest; 6560 if (src_offset != NULL || dest_offset != NULL) { 6561 assert(src_offset != NULL && dest_offset != NULL, ""); 6562 src_start = array_element_address(src, src_offset, T_BYTE); 6563 dest_start = array_element_address(dest, dest_offset, T_BYTE); 6564 } 6565 6566 // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object 6567 // (because of the predicated logic executed earlier). 6568 // so we cast it here safely. 6569 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java 6570 6571 Node* embeddedCipherObj = load_field_from_object(cipherBlockChaining_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6572 if (embeddedCipherObj == NULL) return false; 6573 6574 // cast it to what we know it will be at runtime 6575 const TypeInstPtr* tinst = _gvn.type(cipherBlockChaining_object)->isa_instptr(); 6576 assert(tinst != NULL, "CBC obj is null"); 6577 assert(tinst->is_loaded(), "CBC obj is not loaded"); 6578 ciKlass* klass_AESCrypt = tinst->instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6579 assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded"); 6580 6581 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6582 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt); 6583 const TypeOopPtr* xtype = aklass->as_instance_type()->cast_to_ptr_type(TypePtr::NotNull); 6584 Node* aescrypt_object = new CheckCastPPNode(control(), embeddedCipherObj, xtype); 6585 aescrypt_object = _gvn.transform(aescrypt_object); 6586 6587 // we need to get the start of the aescrypt_object's expanded key array 6588 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); 6589 if (k_start == NULL) return false; 6590 6591 // similarly, get the start address of the r vector 6592 Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B"); 6593 if (objRvec == NULL) return false; 6594 Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE); 6595 6596 // Call the stub, passing src_start, dest_start, k_start, r_start and src_len 6597 Node* cbcCrypt = make_runtime_call(RC_LEAF|RC_NO_FP, 6598 OptoRuntime::cipherBlockChaining_aescrypt_Type(), 6599 stubAddr, stubName, TypePtr::BOTTOM, 6600 src_start, dest_start, k_start, r_start, len); 6601 6602 // return cipher length (int) 6603 Node* retvalue = _gvn.transform(new ProjNode(cbcCrypt, TypeFunc::Parms)); 6604 set_result(retvalue); 6605 return true; 6606 } 6607 6608 //------------------------------inline_electronicCodeBook_AESCrypt----------------------- 6609 bool LibraryCallKit::inline_electronicCodeBook_AESCrypt(vmIntrinsics::ID id) { 6610 address stubAddr = NULL; 6611 const char *stubName = NULL; 6612 6613 assert(UseAES, "need AES instruction support"); 6614 6615 switch (id) { 6616 case vmIntrinsics::_electronicCodeBook_encryptAESCrypt: 6617 stubAddr = StubRoutines::electronicCodeBook_encryptAESCrypt(); 6618 stubName = "electronicCodeBook_encryptAESCrypt"; 6619 break; 6620 case vmIntrinsics::_electronicCodeBook_decryptAESCrypt: 6621 stubAddr = StubRoutines::electronicCodeBook_decryptAESCrypt(); 6622 stubName = "electronicCodeBook_decryptAESCrypt"; 6623 break; 6624 default: 6625 break; 6626 } 6627 6628 if (stubAddr == NULL) return false; 6629 6630 Node* electronicCodeBook_object = argument(0); 6631 Node* src = argument(1); 6632 Node* src_offset = argument(2); 6633 Node* len = argument(3); 6634 Node* dest = argument(4); 6635 Node* dest_offset = argument(5); 6636 6637 // (1) src and dest are arrays. 6638 const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr(); 6639 const TypeAryPtr* dest_type = dest->Value(&_gvn)->isa_aryptr(); 6640 assert( src_type != nullptr && src_type->elem() != Type::BOTTOM && 6641 dest_type != nullptr && dest_type->elem() != Type::BOTTOM, "args are strange"); 6642 6643 // checks are the responsibility of the caller 6644 Node* src_start = src; 6645 Node* dest_start = dest; 6646 if (src_offset != NULL || dest_offset != NULL) { 6647 assert(src_offset != NULL && dest_offset != NULL, ""); 6648 src_start = array_element_address(src, src_offset, T_BYTE); 6649 dest_start = array_element_address(dest, dest_offset, T_BYTE); 6650 } 6651 6652 // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object 6653 // (because of the predicated logic executed earlier). 6654 // so we cast it here safely. 6655 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java 6656 6657 Node* embeddedCipherObj = load_field_from_object(electronicCodeBook_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6658 if (embeddedCipherObj == NULL) return false; 6659 6660 // cast it to what we know it will be at runtime 6661 const TypeInstPtr* tinst = _gvn.type(electronicCodeBook_object)->isa_instptr(); 6662 assert(tinst != NULL, "ECB obj is null"); 6663 assert(tinst->is_loaded(), "ECB obj is not loaded"); 6664 ciKlass* klass_AESCrypt = tinst->instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6665 assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded"); 6666 6667 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6668 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt); 6669 const TypeOopPtr* xtype = aklass->as_instance_type()->cast_to_ptr_type(TypePtr::NotNull); 6670 Node* aescrypt_object = new CheckCastPPNode(control(), embeddedCipherObj, xtype); 6671 aescrypt_object = _gvn.transform(aescrypt_object); 6672 6673 // we need to get the start of the aescrypt_object's expanded key array 6674 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); 6675 if (k_start == NULL) return false; 6676 6677 // Call the stub, passing src_start, dest_start, k_start, r_start and src_len 6678 Node* ecbCrypt = make_runtime_call(RC_LEAF | RC_NO_FP, 6679 OptoRuntime::electronicCodeBook_aescrypt_Type(), 6680 stubAddr, stubName, TypePtr::BOTTOM, 6681 src_start, dest_start, k_start, len); 6682 6683 // return cipher length (int) 6684 Node* retvalue = _gvn.transform(new ProjNode(ecbCrypt, TypeFunc::Parms)); 6685 set_result(retvalue); 6686 return true; 6687 } 6688 6689 //------------------------------inline_counterMode_AESCrypt----------------------- 6690 bool LibraryCallKit::inline_counterMode_AESCrypt(vmIntrinsics::ID id) { 6691 assert(UseAES, "need AES instruction support"); 6692 if (!UseAESCTRIntrinsics) return false; 6693 6694 address stubAddr = NULL; 6695 const char *stubName = NULL; 6696 if (id == vmIntrinsics::_counterMode_AESCrypt) { 6697 stubAddr = StubRoutines::counterMode_AESCrypt(); 6698 stubName = "counterMode_AESCrypt"; 6699 } 6700 if (stubAddr == NULL) return false; 6701 6702 Node* counterMode_object = argument(0); 6703 Node* src = argument(1); 6704 Node* src_offset = argument(2); 6705 Node* len = argument(3); 6706 Node* dest = argument(4); 6707 Node* dest_offset = argument(5); 6708 6709 // (1) src and dest are arrays. 6710 const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr(); 6711 const TypeAryPtr* dest_type = dest->Value(&_gvn)->isa_aryptr(); 6712 assert( src_type != nullptr && src_type->elem() != Type::BOTTOM && 6713 dest_type != nullptr && dest_type->elem() != Type::BOTTOM, "args are strange"); 6714 6715 // checks are the responsibility of the caller 6716 Node* src_start = src; 6717 Node* dest_start = dest; 6718 if (src_offset != NULL || dest_offset != NULL) { 6719 assert(src_offset != NULL && dest_offset != NULL, ""); 6720 src_start = array_element_address(src, src_offset, T_BYTE); 6721 dest_start = array_element_address(dest, dest_offset, T_BYTE); 6722 } 6723 6724 // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object 6725 // (because of the predicated logic executed earlier). 6726 // so we cast it here safely. 6727 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java 6728 Node* embeddedCipherObj = load_field_from_object(counterMode_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6729 if (embeddedCipherObj == NULL) return false; 6730 // cast it to what we know it will be at runtime 6731 const TypeInstPtr* tinst = _gvn.type(counterMode_object)->isa_instptr(); 6732 assert(tinst != NULL, "CTR obj is null"); 6733 assert(tinst->is_loaded(), "CTR obj is not loaded"); 6734 ciKlass* klass_AESCrypt = tinst->instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6735 assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded"); 6736 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6737 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt); 6738 const TypeOopPtr* xtype = aklass->as_instance_type()->cast_to_ptr_type(TypePtr::NotNull); 6739 Node* aescrypt_object = new CheckCastPPNode(control(), embeddedCipherObj, xtype); 6740 aescrypt_object = _gvn.transform(aescrypt_object); 6741 // we need to get the start of the aescrypt_object's expanded key array 6742 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); 6743 if (k_start == NULL) return false; 6744 // similarly, get the start address of the r vector 6745 Node* obj_counter = load_field_from_object(counterMode_object, "counter", "[B"); 6746 if (obj_counter == NULL) return false; 6747 Node* cnt_start = array_element_address(obj_counter, intcon(0), T_BYTE); 6748 6749 Node* saved_encCounter = load_field_from_object(counterMode_object, "encryptedCounter", "[B"); 6750 if (saved_encCounter == NULL) return false; 6751 Node* saved_encCounter_start = array_element_address(saved_encCounter, intcon(0), T_BYTE); 6752 Node* used = field_address_from_object(counterMode_object, "used", "I", /*is_exact*/ false); 6753 6754 // Call the stub, passing src_start, dest_start, k_start, r_start and src_len 6755 Node* ctrCrypt = make_runtime_call(RC_LEAF|RC_NO_FP, 6756 OptoRuntime::counterMode_aescrypt_Type(), 6757 stubAddr, stubName, TypePtr::BOTTOM, 6758 src_start, dest_start, k_start, cnt_start, len, saved_encCounter_start, used); 6759 6760 // return cipher length (int) 6761 Node* retvalue = _gvn.transform(new ProjNode(ctrCrypt, TypeFunc::Parms)); 6762 set_result(retvalue); 6763 return true; 6764 } 6765 6766 //------------------------------get_key_start_from_aescrypt_object----------------------- 6767 Node * LibraryCallKit::get_key_start_from_aescrypt_object(Node *aescrypt_object) { 6768 #if defined(PPC64) || defined(S390) 6769 // MixColumns for decryption can be reduced by preprocessing MixColumns with round keys. 6770 // Intel's extension is based on this optimization and AESCrypt generates round keys by preprocessing MixColumns. 6771 // However, ppc64 vncipher processes MixColumns and requires the same round keys with encryption. 6772 // The ppc64 stubs of encryption and decryption use the same round keys (sessionK[0]). 6773 Node* objSessionK = load_field_from_object(aescrypt_object, "sessionK", "[[I"); 6774 assert (objSessionK != NULL, "wrong version of com.sun.crypto.provider.AESCrypt"); 6775 if (objSessionK == NULL) { 6776 return (Node *) NULL; 6777 } 6778 Node* objAESCryptKey = load_array_element(objSessionK, intcon(0), TypeAryPtr::OOPS, /* set_ctrl */ true); 6779 #else 6780 Node* objAESCryptKey = load_field_from_object(aescrypt_object, "K", "[I"); 6781 #endif // PPC64 6782 assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt"); 6783 if (objAESCryptKey == NULL) return (Node *) NULL; 6784 6785 // now have the array, need to get the start address of the K array 6786 Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT); 6787 return k_start; 6788 } 6789 6790 //----------------------------inline_cipherBlockChaining_AESCrypt_predicate---------------------------- 6791 // Return node representing slow path of predicate check. 6792 // the pseudo code we want to emulate with this predicate is: 6793 // for encryption: 6794 // if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath 6795 // for decryption: 6796 // if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath 6797 // note cipher==plain is more conservative than the original java code but that's OK 6798 // 6799 Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting) { 6800 // The receiver was checked for NULL already. 6801 Node* objCBC = argument(0); 6802 6803 Node* src = argument(1); 6804 Node* dest = argument(4); 6805 6806 // Load embeddedCipher field of CipherBlockChaining object. 6807 Node* embeddedCipherObj = load_field_from_object(objCBC, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6808 6809 // get AESCrypt klass for instanceOf check 6810 // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point 6811 // will have same classloader as CipherBlockChaining object 6812 const TypeInstPtr* tinst = _gvn.type(objCBC)->isa_instptr(); 6813 assert(tinst != NULL, "CBCobj is null"); 6814 assert(tinst->is_loaded(), "CBCobj is not loaded"); 6815 6816 // we want to do an instanceof comparison against the AESCrypt class 6817 ciKlass* klass_AESCrypt = tinst->instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6818 if (!klass_AESCrypt->is_loaded()) { 6819 // if AESCrypt is not even loaded, we never take the intrinsic fast path 6820 Node* ctrl = control(); 6821 set_control(top()); // no regular fast path 6822 return ctrl; 6823 } 6824 6825 src = must_be_not_null(src, true); 6826 dest = must_be_not_null(dest, true); 6827 6828 // Resolve oops to stable for CmpP below. 6829 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6830 6831 Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt))); 6832 Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); 6833 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); 6834 6835 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); 6836 6837 // for encryption, we are done 6838 if (!decrypting) 6839 return instof_false; // even if it is NULL 6840 6841 // for decryption, we need to add a further check to avoid 6842 // taking the intrinsic path when cipher and plain are the same 6843 // see the original java code for why. 6844 RegionNode* region = new RegionNode(3); 6845 region->init_req(1, instof_false); 6846 6847 Node* cmp_src_dest = _gvn.transform(new CmpPNode(src, dest)); 6848 Node* bool_src_dest = _gvn.transform(new BoolNode(cmp_src_dest, BoolTest::eq)); 6849 Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN); 6850 region->init_req(2, src_dest_conjoint); 6851 6852 record_for_igvn(region); 6853 return _gvn.transform(region); 6854 } 6855 6856 //----------------------------inline_electronicCodeBook_AESCrypt_predicate---------------------------- 6857 // Return node representing slow path of predicate check. 6858 // the pseudo code we want to emulate with this predicate is: 6859 // for encryption: 6860 // if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath 6861 // for decryption: 6862 // if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath 6863 // note cipher==plain is more conservative than the original java code but that's OK 6864 // 6865 Node* LibraryCallKit::inline_electronicCodeBook_AESCrypt_predicate(bool decrypting) { 6866 // The receiver was checked for NULL already. 6867 Node* objECB = argument(0); 6868 6869 // Load embeddedCipher field of ElectronicCodeBook object. 6870 Node* embeddedCipherObj = load_field_from_object(objECB, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6871 6872 // get AESCrypt klass for instanceOf check 6873 // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point 6874 // will have same classloader as ElectronicCodeBook object 6875 const TypeInstPtr* tinst = _gvn.type(objECB)->isa_instptr(); 6876 assert(tinst != NULL, "ECBobj is null"); 6877 assert(tinst->is_loaded(), "ECBobj is not loaded"); 6878 6879 // we want to do an instanceof comparison against the AESCrypt class 6880 ciKlass* klass_AESCrypt = tinst->instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6881 if (!klass_AESCrypt->is_loaded()) { 6882 // if AESCrypt is not even loaded, we never take the intrinsic fast path 6883 Node* ctrl = control(); 6884 set_control(top()); // no regular fast path 6885 return ctrl; 6886 } 6887 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6888 6889 Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt))); 6890 Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); 6891 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); 6892 6893 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); 6894 6895 // for encryption, we are done 6896 if (!decrypting) 6897 return instof_false; // even if it is NULL 6898 6899 // for decryption, we need to add a further check to avoid 6900 // taking the intrinsic path when cipher and plain are the same 6901 // see the original java code for why. 6902 RegionNode* region = new RegionNode(3); 6903 region->init_req(1, instof_false); 6904 Node* src = argument(1); 6905 Node* dest = argument(4); 6906 Node* cmp_src_dest = _gvn.transform(new CmpPNode(src, dest)); 6907 Node* bool_src_dest = _gvn.transform(new BoolNode(cmp_src_dest, BoolTest::eq)); 6908 Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN); 6909 region->init_req(2, src_dest_conjoint); 6910 6911 record_for_igvn(region); 6912 return _gvn.transform(region); 6913 } 6914 6915 //----------------------------inline_counterMode_AESCrypt_predicate---------------------------- 6916 // Return node representing slow path of predicate check. 6917 // the pseudo code we want to emulate with this predicate is: 6918 // for encryption: 6919 // if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath 6920 // for decryption: 6921 // if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath 6922 // note cipher==plain is more conservative than the original java code but that's OK 6923 // 6924 6925 Node* LibraryCallKit::inline_counterMode_AESCrypt_predicate() { 6926 // The receiver was checked for NULL already. 6927 Node* objCTR = argument(0); 6928 6929 // Load embeddedCipher field of CipherBlockChaining object. 6930 Node* embeddedCipherObj = load_field_from_object(objCTR, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6931 6932 // get AESCrypt klass for instanceOf check 6933 // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point 6934 // will have same classloader as CipherBlockChaining object 6935 const TypeInstPtr* tinst = _gvn.type(objCTR)->isa_instptr(); 6936 assert(tinst != NULL, "CTRobj is null"); 6937 assert(tinst->is_loaded(), "CTRobj is not loaded"); 6938 6939 // we want to do an instanceof comparison against the AESCrypt class 6940 ciKlass* klass_AESCrypt = tinst->instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6941 if (!klass_AESCrypt->is_loaded()) { 6942 // if AESCrypt is not even loaded, we never take the intrinsic fast path 6943 Node* ctrl = control(); 6944 set_control(top()); // no regular fast path 6945 return ctrl; 6946 } 6947 6948 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6949 Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt))); 6950 Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); 6951 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); 6952 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); 6953 6954 return instof_false; // even if it is NULL 6955 } 6956 6957 //------------------------------inline_ghash_processBlocks 6958 bool LibraryCallKit::inline_ghash_processBlocks() { 6959 address stubAddr; 6960 const char *stubName; 6961 assert(UseGHASHIntrinsics, "need GHASH intrinsics support"); 6962 6963 stubAddr = StubRoutines::ghash_processBlocks(); 6964 stubName = "ghash_processBlocks"; 6965 6966 Node* data = argument(0); 6967 Node* offset = argument(1); 6968 Node* len = argument(2); 6969 Node* state = argument(3); 6970 Node* subkeyH = argument(4); 6971 6972 state = must_be_not_null(state, true); 6973 subkeyH = must_be_not_null(subkeyH, true); 6974 data = must_be_not_null(data, true); 6975 6976 Node* state_start = array_element_address(state, intcon(0), T_LONG); 6977 assert(state_start, "state is NULL"); 6978 Node* subkeyH_start = array_element_address(subkeyH, intcon(0), T_LONG); 6979 assert(subkeyH_start, "subkeyH is NULL"); 6980 Node* data_start = array_element_address(data, offset, T_BYTE); 6981 assert(data_start, "data is NULL"); 6982 6983 Node* ghash = make_runtime_call(RC_LEAF|RC_NO_FP, 6984 OptoRuntime::ghash_processBlocks_Type(), 6985 stubAddr, stubName, TypePtr::BOTTOM, 6986 state_start, subkeyH_start, data_start, len); 6987 return true; 6988 } 6989 6990 //------------------------------inline_chacha20Block 6991 bool LibraryCallKit::inline_chacha20Block() { 6992 address stubAddr; 6993 const char *stubName; 6994 assert(UseChaCha20Intrinsics, "need ChaCha20 intrinsics support"); 6995 6996 stubAddr = StubRoutines::chacha20Block(); 6997 stubName = "chacha20Block"; 6998 6999 Node* state = argument(0); 7000 Node* result = argument(1); 7001 7002 state = must_be_not_null(state, true); 7003 result = must_be_not_null(result, true); 7004 7005 Node* state_start = array_element_address(state, intcon(0), T_INT); 7006 assert(state_start, "state is NULL"); 7007 Node* result_start = array_element_address(result, intcon(0), T_BYTE); 7008 assert(result_start, "result is NULL"); 7009 7010 Node* cc20Blk = make_runtime_call(RC_LEAF|RC_NO_FP, 7011 OptoRuntime::chacha20Block_Type(), 7012 stubAddr, stubName, TypePtr::BOTTOM, 7013 state_start, result_start); 7014 // return key stream length (int) 7015 Node* retvalue = _gvn.transform(new ProjNode(cc20Blk, TypeFunc::Parms)); 7016 set_result(retvalue); 7017 return true; 7018 } 7019 7020 bool LibraryCallKit::inline_base64_encodeBlock() { 7021 address stubAddr; 7022 const char *stubName; 7023 assert(UseBASE64Intrinsics, "need Base64 intrinsics support"); 7024 assert(callee()->signature()->size() == 6, "base64_encodeBlock has 6 parameters"); 7025 stubAddr = StubRoutines::base64_encodeBlock(); 7026 stubName = "encodeBlock"; 7027 7028 if (!stubAddr) return false; 7029 Node* base64obj = argument(0); 7030 Node* src = argument(1); 7031 Node* offset = argument(2); 7032 Node* len = argument(3); 7033 Node* dest = argument(4); 7034 Node* dp = argument(5); 7035 Node* isURL = argument(6); 7036 7037 src = must_be_not_null(src, true); 7038 dest = must_be_not_null(dest, true); 7039 7040 Node* src_start = array_element_address(src, intcon(0), T_BYTE); 7041 assert(src_start, "source array is NULL"); 7042 Node* dest_start = array_element_address(dest, intcon(0), T_BYTE); 7043 assert(dest_start, "destination array is NULL"); 7044 7045 Node* base64 = make_runtime_call(RC_LEAF, 7046 OptoRuntime::base64_encodeBlock_Type(), 7047 stubAddr, stubName, TypePtr::BOTTOM, 7048 src_start, offset, len, dest_start, dp, isURL); 7049 return true; 7050 } 7051 7052 bool LibraryCallKit::inline_base64_decodeBlock() { 7053 address stubAddr; 7054 const char *stubName; 7055 assert(UseBASE64Intrinsics, "need Base64 intrinsics support"); 7056 assert(callee()->signature()->size() == 7, "base64_decodeBlock has 7 parameters"); 7057 stubAddr = StubRoutines::base64_decodeBlock(); 7058 stubName = "decodeBlock"; 7059 7060 if (!stubAddr) return false; 7061 Node* base64obj = argument(0); 7062 Node* src = argument(1); 7063 Node* src_offset = argument(2); 7064 Node* len = argument(3); 7065 Node* dest = argument(4); 7066 Node* dest_offset = argument(5); 7067 Node* isURL = argument(6); 7068 Node* isMIME = argument(7); 7069 7070 src = must_be_not_null(src, true); 7071 dest = must_be_not_null(dest, true); 7072 7073 Node* src_start = array_element_address(src, intcon(0), T_BYTE); 7074 assert(src_start, "source array is NULL"); 7075 Node* dest_start = array_element_address(dest, intcon(0), T_BYTE); 7076 assert(dest_start, "destination array is NULL"); 7077 7078 Node* call = make_runtime_call(RC_LEAF, 7079 OptoRuntime::base64_decodeBlock_Type(), 7080 stubAddr, stubName, TypePtr::BOTTOM, 7081 src_start, src_offset, len, dest_start, dest_offset, isURL, isMIME); 7082 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 7083 set_result(result); 7084 return true; 7085 } 7086 7087 bool LibraryCallKit::inline_poly1305_processBlocks() { 7088 address stubAddr; 7089 const char *stubName; 7090 assert(UsePoly1305Intrinsics, "need Poly intrinsics support"); 7091 assert(callee()->signature()->size() == 5, "poly1305_processBlocks has %d parameters", callee()->signature()->size()); 7092 stubAddr = StubRoutines::poly1305_processBlocks(); 7093 stubName = "poly1305_processBlocks"; 7094 7095 if (!stubAddr) return false; 7096 null_check_receiver(); // null-check receiver 7097 if (stopped()) return true; 7098 7099 Node* input = argument(1); 7100 Node* input_offset = argument(2); 7101 Node* len = argument(3); 7102 Node* alimbs = argument(4); 7103 Node* rlimbs = argument(5); 7104 7105 input = must_be_not_null(input, true); 7106 alimbs = must_be_not_null(alimbs, true); 7107 rlimbs = must_be_not_null(rlimbs, true); 7108 7109 Node* input_start = array_element_address(input, input_offset, T_BYTE); 7110 assert(input_start, "input array is NULL"); 7111 Node* acc_start = array_element_address(alimbs, intcon(0), T_LONG); 7112 assert(acc_start, "acc array is NULL"); 7113 Node* r_start = array_element_address(rlimbs, intcon(0), T_LONG); 7114 assert(r_start, "r array is NULL"); 7115 7116 Node* call = make_runtime_call(RC_LEAF | RC_NO_FP, 7117 OptoRuntime::poly1305_processBlocks_Type(), 7118 stubAddr, stubName, TypePtr::BOTTOM, 7119 input_start, len, acc_start, r_start); 7120 return true; 7121 } 7122 7123 //------------------------------inline_digestBase_implCompress----------------------- 7124 // 7125 // Calculate MD5 for single-block byte[] array. 7126 // void com.sun.security.provider.MD5.implCompress(byte[] buf, int ofs) 7127 // 7128 // Calculate SHA (i.e., SHA-1) for single-block byte[] array. 7129 // void com.sun.security.provider.SHA.implCompress(byte[] buf, int ofs) 7130 // 7131 // Calculate SHA2 (i.e., SHA-244 or SHA-256) for single-block byte[] array. 7132 // void com.sun.security.provider.SHA2.implCompress(byte[] buf, int ofs) 7133 // 7134 // Calculate SHA5 (i.e., SHA-384 or SHA-512) for single-block byte[] array. 7135 // void com.sun.security.provider.SHA5.implCompress(byte[] buf, int ofs) 7136 // 7137 // Calculate SHA3 (i.e., SHA3-224 or SHA3-256 or SHA3-384 or SHA3-512) for single-block byte[] array. 7138 // void com.sun.security.provider.SHA3.implCompress(byte[] buf, int ofs) 7139 // 7140 bool LibraryCallKit::inline_digestBase_implCompress(vmIntrinsics::ID id) { 7141 assert(callee()->signature()->size() == 2, "sha_implCompress has 2 parameters"); 7142 7143 Node* digestBase_obj = argument(0); 7144 Node* src = argument(1); // type oop 7145 Node* ofs = argument(2); // type int 7146 7147 const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr(); 7148 if (src_type == nullptr || src_type->elem() == Type::BOTTOM) { 7149 // failed array check 7150 return false; 7151 } 7152 // Figure out the size and type of the elements we will be copying. 7153 BasicType src_elem = src_type->elem()->array_element_basic_type(); 7154 if (src_elem != T_BYTE) { 7155 return false; 7156 } 7157 // 'src_start' points to src array + offset 7158 src = must_be_not_null(src, true); 7159 Node* src_start = array_element_address(src, ofs, src_elem); 7160 Node* state = NULL; 7161 Node* block_size = NULL; 7162 address stubAddr; 7163 const char *stubName; 7164 7165 switch(id) { 7166 case vmIntrinsics::_md5_implCompress: 7167 assert(UseMD5Intrinsics, "need MD5 instruction support"); 7168 state = get_state_from_digest_object(digestBase_obj, T_INT); 7169 stubAddr = StubRoutines::md5_implCompress(); 7170 stubName = "md5_implCompress"; 7171 break; 7172 case vmIntrinsics::_sha_implCompress: 7173 assert(UseSHA1Intrinsics, "need SHA1 instruction support"); 7174 state = get_state_from_digest_object(digestBase_obj, T_INT); 7175 stubAddr = StubRoutines::sha1_implCompress(); 7176 stubName = "sha1_implCompress"; 7177 break; 7178 case vmIntrinsics::_sha2_implCompress: 7179 assert(UseSHA256Intrinsics, "need SHA256 instruction support"); 7180 state = get_state_from_digest_object(digestBase_obj, T_INT); 7181 stubAddr = StubRoutines::sha256_implCompress(); 7182 stubName = "sha256_implCompress"; 7183 break; 7184 case vmIntrinsics::_sha5_implCompress: 7185 assert(UseSHA512Intrinsics, "need SHA512 instruction support"); 7186 state = get_state_from_digest_object(digestBase_obj, T_LONG); 7187 stubAddr = StubRoutines::sha512_implCompress(); 7188 stubName = "sha512_implCompress"; 7189 break; 7190 case vmIntrinsics::_sha3_implCompress: 7191 assert(UseSHA3Intrinsics, "need SHA3 instruction support"); 7192 state = get_state_from_digest_object(digestBase_obj, T_BYTE); 7193 stubAddr = StubRoutines::sha3_implCompress(); 7194 stubName = "sha3_implCompress"; 7195 block_size = get_block_size_from_digest_object(digestBase_obj); 7196 if (block_size == NULL) return false; 7197 break; 7198 default: 7199 fatal_unexpected_iid(id); 7200 return false; 7201 } 7202 if (state == NULL) return false; 7203 7204 assert(stubAddr != NULL, "Stub is generated"); 7205 if (stubAddr == NULL) return false; 7206 7207 // Call the stub. 7208 Node* call; 7209 if (block_size == NULL) { 7210 call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::digestBase_implCompress_Type(false), 7211 stubAddr, stubName, TypePtr::BOTTOM, 7212 src_start, state); 7213 } else { 7214 call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::digestBase_implCompress_Type(true), 7215 stubAddr, stubName, TypePtr::BOTTOM, 7216 src_start, state, block_size); 7217 } 7218 7219 return true; 7220 } 7221 7222 //------------------------------inline_digestBase_implCompressMB----------------------- 7223 // 7224 // Calculate MD5/SHA/SHA2/SHA5/SHA3 for multi-block byte[] array. 7225 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit) 7226 // 7227 bool LibraryCallKit::inline_digestBase_implCompressMB(int predicate) { 7228 assert(UseMD5Intrinsics || UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics || UseSHA3Intrinsics, 7229 "need MD5/SHA1/SHA256/SHA512/SHA3 instruction support"); 7230 assert((uint)predicate < 5, "sanity"); 7231 assert(callee()->signature()->size() == 3, "digestBase_implCompressMB has 3 parameters"); 7232 7233 Node* digestBase_obj = argument(0); // The receiver was checked for NULL already. 7234 Node* src = argument(1); // byte[] array 7235 Node* ofs = argument(2); // type int 7236 Node* limit = argument(3); // type int 7237 7238 const TypeAryPtr* src_type = src->Value(&_gvn)->isa_aryptr(); 7239 if (src_type == nullptr || src_type->elem() == Type::BOTTOM) { 7240 // failed array check 7241 return false; 7242 } 7243 // Figure out the size and type of the elements we will be copying. 7244 BasicType src_elem = src_type->elem()->array_element_basic_type(); 7245 if (src_elem != T_BYTE) { 7246 return false; 7247 } 7248 // 'src_start' points to src array + offset 7249 src = must_be_not_null(src, false); 7250 Node* src_start = array_element_address(src, ofs, src_elem); 7251 7252 const char* klass_digestBase_name = NULL; 7253 const char* stub_name = NULL; 7254 address stub_addr = NULL; 7255 BasicType elem_type = T_INT; 7256 7257 switch (predicate) { 7258 case 0: 7259 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_md5_implCompress)) { 7260 klass_digestBase_name = "sun/security/provider/MD5"; 7261 stub_name = "md5_implCompressMB"; 7262 stub_addr = StubRoutines::md5_implCompressMB(); 7263 } 7264 break; 7265 case 1: 7266 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_sha_implCompress)) { 7267 klass_digestBase_name = "sun/security/provider/SHA"; 7268 stub_name = "sha1_implCompressMB"; 7269 stub_addr = StubRoutines::sha1_implCompressMB(); 7270 } 7271 break; 7272 case 2: 7273 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_sha2_implCompress)) { 7274 klass_digestBase_name = "sun/security/provider/SHA2"; 7275 stub_name = "sha256_implCompressMB"; 7276 stub_addr = StubRoutines::sha256_implCompressMB(); 7277 } 7278 break; 7279 case 3: 7280 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_sha5_implCompress)) { 7281 klass_digestBase_name = "sun/security/provider/SHA5"; 7282 stub_name = "sha512_implCompressMB"; 7283 stub_addr = StubRoutines::sha512_implCompressMB(); 7284 elem_type = T_LONG; 7285 } 7286 break; 7287 case 4: 7288 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_sha3_implCompress)) { 7289 klass_digestBase_name = "sun/security/provider/SHA3"; 7290 stub_name = "sha3_implCompressMB"; 7291 stub_addr = StubRoutines::sha3_implCompressMB(); 7292 elem_type = T_BYTE; 7293 } 7294 break; 7295 default: 7296 fatal("unknown DigestBase intrinsic predicate: %d", predicate); 7297 } 7298 if (klass_digestBase_name != NULL) { 7299 assert(stub_addr != NULL, "Stub is generated"); 7300 if (stub_addr == NULL) return false; 7301 7302 // get DigestBase klass to lookup for SHA klass 7303 const TypeInstPtr* tinst = _gvn.type(digestBase_obj)->isa_instptr(); 7304 assert(tinst != NULL, "digestBase_obj is not instance???"); 7305 assert(tinst->is_loaded(), "DigestBase is not loaded"); 7306 7307 ciKlass* klass_digestBase = tinst->instance_klass()->find_klass(ciSymbol::make(klass_digestBase_name)); 7308 assert(klass_digestBase->is_loaded(), "predicate checks that this class is loaded"); 7309 ciInstanceKlass* instklass_digestBase = klass_digestBase->as_instance_klass(); 7310 return inline_digestBase_implCompressMB(digestBase_obj, instklass_digestBase, elem_type, stub_addr, stub_name, src_start, ofs, limit); 7311 } 7312 return false; 7313 } 7314 7315 //------------------------------inline_digestBase_implCompressMB----------------------- 7316 bool LibraryCallKit::inline_digestBase_implCompressMB(Node* digestBase_obj, ciInstanceKlass* instklass_digestBase, 7317 BasicType elem_type, address stubAddr, const char *stubName, 7318 Node* src_start, Node* ofs, Node* limit) { 7319 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_digestBase); 7320 const TypeOopPtr* xtype = aklass->cast_to_exactness(false)->as_instance_type()->cast_to_ptr_type(TypePtr::NotNull); 7321 Node* digest_obj = new CheckCastPPNode(control(), digestBase_obj, xtype); 7322 digest_obj = _gvn.transform(digest_obj); 7323 7324 Node* state = get_state_from_digest_object(digest_obj, elem_type); 7325 if (state == NULL) return false; 7326 7327 Node* block_size = NULL; 7328 if (strcmp("sha3_implCompressMB", stubName) == 0) { 7329 block_size = get_block_size_from_digest_object(digest_obj); 7330 if (block_size == NULL) return false; 7331 } 7332 7333 // Call the stub. 7334 Node* call; 7335 if (block_size == NULL) { 7336 call = make_runtime_call(RC_LEAF|RC_NO_FP, 7337 OptoRuntime::digestBase_implCompressMB_Type(false), 7338 stubAddr, stubName, TypePtr::BOTTOM, 7339 src_start, state, ofs, limit); 7340 } else { 7341 call = make_runtime_call(RC_LEAF|RC_NO_FP, 7342 OptoRuntime::digestBase_implCompressMB_Type(true), 7343 stubAddr, stubName, TypePtr::BOTTOM, 7344 src_start, state, block_size, ofs, limit); 7345 } 7346 7347 // return ofs (int) 7348 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 7349 set_result(result); 7350 7351 return true; 7352 } 7353 7354 //------------------------------inline_galoisCounterMode_AESCrypt----------------------- 7355 bool LibraryCallKit::inline_galoisCounterMode_AESCrypt() { 7356 assert(UseAES, "need AES instruction support"); 7357 address stubAddr = NULL; 7358 const char *stubName = NULL; 7359 stubAddr = StubRoutines::galoisCounterMode_AESCrypt(); 7360 stubName = "galoisCounterMode_AESCrypt"; 7361 7362 if (stubAddr == NULL) return false; 7363 7364 Node* in = argument(0); 7365 Node* inOfs = argument(1); 7366 Node* len = argument(2); 7367 Node* ct = argument(3); 7368 Node* ctOfs = argument(4); 7369 Node* out = argument(5); 7370 Node* outOfs = argument(6); 7371 Node* gctr_object = argument(7); 7372 Node* ghash_object = argument(8); 7373 7374 // (1) in, ct and out are arrays. 7375 const TypeAryPtr* in_type = in->Value(&_gvn)->isa_aryptr(); 7376 const TypeAryPtr* ct_type = ct->Value(&_gvn)->isa_aryptr(); 7377 const TypeAryPtr* out_type = out->Value(&_gvn)->isa_aryptr(); 7378 assert( in_type != nullptr && in_type->elem() != Type::BOTTOM && 7379 ct_type != nullptr && ct_type->elem() != Type::BOTTOM && 7380 out_type != nullptr && out_type->elem() != Type::BOTTOM, "args are strange"); 7381 7382 // checks are the responsibility of the caller 7383 Node* in_start = in; 7384 Node* ct_start = ct; 7385 Node* out_start = out; 7386 if (inOfs != NULL || ctOfs != NULL || outOfs != NULL) { 7387 assert(inOfs != NULL && ctOfs != NULL && outOfs != NULL, ""); 7388 in_start = array_element_address(in, inOfs, T_BYTE); 7389 ct_start = array_element_address(ct, ctOfs, T_BYTE); 7390 out_start = array_element_address(out, outOfs, T_BYTE); 7391 } 7392 7393 // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object 7394 // (because of the predicated logic executed earlier). 7395 // so we cast it here safely. 7396 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java 7397 Node* embeddedCipherObj = load_field_from_object(gctr_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 7398 Node* counter = load_field_from_object(gctr_object, "counter", "[B"); 7399 Node* subkeyHtbl = load_field_from_object(ghash_object, "subkeyHtbl", "[J"); 7400 Node* state = load_field_from_object(ghash_object, "state", "[J"); 7401 7402 if (embeddedCipherObj == NULL || counter == NULL || subkeyHtbl == NULL || state == NULL) { 7403 return false; 7404 } 7405 // cast it to what we know it will be at runtime 7406 const TypeInstPtr* tinst = _gvn.type(gctr_object)->isa_instptr(); 7407 assert(tinst != NULL, "GCTR obj is null"); 7408 assert(tinst->is_loaded(), "GCTR obj is not loaded"); 7409 ciKlass* klass_AESCrypt = tinst->instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 7410 assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded"); 7411 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 7412 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt); 7413 const TypeOopPtr* xtype = aklass->as_instance_type(); 7414 Node* aescrypt_object = new CheckCastPPNode(control(), embeddedCipherObj, xtype); 7415 aescrypt_object = _gvn.transform(aescrypt_object); 7416 // we need to get the start of the aescrypt_object's expanded key array 7417 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); 7418 if (k_start == NULL) return false; 7419 // similarly, get the start address of the r vector 7420 Node* cnt_start = array_element_address(counter, intcon(0), T_BYTE); 7421 Node* state_start = array_element_address(state, intcon(0), T_LONG); 7422 Node* subkeyHtbl_start = array_element_address(subkeyHtbl, intcon(0), T_LONG); 7423 7424 7425 // Call the stub, passing params 7426 Node* gcmCrypt = make_runtime_call(RC_LEAF|RC_NO_FP, 7427 OptoRuntime::galoisCounterMode_aescrypt_Type(), 7428 stubAddr, stubName, TypePtr::BOTTOM, 7429 in_start, len, ct_start, out_start, k_start, state_start, subkeyHtbl_start, cnt_start); 7430 7431 // return cipher length (int) 7432 Node* retvalue = _gvn.transform(new ProjNode(gcmCrypt, TypeFunc::Parms)); 7433 set_result(retvalue); 7434 7435 return true; 7436 } 7437 7438 //----------------------------inline_galoisCounterMode_AESCrypt_predicate---------------------------- 7439 // Return node representing slow path of predicate check. 7440 // the pseudo code we want to emulate with this predicate is: 7441 // for encryption: 7442 // if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath 7443 // for decryption: 7444 // if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath 7445 // note cipher==plain is more conservative than the original java code but that's OK 7446 // 7447 7448 Node* LibraryCallKit::inline_galoisCounterMode_AESCrypt_predicate() { 7449 // The receiver was checked for NULL already. 7450 Node* objGCTR = argument(7); 7451 // Load embeddedCipher field of GCTR object. 7452 Node* embeddedCipherObj = load_field_from_object(objGCTR, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 7453 assert(embeddedCipherObj != NULL, "embeddedCipherObj is null"); 7454 7455 // get AESCrypt klass for instanceOf check 7456 // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point 7457 // will have same classloader as CipherBlockChaining object 7458 const TypeInstPtr* tinst = _gvn.type(objGCTR)->isa_instptr(); 7459 assert(tinst != NULL, "GCTR obj is null"); 7460 assert(tinst->is_loaded(), "GCTR obj is not loaded"); 7461 7462 // we want to do an instanceof comparison against the AESCrypt class 7463 ciKlass* klass_AESCrypt = tinst->instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 7464 if (!klass_AESCrypt->is_loaded()) { 7465 // if AESCrypt is not even loaded, we never take the intrinsic fast path 7466 Node* ctrl = control(); 7467 set_control(top()); // no regular fast path 7468 return ctrl; 7469 } 7470 7471 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 7472 Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt))); 7473 Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); 7474 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); 7475 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); 7476 7477 return instof_false; // even if it is NULL 7478 } 7479 7480 //------------------------------get_state_from_digest_object----------------------- 7481 Node * LibraryCallKit::get_state_from_digest_object(Node *digest_object, BasicType elem_type) { 7482 const char* state_type; 7483 switch (elem_type) { 7484 case T_BYTE: state_type = "[B"; break; 7485 case T_INT: state_type = "[I"; break; 7486 case T_LONG: state_type = "[J"; break; 7487 default: ShouldNotReachHere(); 7488 } 7489 Node* digest_state = load_field_from_object(digest_object, "state", state_type); 7490 assert (digest_state != NULL, "wrong version of sun.security.provider.MD5/SHA/SHA2/SHA5/SHA3"); 7491 if (digest_state == NULL) return (Node *) NULL; 7492 7493 // now have the array, need to get the start address of the state array 7494 Node* state = array_element_address(digest_state, intcon(0), elem_type); 7495 return state; 7496 } 7497 7498 //------------------------------get_block_size_from_sha3_object---------------------------------- 7499 Node * LibraryCallKit::get_block_size_from_digest_object(Node *digest_object) { 7500 Node* block_size = load_field_from_object(digest_object, "blockSize", "I"); 7501 assert (block_size != NULL, "sanity"); 7502 return block_size; 7503 } 7504 7505 //----------------------------inline_digestBase_implCompressMB_predicate---------------------------- 7506 // Return node representing slow path of predicate check. 7507 // the pseudo code we want to emulate with this predicate is: 7508 // if (digestBaseObj instanceof MD5/SHA/SHA2/SHA5/SHA3) do_intrinsic, else do_javapath 7509 // 7510 Node* LibraryCallKit::inline_digestBase_implCompressMB_predicate(int predicate) { 7511 assert(UseMD5Intrinsics || UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics || UseSHA3Intrinsics, 7512 "need MD5/SHA1/SHA256/SHA512/SHA3 instruction support"); 7513 assert((uint)predicate < 5, "sanity"); 7514 7515 // The receiver was checked for NULL already. 7516 Node* digestBaseObj = argument(0); 7517 7518 // get DigestBase klass for instanceOf check 7519 const TypeInstPtr* tinst = _gvn.type(digestBaseObj)->isa_instptr(); 7520 assert(tinst != NULL, "digestBaseObj is null"); 7521 assert(tinst->is_loaded(), "DigestBase is not loaded"); 7522 7523 const char* klass_name = NULL; 7524 switch (predicate) { 7525 case 0: 7526 if (UseMD5Intrinsics) { 7527 // we want to do an instanceof comparison against the MD5 class 7528 klass_name = "sun/security/provider/MD5"; 7529 } 7530 break; 7531 case 1: 7532 if (UseSHA1Intrinsics) { 7533 // we want to do an instanceof comparison against the SHA class 7534 klass_name = "sun/security/provider/SHA"; 7535 } 7536 break; 7537 case 2: 7538 if (UseSHA256Intrinsics) { 7539 // we want to do an instanceof comparison against the SHA2 class 7540 klass_name = "sun/security/provider/SHA2"; 7541 } 7542 break; 7543 case 3: 7544 if (UseSHA512Intrinsics) { 7545 // we want to do an instanceof comparison against the SHA5 class 7546 klass_name = "sun/security/provider/SHA5"; 7547 } 7548 break; 7549 case 4: 7550 if (UseSHA3Intrinsics) { 7551 // we want to do an instanceof comparison against the SHA3 class 7552 klass_name = "sun/security/provider/SHA3"; 7553 } 7554 break; 7555 default: 7556 fatal("unknown SHA intrinsic predicate: %d", predicate); 7557 } 7558 7559 ciKlass* klass = NULL; 7560 if (klass_name != NULL) { 7561 klass = tinst->instance_klass()->find_klass(ciSymbol::make(klass_name)); 7562 } 7563 if ((klass == NULL) || !klass->is_loaded()) { 7564 // if none of MD5/SHA/SHA2/SHA5 is loaded, we never take the intrinsic fast path 7565 Node* ctrl = control(); 7566 set_control(top()); // no intrinsic path 7567 return ctrl; 7568 } 7569 ciInstanceKlass* instklass = klass->as_instance_klass(); 7570 7571 Node* instof = gen_instanceof(digestBaseObj, makecon(TypeKlassPtr::make(instklass))); 7572 Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); 7573 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); 7574 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); 7575 7576 return instof_false; // even if it is NULL 7577 } 7578 7579 //-------------inline_fma----------------------------------- 7580 bool LibraryCallKit::inline_fma(vmIntrinsics::ID id) { 7581 Node *a = NULL; 7582 Node *b = NULL; 7583 Node *c = NULL; 7584 Node* result = NULL; 7585 switch (id) { 7586 case vmIntrinsics::_fmaD: 7587 assert(callee()->signature()->size() == 6, "fma has 3 parameters of size 2 each."); 7588 // no receiver since it is static method 7589 a = round_double_node(argument(0)); 7590 b = round_double_node(argument(2)); 7591 c = round_double_node(argument(4)); 7592 result = _gvn.transform(new FmaDNode(control(), a, b, c)); 7593 break; 7594 case vmIntrinsics::_fmaF: 7595 assert(callee()->signature()->size() == 3, "fma has 3 parameters of size 1 each."); 7596 a = argument(0); 7597 b = argument(1); 7598 c = argument(2); 7599 result = _gvn.transform(new FmaFNode(control(), a, b, c)); 7600 break; 7601 default: 7602 fatal_unexpected_iid(id); break; 7603 } 7604 set_result(result); 7605 return true; 7606 } 7607 7608 bool LibraryCallKit::inline_character_compare(vmIntrinsics::ID id) { 7609 // argument(0) is receiver 7610 Node* codePoint = argument(1); 7611 Node* n = NULL; 7612 7613 switch (id) { 7614 case vmIntrinsics::_isDigit : 7615 n = new DigitNode(control(), codePoint); 7616 break; 7617 case vmIntrinsics::_isLowerCase : 7618 n = new LowerCaseNode(control(), codePoint); 7619 break; 7620 case vmIntrinsics::_isUpperCase : 7621 n = new UpperCaseNode(control(), codePoint); 7622 break; 7623 case vmIntrinsics::_isWhitespace : 7624 n = new WhitespaceNode(control(), codePoint); 7625 break; 7626 default: 7627 fatal_unexpected_iid(id); 7628 } 7629 7630 set_result(_gvn.transform(n)); 7631 return true; 7632 } 7633 7634 //------------------------------inline_fp_min_max------------------------------ 7635 bool LibraryCallKit::inline_fp_min_max(vmIntrinsics::ID id) { 7636 /* DISABLED BECAUSE METHOD DATA ISN'T COLLECTED PER CALL-SITE, SEE JDK-8015416. 7637 7638 // The intrinsic should be used only when the API branches aren't predictable, 7639 // the last one performing the most important comparison. The following heuristic 7640 // uses the branch statistics to eventually bail out if necessary. 7641 7642 ciMethodData *md = callee()->method_data(); 7643 7644 if ( md != NULL && md->is_mature() && md->invocation_count() > 0 ) { 7645 ciCallProfile cp = caller()->call_profile_at_bci(bci()); 7646 7647 if ( ((double)cp.count()) / ((double)md->invocation_count()) < 0.8 ) { 7648 // Bail out if the call-site didn't contribute enough to the statistics. 7649 return false; 7650 } 7651 7652 uint taken = 0, not_taken = 0; 7653 7654 for (ciProfileData *p = md->first_data(); md->is_valid(p); p = md->next_data(p)) { 7655 if (p->is_BranchData()) { 7656 taken = ((ciBranchData*)p)->taken(); 7657 not_taken = ((ciBranchData*)p)->not_taken(); 7658 } 7659 } 7660 7661 double balance = (((double)taken) - ((double)not_taken)) / ((double)md->invocation_count()); 7662 balance = balance < 0 ? -balance : balance; 7663 if ( balance > 0.2 ) { 7664 // Bail out if the most important branch is predictable enough. 7665 return false; 7666 } 7667 } 7668 */ 7669 7670 Node *a = NULL; 7671 Node *b = NULL; 7672 Node *n = NULL; 7673 switch (id) { 7674 case vmIntrinsics::_maxF: 7675 case vmIntrinsics::_minF: 7676 case vmIntrinsics::_maxF_strict: 7677 case vmIntrinsics::_minF_strict: 7678 assert(callee()->signature()->size() == 2, "minF/maxF has 2 parameters of size 1 each."); 7679 a = argument(0); 7680 b = argument(1); 7681 break; 7682 case vmIntrinsics::_maxD: 7683 case vmIntrinsics::_minD: 7684 case vmIntrinsics::_maxD_strict: 7685 case vmIntrinsics::_minD_strict: 7686 assert(callee()->signature()->size() == 4, "minD/maxD has 2 parameters of size 2 each."); 7687 a = round_double_node(argument(0)); 7688 b = round_double_node(argument(2)); 7689 break; 7690 default: 7691 fatal_unexpected_iid(id); 7692 break; 7693 } 7694 switch (id) { 7695 case vmIntrinsics::_maxF: 7696 case vmIntrinsics::_maxF_strict: 7697 n = new MaxFNode(a, b); 7698 break; 7699 case vmIntrinsics::_minF: 7700 case vmIntrinsics::_minF_strict: 7701 n = new MinFNode(a, b); 7702 break; 7703 case vmIntrinsics::_maxD: 7704 case vmIntrinsics::_maxD_strict: 7705 n = new MaxDNode(a, b); 7706 break; 7707 case vmIntrinsics::_minD: 7708 case vmIntrinsics::_minD_strict: 7709 n = new MinDNode(a, b); 7710 break; 7711 default: 7712 fatal_unexpected_iid(id); 7713 break; 7714 } 7715 set_result(_gvn.transform(n)); 7716 return true; 7717 } 7718 7719 bool LibraryCallKit::inline_profileBoolean() { 7720 Node* counts = argument(1); 7721 const TypeAryPtr* ary = NULL; 7722 ciArray* aobj = NULL; 7723 if (counts->is_Con() 7724 && (ary = counts->bottom_type()->isa_aryptr()) != NULL 7725 && (aobj = ary->const_oop()->as_array()) != NULL 7726 && (aobj->length() == 2)) { 7727 // Profile is int[2] where [0] and [1] correspond to false and true value occurrences respectively. 7728 jint false_cnt = aobj->element_value(0).as_int(); 7729 jint true_cnt = aobj->element_value(1).as_int(); 7730 7731 if (C->log() != NULL) { 7732 C->log()->elem("observe source='profileBoolean' false='%d' true='%d'", 7733 false_cnt, true_cnt); 7734 } 7735 7736 if (false_cnt + true_cnt == 0) { 7737 // According to profile, never executed. 7738 uncommon_trap_exact(Deoptimization::Reason_intrinsic, 7739 Deoptimization::Action_reinterpret); 7740 return true; 7741 } 7742 7743 // result is a boolean (0 or 1) and its profile (false_cnt & true_cnt) 7744 // is a number of each value occurrences. 7745 Node* result = argument(0); 7746 if (false_cnt == 0 || true_cnt == 0) { 7747 // According to profile, one value has been never seen. 7748 int expected_val = (false_cnt == 0) ? 1 : 0; 7749 7750 Node* cmp = _gvn.transform(new CmpINode(result, intcon(expected_val))); 7751 Node* test = _gvn.transform(new BoolNode(cmp, BoolTest::eq)); 7752 7753 IfNode* check = create_and_map_if(control(), test, PROB_ALWAYS, COUNT_UNKNOWN); 7754 Node* fast_path = _gvn.transform(new IfTrueNode(check)); 7755 Node* slow_path = _gvn.transform(new IfFalseNode(check)); 7756 7757 { // Slow path: uncommon trap for never seen value and then reexecute 7758 // MethodHandleImpl::profileBoolean() to bump the count, so JIT knows 7759 // the value has been seen at least once. 7760 PreserveJVMState pjvms(this); 7761 PreserveReexecuteState preexecs(this); 7762 jvms()->set_should_reexecute(true); 7763 7764 set_control(slow_path); 7765 set_i_o(i_o()); 7766 7767 uncommon_trap_exact(Deoptimization::Reason_intrinsic, 7768 Deoptimization::Action_reinterpret); 7769 } 7770 // The guard for never seen value enables sharpening of the result and 7771 // returning a constant. It allows to eliminate branches on the same value 7772 // later on. 7773 set_control(fast_path); 7774 result = intcon(expected_val); 7775 } 7776 // Stop profiling. 7777 // MethodHandleImpl::profileBoolean() has profiling logic in its bytecode. 7778 // By replacing method body with profile data (represented as ProfileBooleanNode 7779 // on IR level) we effectively disable profiling. 7780 // It enables full speed execution once optimized code is generated. 7781 Node* profile = _gvn.transform(new ProfileBooleanNode(result, false_cnt, true_cnt)); 7782 C->record_for_igvn(profile); 7783 set_result(profile); 7784 return true; 7785 } else { 7786 // Continue profiling. 7787 // Profile data isn't available at the moment. So, execute method's bytecode version. 7788 // Usually, when GWT LambdaForms are profiled it means that a stand-alone nmethod 7789 // is compiled and counters aren't available since corresponding MethodHandle 7790 // isn't a compile-time constant. 7791 return false; 7792 } 7793 } 7794 7795 bool LibraryCallKit::inline_isCompileConstant() { 7796 Node* n = argument(0); 7797 set_result(n->is_Con() ? intcon(1) : intcon(0)); 7798 return true; 7799 } 7800 7801 //------------------------------- inline_getObjectSize/sizeOf -------------------------------------- 7802 // 7803 // Calculate the runtime size of the object/array. 7804 // 7805 // long java.lang.Runtime.sizeOf(Object obj) 7806 // native long sun.instrument.InstrumentationImpl.getObjectSize0(long nativeAgent, Object objectToSize); 7807 // 7808 bool LibraryCallKit::inline_sizeOf() { 7809 Node* obj = argument(0); 7810 return inline_sizeOf_impl(obj); 7811 } 7812 7813 bool LibraryCallKit::inline_getObjectSize() { 7814 Node* obj = argument(3); 7815 return inline_sizeOf_impl(obj); 7816 } 7817 7818 bool LibraryCallKit::inline_sizeOf_impl(Node* obj) { 7819 Node* klass_node = load_object_klass(obj); 7820 7821 jint layout_con = Klass::_lh_neutral_value; 7822 Node* layout_val = get_layout_helper(klass_node, layout_con); 7823 int layout_is_con = (layout_val == NULL); 7824 7825 if (layout_is_con) { 7826 // Layout helper is constant, can figure out things at compile time. 7827 7828 if (Klass::layout_helper_is_instance(layout_con)) { 7829 // Instance case: layout_con contains the size itself. 7830 Node *size = longcon(Klass::layout_helper_size_in_bytes(layout_con)); 7831 set_result(size); 7832 } else { 7833 // Array case: size is round(header + element_size*arraylength). 7834 // Since arraylength is different for every array instance, we have to 7835 // compute the whole thing at runtime. 7836 7837 Node* arr_length = load_array_length(obj); 7838 7839 int round_mask = MinObjAlignmentInBytes - 1; 7840 int hsize = Klass::layout_helper_header_size(layout_con); 7841 int eshift = Klass::layout_helper_log2_element_size(layout_con); 7842 7843 if ((round_mask & ~right_n_bits(eshift)) == 0) { 7844 round_mask = 0; // strength-reduce it if it goes away completely 7845 } 7846 assert((hsize & right_n_bits(eshift)) == 0, "hsize is pre-rounded"); 7847 Node* header_size = intcon(hsize + round_mask); 7848 7849 Node* lengthx = ConvI2X(arr_length); 7850 Node* headerx = ConvI2X(header_size); 7851 7852 Node* abody = lengthx; 7853 if (eshift != 0) { 7854 abody = _gvn.transform(new LShiftXNode(lengthx, intcon(eshift))); 7855 } 7856 Node* size = _gvn.transform( new AddXNode(headerx, abody) ); 7857 if (round_mask != 0) { 7858 size = _gvn.transform( new AndXNode(size, MakeConX(~round_mask)) ); 7859 } 7860 size = ConvX2L(size); 7861 set_result(size); 7862 } 7863 } else { 7864 // Layout helper is not constant, need to test for array-ness at runtime. 7865 7866 enum { _instance_path = 1, _array_path, PATH_LIMIT }; 7867 RegionNode* result_reg = new RegionNode(PATH_LIMIT); 7868 PhiNode* result_val = new PhiNode(result_reg, TypeLong::LONG); 7869 record_for_igvn(result_reg); 7870 7871 Node* array_ctl = generate_array_guard(klass_node, NULL); 7872 if (array_ctl != NULL) { 7873 // Array case: size is round(header + element_size*arraylength). 7874 // Since arraylength is different for every array instance, we have to 7875 // compute the whole thing at runtime. 7876 7877 PreserveJVMState pjvms(this); 7878 set_control(array_ctl); 7879 Node* arr_length = load_array_length(obj); 7880 7881 int round_mask = MinObjAlignmentInBytes - 1; 7882 Node* mask = intcon(round_mask); 7883 7884 Node* hss = intcon(Klass::_lh_header_size_shift); 7885 Node* hsm = intcon(Klass::_lh_header_size_mask); 7886 Node* header_size = _gvn.transform(new URShiftINode(layout_val, hss)); 7887 header_size = _gvn.transform(new AndINode(header_size, hsm)); 7888 header_size = _gvn.transform(new AddINode(header_size, mask)); 7889 7890 // There is no need to mask or shift this value. 7891 // The semantics of LShiftINode include an implicit mask to 0x1F. 7892 assert(Klass::_lh_log2_element_size_shift == 0, "use shift in place"); 7893 Node* elem_shift = layout_val; 7894 7895 Node* lengthx = ConvI2X(arr_length); 7896 Node* headerx = ConvI2X(header_size); 7897 7898 Node* abody = _gvn.transform(new LShiftXNode(lengthx, elem_shift)); 7899 Node* size = _gvn.transform(new AddXNode(headerx, abody)); 7900 if (round_mask != 0) { 7901 size = _gvn.transform(new AndXNode(size, MakeConX(~round_mask))); 7902 } 7903 size = ConvX2L(size); 7904 7905 result_reg->init_req(_array_path, control()); 7906 result_val->init_req(_array_path, size); 7907 } 7908 7909 if (!stopped()) { 7910 // Instance case: the layout helper gives us instance size almost directly, 7911 // but we need to mask out the _lh_instance_slow_path_bit. 7912 Node* size = ConvI2X(layout_val); 7913 assert((int) Klass::_lh_instance_slow_path_bit < BytesPerLong, "clear bit"); 7914 Node* mask = MakeConX(~(intptr_t) right_n_bits(LogBytesPerLong)); 7915 size = _gvn.transform(new AndXNode(size, mask)); 7916 size = ConvX2L(size); 7917 7918 result_reg->init_req(_instance_path, control()); 7919 result_val->init_req(_instance_path, size); 7920 } 7921 7922 set_result(result_reg, result_val); 7923 } 7924 7925 return true; 7926 } 7927 7928 //------------------------------- inline_blackhole -------------------------------------- 7929 // 7930 // Make sure all arguments to this node are alive. 7931 // This matches methods that were requested to be blackholed through compile commands. 7932 // 7933 bool LibraryCallKit::inline_blackhole() { 7934 assert(callee()->is_static(), "Should have been checked before: only static methods here"); 7935 assert(callee()->is_empty(), "Should have been checked before: only empty methods here"); 7936 assert(callee()->holder()->is_loaded(), "Should have been checked before: only methods for loaded classes here"); 7937 7938 // Blackhole node pinches only the control, not memory. This allows 7939 // the blackhole to be pinned in the loop that computes blackholed 7940 // values, but have no other side effects, like breaking the optimizations 7941 // across the blackhole. 7942 7943 Node* bh = _gvn.transform(new BlackholeNode(control())); 7944 set_control(_gvn.transform(new ProjNode(bh, TypeFunc::Control))); 7945 7946 // Bind call arguments as blackhole arguments to keep them alive 7947 uint nargs = callee()->arg_size(); 7948 for (uint i = 0; i < nargs; i++) { 7949 bh->add_req(argument(i)); 7950 } 7951 7952 return true; 7953 } 7954 7955 bool LibraryCallKit::inline_timestamp(bool serial) { 7956 insert_mem_bar(Op_MemBarCPUOrder); 7957 Node* node = serial ? 7958 _gvn.transform(new TimestampSerialNode(control())) : 7959 _gvn.transform(new TimestampNode(control())); 7960 set_result(node); 7961 insert_mem_bar(Op_MemBarCPUOrder); 7962 return true; 7963 } 7964 7965 bool LibraryCallKit::inline_addressOf() { 7966 Node* obj = argument(0); 7967 Node* raw_val = _gvn.transform(new CastP2XNode(NULL, obj)); 7968 Node* long_val = ConvX2L(raw_val); 7969 set_result(long_val); 7970 return true; 7971 }