1 /* 2 * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "asm/macroAssembler.hpp" 27 #include "ci/ciUtilities.inline.hpp" 28 #include "classfile/vmIntrinsics.hpp" 29 #include "compiler/compileBroker.hpp" 30 #include "compiler/compileLog.hpp" 31 #include "gc/shared/barrierSet.hpp" 32 #include "jfr/support/jfrIntrinsics.hpp" 33 #include "memory/resourceArea.hpp" 34 #include "oops/klass.inline.hpp" 35 #include "oops/objArrayKlass.hpp" 36 #include "opto/addnode.hpp" 37 #include "opto/arraycopynode.hpp" 38 #include "opto/c2compiler.hpp" 39 #include "opto/castnode.hpp" 40 #include "opto/cfgnode.hpp" 41 #include "opto/convertnode.hpp" 42 #include "opto/countbitsnode.hpp" 43 #include "opto/idealKit.hpp" 44 #include "opto/library_call.hpp" 45 #include "opto/mathexactnode.hpp" 46 #include "opto/mulnode.hpp" 47 #include "opto/narrowptrnode.hpp" 48 #include "opto/opaquenode.hpp" 49 #include "opto/parse.hpp" 50 #include "opto/runtime.hpp" 51 #include "opto/rootnode.hpp" 52 #include "opto/subnode.hpp" 53 #include "prims/unsafe.hpp" 54 #include "runtime/objectMonitor.hpp" 55 #include "runtime/sharedRuntime.hpp" 56 #include "runtime/stubRoutines.hpp" 57 #include "utilities/macros.hpp" 58 #include "utilities/powerOfTwo.hpp" 59 60 #if INCLUDE_JFR 61 #include "jfr/jfr.hpp" 62 #endif 63 64 //---------------------------make_vm_intrinsic---------------------------- 65 CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) { 66 vmIntrinsicID id = m->intrinsic_id(); 67 assert(id != vmIntrinsics::_none, "must be a VM intrinsic"); 68 69 if (!m->is_loaded()) { 70 // Do not attempt to inline unloaded methods. 71 return NULL; 72 } 73 74 C2Compiler* compiler = (C2Compiler*)CompileBroker::compiler(CompLevel_full_optimization); 75 bool is_available = false; 76 77 { 78 // For calling is_intrinsic_supported and is_intrinsic_disabled_by_flag 79 // the compiler must transition to '_thread_in_vm' state because both 80 // methods access VM-internal data. 81 VM_ENTRY_MARK; 82 methodHandle mh(THREAD, m->get_Method()); 83 is_available = compiler != NULL && compiler->is_intrinsic_supported(mh, is_virtual) && 84 !C->directive()->is_intrinsic_disabled(mh) && 85 !vmIntrinsics::is_disabled_by_flags(mh); 86 87 } 88 89 if (is_available) { 90 assert(id <= vmIntrinsics::LAST_COMPILER_INLINE, "caller responsibility"); 91 assert(id != vmIntrinsics::_Object_init && id != vmIntrinsics::_invoke, "enum out of order?"); 92 return new LibraryIntrinsic(m, is_virtual, 93 vmIntrinsics::predicates_needed(id), 94 vmIntrinsics::does_virtual_dispatch(id), 95 id); 96 } else { 97 return NULL; 98 } 99 } 100 101 JVMState* LibraryIntrinsic::generate(JVMState* jvms) { 102 LibraryCallKit kit(jvms, this); 103 Compile* C = kit.C; 104 int nodes = C->unique(); 105 #ifndef PRODUCT 106 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 107 char buf[1000]; 108 const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf)); 109 tty->print_cr("Intrinsic %s", str); 110 } 111 #endif 112 ciMethod* callee = kit.callee(); 113 const int bci = kit.bci(); 114 #ifdef ASSERT 115 Node* ctrl = kit.control(); 116 #endif 117 // Try to inline the intrinsic. 118 if (callee->check_intrinsic_candidate() && 119 kit.try_to_inline(_last_predicate)) { 120 const char *inline_msg = is_virtual() ? "(intrinsic, virtual)" 121 : "(intrinsic)"; 122 CompileTask::print_inlining_ul(callee, jvms->depth() - 1, bci, inline_msg); 123 if (C->print_intrinsics() || C->print_inlining()) { 124 C->print_inlining(callee, jvms->depth() - 1, bci, inline_msg); 125 } 126 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked); 127 if (C->log()) { 128 C->log()->elem("intrinsic id='%s'%s nodes='%d'", 129 vmIntrinsics::name_at(intrinsic_id()), 130 (is_virtual() ? " virtual='1'" : ""), 131 C->unique() - nodes); 132 } 133 // Push the result from the inlined method onto the stack. 134 kit.push_result(); 135 C->print_inlining_update(this); 136 return kit.transfer_exceptions_into_jvms(); 137 } 138 139 // The intrinsic bailed out 140 assert(ctrl == kit.control(), "Control flow was added although the intrinsic bailed out"); 141 if (jvms->has_method()) { 142 // Not a root compile. 143 const char* msg; 144 if (callee->intrinsic_candidate()) { 145 msg = is_virtual() ? "failed to inline (intrinsic, virtual)" : "failed to inline (intrinsic)"; 146 } else { 147 msg = is_virtual() ? "failed to inline (intrinsic, virtual), method not annotated" 148 : "failed to inline (intrinsic), method not annotated"; 149 } 150 CompileTask::print_inlining_ul(callee, jvms->depth() - 1, bci, msg); 151 if (C->print_intrinsics() || C->print_inlining()) { 152 C->print_inlining(callee, jvms->depth() - 1, bci, msg); 153 } 154 } else { 155 // Root compile 156 ResourceMark rm; 157 stringStream msg_stream; 158 msg_stream.print("Did not generate intrinsic %s%s at bci:%d in", 159 vmIntrinsics::name_at(intrinsic_id()), 160 is_virtual() ? " (virtual)" : "", bci); 161 const char *msg = msg_stream.as_string(); 162 log_debug(jit, inlining)("%s", msg); 163 if (C->print_intrinsics() || C->print_inlining()) { 164 tty->print("%s", msg); 165 } 166 } 167 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed); 168 C->print_inlining_update(this); 169 170 return NULL; 171 } 172 173 Node* LibraryIntrinsic::generate_predicate(JVMState* jvms, int predicate) { 174 LibraryCallKit kit(jvms, this); 175 Compile* C = kit.C; 176 int nodes = C->unique(); 177 _last_predicate = predicate; 178 #ifndef PRODUCT 179 assert(is_predicated() && predicate < predicates_count(), "sanity"); 180 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 181 char buf[1000]; 182 const char* str = vmIntrinsics::short_name_as_C_string(intrinsic_id(), buf, sizeof(buf)); 183 tty->print_cr("Predicate for intrinsic %s", str); 184 } 185 #endif 186 ciMethod* callee = kit.callee(); 187 const int bci = kit.bci(); 188 189 Node* slow_ctl = kit.try_to_predicate(predicate); 190 if (!kit.failing()) { 191 const char *inline_msg = is_virtual() ? "(intrinsic, virtual, predicate)" 192 : "(intrinsic, predicate)"; 193 CompileTask::print_inlining_ul(callee, jvms->depth() - 1, bci, inline_msg); 194 if (C->print_intrinsics() || C->print_inlining()) { 195 C->print_inlining(callee, jvms->depth() - 1, bci, inline_msg); 196 } 197 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_worked); 198 if (C->log()) { 199 C->log()->elem("predicate_intrinsic id='%s'%s nodes='%d'", 200 vmIntrinsics::name_at(intrinsic_id()), 201 (is_virtual() ? " virtual='1'" : ""), 202 C->unique() - nodes); 203 } 204 return slow_ctl; // Could be NULL if the check folds. 205 } 206 207 // The intrinsic bailed out 208 if (jvms->has_method()) { 209 // Not a root compile. 210 const char* msg = "failed to generate predicate for intrinsic"; 211 CompileTask::print_inlining_ul(kit.callee(), jvms->depth() - 1, bci, msg); 212 if (C->print_intrinsics() || C->print_inlining()) { 213 C->print_inlining(kit.callee(), jvms->depth() - 1, bci, msg); 214 } 215 } else { 216 // Root compile 217 ResourceMark rm; 218 stringStream msg_stream; 219 msg_stream.print("Did not generate intrinsic %s%s at bci:%d in", 220 vmIntrinsics::name_at(intrinsic_id()), 221 is_virtual() ? " (virtual)" : "", bci); 222 const char *msg = msg_stream.as_string(); 223 log_debug(jit, inlining)("%s", msg); 224 if (C->print_intrinsics() || C->print_inlining()) { 225 C->print_inlining_stream()->print("%s", msg); 226 } 227 } 228 C->gather_intrinsic_statistics(intrinsic_id(), is_virtual(), Compile::_intrinsic_failed); 229 return NULL; 230 } 231 232 bool LibraryCallKit::try_to_inline(int predicate) { 233 // Handle symbolic names for otherwise undistinguished boolean switches: 234 const bool is_store = true; 235 const bool is_compress = true; 236 const bool is_static = true; 237 const bool is_volatile = true; 238 239 if (!jvms()->has_method()) { 240 // Root JVMState has a null method. 241 assert(map()->memory()->Opcode() == Op_Parm, ""); 242 // Insert the memory aliasing node 243 set_all_memory(reset_memory()); 244 } 245 assert(merged_memory(), ""); 246 247 switch (intrinsic_id()) { 248 case vmIntrinsics::_hashCode: return inline_native_hashcode(intrinsic()->is_virtual(), !is_static); 249 case vmIntrinsics::_identityHashCode: return inline_native_hashcode(/*!virtual*/ false, is_static); 250 case vmIntrinsics::_getClass: return inline_native_getClass(); 251 252 case vmIntrinsics::_ceil: 253 case vmIntrinsics::_floor: 254 case vmIntrinsics::_rint: 255 case vmIntrinsics::_dsin: 256 case vmIntrinsics::_dcos: 257 case vmIntrinsics::_dtan: 258 case vmIntrinsics::_dabs: 259 case vmIntrinsics::_fabs: 260 case vmIntrinsics::_iabs: 261 case vmIntrinsics::_labs: 262 case vmIntrinsics::_datan2: 263 case vmIntrinsics::_dsqrt: 264 case vmIntrinsics::_dsqrt_strict: 265 case vmIntrinsics::_dexp: 266 case vmIntrinsics::_dlog: 267 case vmIntrinsics::_dlog10: 268 case vmIntrinsics::_dpow: 269 case vmIntrinsics::_dcopySign: 270 case vmIntrinsics::_fcopySign: 271 case vmIntrinsics::_dsignum: 272 case vmIntrinsics::_roundF: 273 case vmIntrinsics::_roundD: 274 case vmIntrinsics::_fsignum: return inline_math_native(intrinsic_id()); 275 276 case vmIntrinsics::_notify: 277 case vmIntrinsics::_notifyAll: 278 return inline_notify(intrinsic_id()); 279 280 case vmIntrinsics::_addExactI: return inline_math_addExactI(false /* add */); 281 case vmIntrinsics::_addExactL: return inline_math_addExactL(false /* add */); 282 case vmIntrinsics::_decrementExactI: return inline_math_subtractExactI(true /* decrement */); 283 case vmIntrinsics::_decrementExactL: return inline_math_subtractExactL(true /* decrement */); 284 case vmIntrinsics::_incrementExactI: return inline_math_addExactI(true /* increment */); 285 case vmIntrinsics::_incrementExactL: return inline_math_addExactL(true /* increment */); 286 case vmIntrinsics::_multiplyExactI: return inline_math_multiplyExactI(); 287 case vmIntrinsics::_multiplyExactL: return inline_math_multiplyExactL(); 288 case vmIntrinsics::_multiplyHigh: return inline_math_multiplyHigh(); 289 case vmIntrinsics::_unsignedMultiplyHigh: return inline_math_unsignedMultiplyHigh(); 290 case vmIntrinsics::_negateExactI: return inline_math_negateExactI(); 291 case vmIntrinsics::_negateExactL: return inline_math_negateExactL(); 292 case vmIntrinsics::_subtractExactI: return inline_math_subtractExactI(false /* subtract */); 293 case vmIntrinsics::_subtractExactL: return inline_math_subtractExactL(false /* subtract */); 294 295 case vmIntrinsics::_arraycopy: return inline_arraycopy(); 296 297 case vmIntrinsics::_compareToL: return inline_string_compareTo(StrIntrinsicNode::LL); 298 case vmIntrinsics::_compareToU: return inline_string_compareTo(StrIntrinsicNode::UU); 299 case vmIntrinsics::_compareToLU: return inline_string_compareTo(StrIntrinsicNode::LU); 300 case vmIntrinsics::_compareToUL: return inline_string_compareTo(StrIntrinsicNode::UL); 301 302 case vmIntrinsics::_indexOfL: return inline_string_indexOf(StrIntrinsicNode::LL); 303 case vmIntrinsics::_indexOfU: return inline_string_indexOf(StrIntrinsicNode::UU); 304 case vmIntrinsics::_indexOfUL: return inline_string_indexOf(StrIntrinsicNode::UL); 305 case vmIntrinsics::_indexOfIL: return inline_string_indexOfI(StrIntrinsicNode::LL); 306 case vmIntrinsics::_indexOfIU: return inline_string_indexOfI(StrIntrinsicNode::UU); 307 case vmIntrinsics::_indexOfIUL: return inline_string_indexOfI(StrIntrinsicNode::UL); 308 case vmIntrinsics::_indexOfU_char: return inline_string_indexOfChar(StrIntrinsicNode::U); 309 case vmIntrinsics::_indexOfL_char: return inline_string_indexOfChar(StrIntrinsicNode::L); 310 311 case vmIntrinsics::_equalsL: return inline_string_equals(StrIntrinsicNode::LL); 312 case vmIntrinsics::_equalsU: return inline_string_equals(StrIntrinsicNode::UU); 313 314 case vmIntrinsics::_toBytesStringU: return inline_string_toBytesU(); 315 case vmIntrinsics::_getCharsStringU: return inline_string_getCharsU(); 316 case vmIntrinsics::_getCharStringU: return inline_string_char_access(!is_store); 317 case vmIntrinsics::_putCharStringU: return inline_string_char_access( is_store); 318 319 case vmIntrinsics::_compressStringC: 320 case vmIntrinsics::_compressStringB: return inline_string_copy( is_compress); 321 case vmIntrinsics::_inflateStringC: 322 case vmIntrinsics::_inflateStringB: return inline_string_copy(!is_compress); 323 324 case vmIntrinsics::_getReference: return inline_unsafe_access(!is_store, T_OBJECT, Relaxed, false); 325 case vmIntrinsics::_getBoolean: return inline_unsafe_access(!is_store, T_BOOLEAN, Relaxed, false); 326 case vmIntrinsics::_getByte: return inline_unsafe_access(!is_store, T_BYTE, Relaxed, false); 327 case vmIntrinsics::_getShort: return inline_unsafe_access(!is_store, T_SHORT, Relaxed, false); 328 case vmIntrinsics::_getChar: return inline_unsafe_access(!is_store, T_CHAR, Relaxed, false); 329 case vmIntrinsics::_getInt: return inline_unsafe_access(!is_store, T_INT, Relaxed, false); 330 case vmIntrinsics::_getLong: return inline_unsafe_access(!is_store, T_LONG, Relaxed, false); 331 case vmIntrinsics::_getFloat: return inline_unsafe_access(!is_store, T_FLOAT, Relaxed, false); 332 case vmIntrinsics::_getDouble: return inline_unsafe_access(!is_store, T_DOUBLE, Relaxed, false); 333 334 case vmIntrinsics::_putReference: return inline_unsafe_access( is_store, T_OBJECT, Relaxed, false); 335 case vmIntrinsics::_putBoolean: return inline_unsafe_access( is_store, T_BOOLEAN, Relaxed, false); 336 case vmIntrinsics::_putByte: return inline_unsafe_access( is_store, T_BYTE, Relaxed, false); 337 case vmIntrinsics::_putShort: return inline_unsafe_access( is_store, T_SHORT, Relaxed, false); 338 case vmIntrinsics::_putChar: return inline_unsafe_access( is_store, T_CHAR, Relaxed, false); 339 case vmIntrinsics::_putInt: return inline_unsafe_access( is_store, T_INT, Relaxed, false); 340 case vmIntrinsics::_putLong: return inline_unsafe_access( is_store, T_LONG, Relaxed, false); 341 case vmIntrinsics::_putFloat: return inline_unsafe_access( is_store, T_FLOAT, Relaxed, false); 342 case vmIntrinsics::_putDouble: return inline_unsafe_access( is_store, T_DOUBLE, Relaxed, false); 343 344 case vmIntrinsics::_getReferenceVolatile: return inline_unsafe_access(!is_store, T_OBJECT, Volatile, false); 345 case vmIntrinsics::_getBooleanVolatile: return inline_unsafe_access(!is_store, T_BOOLEAN, Volatile, false); 346 case vmIntrinsics::_getByteVolatile: return inline_unsafe_access(!is_store, T_BYTE, Volatile, false); 347 case vmIntrinsics::_getShortVolatile: return inline_unsafe_access(!is_store, T_SHORT, Volatile, false); 348 case vmIntrinsics::_getCharVolatile: return inline_unsafe_access(!is_store, T_CHAR, Volatile, false); 349 case vmIntrinsics::_getIntVolatile: return inline_unsafe_access(!is_store, T_INT, Volatile, false); 350 case vmIntrinsics::_getLongVolatile: return inline_unsafe_access(!is_store, T_LONG, Volatile, false); 351 case vmIntrinsics::_getFloatVolatile: return inline_unsafe_access(!is_store, T_FLOAT, Volatile, false); 352 case vmIntrinsics::_getDoubleVolatile: return inline_unsafe_access(!is_store, T_DOUBLE, Volatile, false); 353 354 case vmIntrinsics::_putReferenceVolatile: return inline_unsafe_access( is_store, T_OBJECT, Volatile, false); 355 case vmIntrinsics::_putBooleanVolatile: return inline_unsafe_access( is_store, T_BOOLEAN, Volatile, false); 356 case vmIntrinsics::_putByteVolatile: return inline_unsafe_access( is_store, T_BYTE, Volatile, false); 357 case vmIntrinsics::_putShortVolatile: return inline_unsafe_access( is_store, T_SHORT, Volatile, false); 358 case vmIntrinsics::_putCharVolatile: return inline_unsafe_access( is_store, T_CHAR, Volatile, false); 359 case vmIntrinsics::_putIntVolatile: return inline_unsafe_access( is_store, T_INT, Volatile, false); 360 case vmIntrinsics::_putLongVolatile: return inline_unsafe_access( is_store, T_LONG, Volatile, false); 361 case vmIntrinsics::_putFloatVolatile: return inline_unsafe_access( is_store, T_FLOAT, Volatile, false); 362 case vmIntrinsics::_putDoubleVolatile: return inline_unsafe_access( is_store, T_DOUBLE, Volatile, false); 363 364 case vmIntrinsics::_getShortUnaligned: return inline_unsafe_access(!is_store, T_SHORT, Relaxed, true); 365 case vmIntrinsics::_getCharUnaligned: return inline_unsafe_access(!is_store, T_CHAR, Relaxed, true); 366 case vmIntrinsics::_getIntUnaligned: return inline_unsafe_access(!is_store, T_INT, Relaxed, true); 367 case vmIntrinsics::_getLongUnaligned: return inline_unsafe_access(!is_store, T_LONG, Relaxed, true); 368 369 case vmIntrinsics::_putShortUnaligned: return inline_unsafe_access( is_store, T_SHORT, Relaxed, true); 370 case vmIntrinsics::_putCharUnaligned: return inline_unsafe_access( is_store, T_CHAR, Relaxed, true); 371 case vmIntrinsics::_putIntUnaligned: return inline_unsafe_access( is_store, T_INT, Relaxed, true); 372 case vmIntrinsics::_putLongUnaligned: return inline_unsafe_access( is_store, T_LONG, Relaxed, true); 373 374 case vmIntrinsics::_getReferenceAcquire: return inline_unsafe_access(!is_store, T_OBJECT, Acquire, false); 375 case vmIntrinsics::_getBooleanAcquire: return inline_unsafe_access(!is_store, T_BOOLEAN, Acquire, false); 376 case vmIntrinsics::_getByteAcquire: return inline_unsafe_access(!is_store, T_BYTE, Acquire, false); 377 case vmIntrinsics::_getShortAcquire: return inline_unsafe_access(!is_store, T_SHORT, Acquire, false); 378 case vmIntrinsics::_getCharAcquire: return inline_unsafe_access(!is_store, T_CHAR, Acquire, false); 379 case vmIntrinsics::_getIntAcquire: return inline_unsafe_access(!is_store, T_INT, Acquire, false); 380 case vmIntrinsics::_getLongAcquire: return inline_unsafe_access(!is_store, T_LONG, Acquire, false); 381 case vmIntrinsics::_getFloatAcquire: return inline_unsafe_access(!is_store, T_FLOAT, Acquire, false); 382 case vmIntrinsics::_getDoubleAcquire: return inline_unsafe_access(!is_store, T_DOUBLE, Acquire, false); 383 384 case vmIntrinsics::_putReferenceRelease: return inline_unsafe_access( is_store, T_OBJECT, Release, false); 385 case vmIntrinsics::_putBooleanRelease: return inline_unsafe_access( is_store, T_BOOLEAN, Release, false); 386 case vmIntrinsics::_putByteRelease: return inline_unsafe_access( is_store, T_BYTE, Release, false); 387 case vmIntrinsics::_putShortRelease: return inline_unsafe_access( is_store, T_SHORT, Release, false); 388 case vmIntrinsics::_putCharRelease: return inline_unsafe_access( is_store, T_CHAR, Release, false); 389 case vmIntrinsics::_putIntRelease: return inline_unsafe_access( is_store, T_INT, Release, false); 390 case vmIntrinsics::_putLongRelease: return inline_unsafe_access( is_store, T_LONG, Release, false); 391 case vmIntrinsics::_putFloatRelease: return inline_unsafe_access( is_store, T_FLOAT, Release, false); 392 case vmIntrinsics::_putDoubleRelease: return inline_unsafe_access( is_store, T_DOUBLE, Release, false); 393 394 case vmIntrinsics::_getReferenceOpaque: return inline_unsafe_access(!is_store, T_OBJECT, Opaque, false); 395 case vmIntrinsics::_getBooleanOpaque: return inline_unsafe_access(!is_store, T_BOOLEAN, Opaque, false); 396 case vmIntrinsics::_getByteOpaque: return inline_unsafe_access(!is_store, T_BYTE, Opaque, false); 397 case vmIntrinsics::_getShortOpaque: return inline_unsafe_access(!is_store, T_SHORT, Opaque, false); 398 case vmIntrinsics::_getCharOpaque: return inline_unsafe_access(!is_store, T_CHAR, Opaque, false); 399 case vmIntrinsics::_getIntOpaque: return inline_unsafe_access(!is_store, T_INT, Opaque, false); 400 case vmIntrinsics::_getLongOpaque: return inline_unsafe_access(!is_store, T_LONG, Opaque, false); 401 case vmIntrinsics::_getFloatOpaque: return inline_unsafe_access(!is_store, T_FLOAT, Opaque, false); 402 case vmIntrinsics::_getDoubleOpaque: return inline_unsafe_access(!is_store, T_DOUBLE, Opaque, false); 403 404 case vmIntrinsics::_putReferenceOpaque: return inline_unsafe_access( is_store, T_OBJECT, Opaque, false); 405 case vmIntrinsics::_putBooleanOpaque: return inline_unsafe_access( is_store, T_BOOLEAN, Opaque, false); 406 case vmIntrinsics::_putByteOpaque: return inline_unsafe_access( is_store, T_BYTE, Opaque, false); 407 case vmIntrinsics::_putShortOpaque: return inline_unsafe_access( is_store, T_SHORT, Opaque, false); 408 case vmIntrinsics::_putCharOpaque: return inline_unsafe_access( is_store, T_CHAR, Opaque, false); 409 case vmIntrinsics::_putIntOpaque: return inline_unsafe_access( is_store, T_INT, Opaque, false); 410 case vmIntrinsics::_putLongOpaque: return inline_unsafe_access( is_store, T_LONG, Opaque, false); 411 case vmIntrinsics::_putFloatOpaque: return inline_unsafe_access( is_store, T_FLOAT, Opaque, false); 412 case vmIntrinsics::_putDoubleOpaque: return inline_unsafe_access( is_store, T_DOUBLE, Opaque, false); 413 414 case vmIntrinsics::_compareAndSetReference: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap, Volatile); 415 case vmIntrinsics::_compareAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap, Volatile); 416 case vmIntrinsics::_compareAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap, Volatile); 417 case vmIntrinsics::_compareAndSetInt: return inline_unsafe_load_store(T_INT, LS_cmp_swap, Volatile); 418 case vmIntrinsics::_compareAndSetLong: return inline_unsafe_load_store(T_LONG, LS_cmp_swap, Volatile); 419 420 case vmIntrinsics::_weakCompareAndSetReferencePlain: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Relaxed); 421 case vmIntrinsics::_weakCompareAndSetReferenceAcquire: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Acquire); 422 case vmIntrinsics::_weakCompareAndSetReferenceRelease: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Release); 423 case vmIntrinsics::_weakCompareAndSetReference: return inline_unsafe_load_store(T_OBJECT, LS_cmp_swap_weak, Volatile); 424 case vmIntrinsics::_weakCompareAndSetBytePlain: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Relaxed); 425 case vmIntrinsics::_weakCompareAndSetByteAcquire: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Acquire); 426 case vmIntrinsics::_weakCompareAndSetByteRelease: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Release); 427 case vmIntrinsics::_weakCompareAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_cmp_swap_weak, Volatile); 428 case vmIntrinsics::_weakCompareAndSetShortPlain: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Relaxed); 429 case vmIntrinsics::_weakCompareAndSetShortAcquire: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Acquire); 430 case vmIntrinsics::_weakCompareAndSetShortRelease: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Release); 431 case vmIntrinsics::_weakCompareAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_cmp_swap_weak, Volatile); 432 case vmIntrinsics::_weakCompareAndSetIntPlain: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Relaxed); 433 case vmIntrinsics::_weakCompareAndSetIntAcquire: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Acquire); 434 case vmIntrinsics::_weakCompareAndSetIntRelease: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Release); 435 case vmIntrinsics::_weakCompareAndSetInt: return inline_unsafe_load_store(T_INT, LS_cmp_swap_weak, Volatile); 436 case vmIntrinsics::_weakCompareAndSetLongPlain: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Relaxed); 437 case vmIntrinsics::_weakCompareAndSetLongAcquire: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Acquire); 438 case vmIntrinsics::_weakCompareAndSetLongRelease: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Release); 439 case vmIntrinsics::_weakCompareAndSetLong: return inline_unsafe_load_store(T_LONG, LS_cmp_swap_weak, Volatile); 440 441 case vmIntrinsics::_compareAndExchangeReference: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Volatile); 442 case vmIntrinsics::_compareAndExchangeReferenceAcquire: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Acquire); 443 case vmIntrinsics::_compareAndExchangeReferenceRelease: return inline_unsafe_load_store(T_OBJECT, LS_cmp_exchange, Release); 444 case vmIntrinsics::_compareAndExchangeByte: return inline_unsafe_load_store(T_BYTE, LS_cmp_exchange, Volatile); 445 case vmIntrinsics::_compareAndExchangeByteAcquire: return inline_unsafe_load_store(T_BYTE, LS_cmp_exchange, Acquire); 446 case vmIntrinsics::_compareAndExchangeByteRelease: return inline_unsafe_load_store(T_BYTE, LS_cmp_exchange, Release); 447 case vmIntrinsics::_compareAndExchangeShort: return inline_unsafe_load_store(T_SHORT, LS_cmp_exchange, Volatile); 448 case vmIntrinsics::_compareAndExchangeShortAcquire: return inline_unsafe_load_store(T_SHORT, LS_cmp_exchange, Acquire); 449 case vmIntrinsics::_compareAndExchangeShortRelease: return inline_unsafe_load_store(T_SHORT, LS_cmp_exchange, Release); 450 case vmIntrinsics::_compareAndExchangeInt: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Volatile); 451 case vmIntrinsics::_compareAndExchangeIntAcquire: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Acquire); 452 case vmIntrinsics::_compareAndExchangeIntRelease: return inline_unsafe_load_store(T_INT, LS_cmp_exchange, Release); 453 case vmIntrinsics::_compareAndExchangeLong: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Volatile); 454 case vmIntrinsics::_compareAndExchangeLongAcquire: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Acquire); 455 case vmIntrinsics::_compareAndExchangeLongRelease: return inline_unsafe_load_store(T_LONG, LS_cmp_exchange, Release); 456 457 case vmIntrinsics::_getAndAddByte: return inline_unsafe_load_store(T_BYTE, LS_get_add, Volatile); 458 case vmIntrinsics::_getAndAddShort: return inline_unsafe_load_store(T_SHORT, LS_get_add, Volatile); 459 case vmIntrinsics::_getAndAddInt: return inline_unsafe_load_store(T_INT, LS_get_add, Volatile); 460 case vmIntrinsics::_getAndAddLong: return inline_unsafe_load_store(T_LONG, LS_get_add, Volatile); 461 462 case vmIntrinsics::_getAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_get_set, Volatile); 463 case vmIntrinsics::_getAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_get_set, Volatile); 464 case vmIntrinsics::_getAndSetInt: return inline_unsafe_load_store(T_INT, LS_get_set, Volatile); 465 case vmIntrinsics::_getAndSetLong: return inline_unsafe_load_store(T_LONG, LS_get_set, Volatile); 466 case vmIntrinsics::_getAndSetReference: return inline_unsafe_load_store(T_OBJECT, LS_get_set, Volatile); 467 468 case vmIntrinsics::_loadFence: 469 case vmIntrinsics::_storeFence: 470 case vmIntrinsics::_storeStoreFence: 471 case vmIntrinsics::_fullFence: return inline_unsafe_fence(intrinsic_id()); 472 473 case vmIntrinsics::_onSpinWait: return inline_onspinwait(); 474 475 case vmIntrinsics::_currentThread: return inline_native_currentThread(); 476 477 #ifdef JFR_HAVE_INTRINSICS 478 case vmIntrinsics::_counterTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, JFR_TIME_FUNCTION), "counterTime"); 479 case vmIntrinsics::_getClassId: return inline_native_classID(); 480 case vmIntrinsics::_getEventWriter: return inline_native_getEventWriter(); 481 #endif 482 case vmIntrinsics::_currentTimeMillis: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis"); 483 case vmIntrinsics::_nanoTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime"); 484 case vmIntrinsics::_writeback0: return inline_unsafe_writeback0(); 485 case vmIntrinsics::_writebackPreSync0: return inline_unsafe_writebackSync0(true); 486 case vmIntrinsics::_writebackPostSync0: return inline_unsafe_writebackSync0(false); 487 case vmIntrinsics::_allocateInstance: return inline_unsafe_allocate(); 488 case vmIntrinsics::_copyMemory: return inline_unsafe_copyMemory(); 489 case vmIntrinsics::_getLength: return inline_native_getLength(); 490 case vmIntrinsics::_copyOf: return inline_array_copyOf(false); 491 case vmIntrinsics::_copyOfRange: return inline_array_copyOf(true); 492 case vmIntrinsics::_equalsB: return inline_array_equals(StrIntrinsicNode::LL); 493 case vmIntrinsics::_equalsC: return inline_array_equals(StrIntrinsicNode::UU); 494 case vmIntrinsics::_Preconditions_checkIndex: return inline_preconditions_checkIndex(T_INT); 495 case vmIntrinsics::_Preconditions_checkLongIndex: return inline_preconditions_checkIndex(T_LONG); 496 case vmIntrinsics::_clone: return inline_native_clone(intrinsic()->is_virtual()); 497 498 case vmIntrinsics::_allocateUninitializedArray: return inline_unsafe_newArray(true); 499 case vmIntrinsics::_newArray: return inline_unsafe_newArray(false); 500 501 case vmIntrinsics::_isAssignableFrom: return inline_native_subtype_check(); 502 503 case vmIntrinsics::_isInstance: 504 case vmIntrinsics::_getModifiers: 505 case vmIntrinsics::_isInterface: 506 case vmIntrinsics::_isArray: 507 case vmIntrinsics::_isPrimitive: 508 case vmIntrinsics::_isHidden: 509 case vmIntrinsics::_getSuperclass: 510 case vmIntrinsics::_getClassAccessFlags: return inline_native_Class_query(intrinsic_id()); 511 512 case vmIntrinsics::_floatToRawIntBits: 513 case vmIntrinsics::_floatToIntBits: 514 case vmIntrinsics::_intBitsToFloat: 515 case vmIntrinsics::_doubleToRawLongBits: 516 case vmIntrinsics::_doubleToLongBits: 517 case vmIntrinsics::_longBitsToDouble: return inline_fp_conversions(intrinsic_id()); 518 519 case vmIntrinsics::_numberOfLeadingZeros_i: 520 case vmIntrinsics::_numberOfLeadingZeros_l: 521 case vmIntrinsics::_numberOfTrailingZeros_i: 522 case vmIntrinsics::_numberOfTrailingZeros_l: 523 case vmIntrinsics::_bitCount_i: 524 case vmIntrinsics::_bitCount_l: 525 case vmIntrinsics::_reverseBytes_i: 526 case vmIntrinsics::_reverseBytes_l: 527 case vmIntrinsics::_reverseBytes_s: 528 case vmIntrinsics::_reverseBytes_c: return inline_number_methods(intrinsic_id()); 529 530 case vmIntrinsics::_divideUnsigned_i: 531 case vmIntrinsics::_divideUnsigned_l: 532 case vmIntrinsics::_remainderUnsigned_i: 533 case vmIntrinsics::_remainderUnsigned_l: return inline_divmod_methods(intrinsic_id()); 534 535 case vmIntrinsics::_getCallerClass: return inline_native_Reflection_getCallerClass(); 536 537 case vmIntrinsics::_Reference_get: return inline_reference_get(); 538 case vmIntrinsics::_Reference_refersTo0: return inline_reference_refersTo0(false); 539 case vmIntrinsics::_PhantomReference_refersTo0: return inline_reference_refersTo0(true); 540 541 case vmIntrinsics::_Class_cast: return inline_Class_cast(); 542 543 case vmIntrinsics::_aescrypt_encryptBlock: 544 case vmIntrinsics::_aescrypt_decryptBlock: return inline_aescrypt_Block(intrinsic_id()); 545 546 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt: 547 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt: 548 return inline_cipherBlockChaining_AESCrypt(intrinsic_id()); 549 550 case vmIntrinsics::_electronicCodeBook_encryptAESCrypt: 551 case vmIntrinsics::_electronicCodeBook_decryptAESCrypt: 552 return inline_electronicCodeBook_AESCrypt(intrinsic_id()); 553 554 case vmIntrinsics::_counterMode_AESCrypt: 555 return inline_counterMode_AESCrypt(intrinsic_id()); 556 557 case vmIntrinsics::_galoisCounterMode_AESCrypt: 558 return inline_galoisCounterMode_AESCrypt(); 559 560 case vmIntrinsics::_md5_implCompress: 561 case vmIntrinsics::_sha_implCompress: 562 case vmIntrinsics::_sha2_implCompress: 563 case vmIntrinsics::_sha5_implCompress: 564 case vmIntrinsics::_sha3_implCompress: 565 return inline_digestBase_implCompress(intrinsic_id()); 566 567 case vmIntrinsics::_digestBase_implCompressMB: 568 return inline_digestBase_implCompressMB(predicate); 569 570 case vmIntrinsics::_multiplyToLen: 571 return inline_multiplyToLen(); 572 573 case vmIntrinsics::_squareToLen: 574 return inline_squareToLen(); 575 576 case vmIntrinsics::_mulAdd: 577 return inline_mulAdd(); 578 579 case vmIntrinsics::_montgomeryMultiply: 580 return inline_montgomeryMultiply(); 581 case vmIntrinsics::_montgomerySquare: 582 return inline_montgomerySquare(); 583 584 case vmIntrinsics::_bigIntegerRightShiftWorker: 585 return inline_bigIntegerShift(true); 586 case vmIntrinsics::_bigIntegerLeftShiftWorker: 587 return inline_bigIntegerShift(false); 588 589 case vmIntrinsics::_vectorizedMismatch: 590 return inline_vectorizedMismatch(); 591 592 case vmIntrinsics::_ghash_processBlocks: 593 return inline_ghash_processBlocks(); 594 case vmIntrinsics::_base64_encodeBlock: 595 return inline_base64_encodeBlock(); 596 case vmIntrinsics::_base64_decodeBlock: 597 return inline_base64_decodeBlock(); 598 599 case vmIntrinsics::_encodeISOArray: 600 case vmIntrinsics::_encodeByteISOArray: 601 return inline_encodeISOArray(false); 602 case vmIntrinsics::_encodeAsciiArray: 603 return inline_encodeISOArray(true); 604 605 case vmIntrinsics::_updateCRC32: 606 return inline_updateCRC32(); 607 case vmIntrinsics::_updateBytesCRC32: 608 return inline_updateBytesCRC32(); 609 case vmIntrinsics::_updateByteBufferCRC32: 610 return inline_updateByteBufferCRC32(); 611 612 case vmIntrinsics::_updateBytesCRC32C: 613 return inline_updateBytesCRC32C(); 614 case vmIntrinsics::_updateDirectByteBufferCRC32C: 615 return inline_updateDirectByteBufferCRC32C(); 616 617 case vmIntrinsics::_updateBytesAdler32: 618 return inline_updateBytesAdler32(); 619 case vmIntrinsics::_updateByteBufferAdler32: 620 return inline_updateByteBufferAdler32(); 621 622 case vmIntrinsics::_profileBoolean: 623 return inline_profileBoolean(); 624 case vmIntrinsics::_isCompileConstant: 625 return inline_isCompileConstant(); 626 627 case vmIntrinsics::_countPositives: 628 return inline_countPositives(); 629 630 case vmIntrinsics::_fmaD: 631 case vmIntrinsics::_fmaF: 632 return inline_fma(intrinsic_id()); 633 634 case vmIntrinsics::_isDigit: 635 case vmIntrinsics::_isLowerCase: 636 case vmIntrinsics::_isUpperCase: 637 case vmIntrinsics::_isWhitespace: 638 return inline_character_compare(intrinsic_id()); 639 640 case vmIntrinsics::_min: 641 case vmIntrinsics::_max: 642 case vmIntrinsics::_min_strict: 643 case vmIntrinsics::_max_strict: 644 return inline_min_max(intrinsic_id()); 645 646 case vmIntrinsics::_maxF: 647 case vmIntrinsics::_minF: 648 case vmIntrinsics::_maxD: 649 case vmIntrinsics::_minD: 650 case vmIntrinsics::_maxF_strict: 651 case vmIntrinsics::_minF_strict: 652 case vmIntrinsics::_maxD_strict: 653 case vmIntrinsics::_minD_strict: 654 return inline_fp_min_max(intrinsic_id()); 655 656 case vmIntrinsics::_VectorUnaryOp: 657 return inline_vector_nary_operation(1); 658 case vmIntrinsics::_VectorBinaryOp: 659 return inline_vector_nary_operation(2); 660 case vmIntrinsics::_VectorTernaryOp: 661 return inline_vector_nary_operation(3); 662 case vmIntrinsics::_VectorFromBitsCoerced: 663 return inline_vector_frombits_coerced(); 664 case vmIntrinsics::_VectorShuffleIota: 665 return inline_vector_shuffle_iota(); 666 case vmIntrinsics::_VectorMaskOp: 667 return inline_vector_mask_operation(); 668 case vmIntrinsics::_VectorShuffleToVector: 669 return inline_vector_shuffle_to_vector(); 670 case vmIntrinsics::_VectorLoadOp: 671 return inline_vector_mem_operation(/*is_store=*/false); 672 case vmIntrinsics::_VectorLoadMaskedOp: 673 return inline_vector_mem_masked_operation(/*is_store*/false); 674 case vmIntrinsics::_VectorStoreOp: 675 return inline_vector_mem_operation(/*is_store=*/true); 676 case vmIntrinsics::_VectorStoreMaskedOp: 677 return inline_vector_mem_masked_operation(/*is_store=*/true); 678 case vmIntrinsics::_VectorGatherOp: 679 return inline_vector_gather_scatter(/*is_scatter*/ false); 680 case vmIntrinsics::_VectorScatterOp: 681 return inline_vector_gather_scatter(/*is_scatter*/ true); 682 case vmIntrinsics::_VectorReductionCoerced: 683 return inline_vector_reduction(); 684 case vmIntrinsics::_VectorTest: 685 return inline_vector_test(); 686 case vmIntrinsics::_VectorBlend: 687 return inline_vector_blend(); 688 case vmIntrinsics::_VectorRearrange: 689 return inline_vector_rearrange(); 690 case vmIntrinsics::_VectorCompare: 691 return inline_vector_compare(); 692 case vmIntrinsics::_VectorBroadcastInt: 693 return inline_vector_broadcast_int(); 694 case vmIntrinsics::_VectorConvert: 695 return inline_vector_convert(); 696 case vmIntrinsics::_VectorInsert: 697 return inline_vector_insert(); 698 case vmIntrinsics::_VectorExtract: 699 return inline_vector_extract(); 700 701 case vmIntrinsics::_getObjectSize: 702 return inline_getObjectSize(); 703 704 case vmIntrinsics::_blackhole: 705 return inline_blackhole(); 706 707 default: 708 // If you get here, it may be that someone has added a new intrinsic 709 // to the list in vmIntrinsics.hpp without implementing it here. 710 #ifndef PRODUCT 711 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) { 712 tty->print_cr("*** Warning: Unimplemented intrinsic %s(%d)", 713 vmIntrinsics::name_at(intrinsic_id()), vmIntrinsics::as_int(intrinsic_id())); 714 } 715 #endif 716 return false; 717 } 718 } 719 720 Node* LibraryCallKit::try_to_predicate(int predicate) { 721 if (!jvms()->has_method()) { 722 // Root JVMState has a null method. 723 assert(map()->memory()->Opcode() == Op_Parm, ""); 724 // Insert the memory aliasing node 725 set_all_memory(reset_memory()); 726 } 727 assert(merged_memory(), ""); 728 729 switch (intrinsic_id()) { 730 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt: 731 return inline_cipherBlockChaining_AESCrypt_predicate(false); 732 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt: 733 return inline_cipherBlockChaining_AESCrypt_predicate(true); 734 case vmIntrinsics::_electronicCodeBook_encryptAESCrypt: 735 return inline_electronicCodeBook_AESCrypt_predicate(false); 736 case vmIntrinsics::_electronicCodeBook_decryptAESCrypt: 737 return inline_electronicCodeBook_AESCrypt_predicate(true); 738 case vmIntrinsics::_counterMode_AESCrypt: 739 return inline_counterMode_AESCrypt_predicate(); 740 case vmIntrinsics::_digestBase_implCompressMB: 741 return inline_digestBase_implCompressMB_predicate(predicate); 742 case vmIntrinsics::_galoisCounterMode_AESCrypt: 743 return inline_galoisCounterMode_AESCrypt_predicate(); 744 745 default: 746 // If you get here, it may be that someone has added a new intrinsic 747 // to the list in vmIntrinsics.hpp without implementing it here. 748 #ifndef PRODUCT 749 if ((PrintMiscellaneous && (Verbose || WizardMode)) || PrintOpto) { 750 tty->print_cr("*** Warning: Unimplemented predicate for intrinsic %s(%d)", 751 vmIntrinsics::name_at(intrinsic_id()), vmIntrinsics::as_int(intrinsic_id())); 752 } 753 #endif 754 Node* slow_ctl = control(); 755 set_control(top()); // No fast path intrinsic 756 return slow_ctl; 757 } 758 } 759 760 //------------------------------set_result------------------------------- 761 // Helper function for finishing intrinsics. 762 void LibraryCallKit::set_result(RegionNode* region, PhiNode* value) { 763 record_for_igvn(region); 764 set_control(_gvn.transform(region)); 765 set_result( _gvn.transform(value)); 766 assert(value->type()->basic_type() == result()->bottom_type()->basic_type(), "sanity"); 767 } 768 769 //------------------------------generate_guard--------------------------- 770 // Helper function for generating guarded fast-slow graph structures. 771 // The given 'test', if true, guards a slow path. If the test fails 772 // then a fast path can be taken. (We generally hope it fails.) 773 // In all cases, GraphKit::control() is updated to the fast path. 774 // The returned value represents the control for the slow path. 775 // The return value is never 'top'; it is either a valid control 776 // or NULL if it is obvious that the slow path can never be taken. 777 // Also, if region and the slow control are not NULL, the slow edge 778 // is appended to the region. 779 Node* LibraryCallKit::generate_guard(Node* test, RegionNode* region, float true_prob) { 780 if (stopped()) { 781 // Already short circuited. 782 return NULL; 783 } 784 785 // Build an if node and its projections. 786 // If test is true we take the slow path, which we assume is uncommon. 787 if (_gvn.type(test) == TypeInt::ZERO) { 788 // The slow branch is never taken. No need to build this guard. 789 return NULL; 790 } 791 792 IfNode* iff = create_and_map_if(control(), test, true_prob, COUNT_UNKNOWN); 793 794 Node* if_slow = _gvn.transform(new IfTrueNode(iff)); 795 if (if_slow == top()) { 796 // The slow branch is never taken. No need to build this guard. 797 return NULL; 798 } 799 800 if (region != NULL) 801 region->add_req(if_slow); 802 803 Node* if_fast = _gvn.transform(new IfFalseNode(iff)); 804 set_control(if_fast); 805 806 return if_slow; 807 } 808 809 inline Node* LibraryCallKit::generate_slow_guard(Node* test, RegionNode* region) { 810 return generate_guard(test, region, PROB_UNLIKELY_MAG(3)); 811 } 812 inline Node* LibraryCallKit::generate_fair_guard(Node* test, RegionNode* region) { 813 return generate_guard(test, region, PROB_FAIR); 814 } 815 816 inline Node* LibraryCallKit::generate_negative_guard(Node* index, RegionNode* region, 817 Node* *pos_index) { 818 if (stopped()) 819 return NULL; // already stopped 820 if (_gvn.type(index)->higher_equal(TypeInt::POS)) // [0,maxint] 821 return NULL; // index is already adequately typed 822 Node* cmp_lt = _gvn.transform(new CmpINode(index, intcon(0))); 823 Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt)); 824 Node* is_neg = generate_guard(bol_lt, region, PROB_MIN); 825 if (is_neg != NULL && pos_index != NULL) { 826 // Emulate effect of Parse::adjust_map_after_if. 827 Node* ccast = new CastIINode(index, TypeInt::POS); 828 ccast->set_req(0, control()); 829 (*pos_index) = _gvn.transform(ccast); 830 } 831 return is_neg; 832 } 833 834 // Make sure that 'position' is a valid limit index, in [0..length]. 835 // There are two equivalent plans for checking this: 836 // A. (offset + copyLength) unsigned<= arrayLength 837 // B. offset <= (arrayLength - copyLength) 838 // We require that all of the values above, except for the sum and 839 // difference, are already known to be non-negative. 840 // Plan A is robust in the face of overflow, if offset and copyLength 841 // are both hugely positive. 842 // 843 // Plan B is less direct and intuitive, but it does not overflow at 844 // all, since the difference of two non-negatives is always 845 // representable. Whenever Java methods must perform the equivalent 846 // check they generally use Plan B instead of Plan A. 847 // For the moment we use Plan A. 848 inline Node* LibraryCallKit::generate_limit_guard(Node* offset, 849 Node* subseq_length, 850 Node* array_length, 851 RegionNode* region) { 852 if (stopped()) 853 return NULL; // already stopped 854 bool zero_offset = _gvn.type(offset) == TypeInt::ZERO; 855 if (zero_offset && subseq_length->eqv_uncast(array_length)) 856 return NULL; // common case of whole-array copy 857 Node* last = subseq_length; 858 if (!zero_offset) // last += offset 859 last = _gvn.transform(new AddINode(last, offset)); 860 Node* cmp_lt = _gvn.transform(new CmpUNode(array_length, last)); 861 Node* bol_lt = _gvn.transform(new BoolNode(cmp_lt, BoolTest::lt)); 862 Node* is_over = generate_guard(bol_lt, region, PROB_MIN); 863 return is_over; 864 } 865 866 // Emit range checks for the given String.value byte array 867 void LibraryCallKit::generate_string_range_check(Node* array, Node* offset, Node* count, bool char_count) { 868 if (stopped()) { 869 return; // already stopped 870 } 871 RegionNode* bailout = new RegionNode(1); 872 record_for_igvn(bailout); 873 if (char_count) { 874 // Convert char count to byte count 875 count = _gvn.transform(new LShiftINode(count, intcon(1))); 876 } 877 878 // Offset and count must not be negative 879 generate_negative_guard(offset, bailout); 880 generate_negative_guard(count, bailout); 881 // Offset + count must not exceed length of array 882 generate_limit_guard(offset, count, load_array_length(array), bailout); 883 884 if (bailout->req() > 1) { 885 PreserveJVMState pjvms(this); 886 set_control(_gvn.transform(bailout)); 887 uncommon_trap(Deoptimization::Reason_intrinsic, 888 Deoptimization::Action_maybe_recompile); 889 } 890 } 891 892 //--------------------------generate_current_thread-------------------- 893 Node* LibraryCallKit::generate_current_thread(Node* &tls_output) { 894 ciKlass* thread_klass = env()->Thread_klass(); 895 const Type* thread_type = TypeOopPtr::make_from_klass(thread_klass)->cast_to_ptr_type(TypePtr::NotNull); 896 Node* thread = _gvn.transform(new ThreadLocalNode()); 897 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::threadObj_offset())); 898 tls_output = thread; 899 Node* thread_obj_handle = LoadNode::make(_gvn, NULL, immutable_memory(), p, p->bottom_type()->is_ptr(), TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered); 900 thread_obj_handle = _gvn.transform(thread_obj_handle); 901 return access_load(thread_obj_handle, thread_type, T_OBJECT, IN_NATIVE | C2_IMMUTABLE_MEMORY); 902 } 903 904 905 //------------------------------make_string_method_node------------------------ 906 // Helper method for String intrinsic functions. This version is called with 907 // str1 and str2 pointing to byte[] nodes containing Latin1 or UTF16 encoded 908 // characters (depending on 'is_byte'). cnt1 and cnt2 are pointing to Int nodes 909 // containing the lengths of str1 and str2. 910 Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1_start, Node* cnt1, Node* str2_start, Node* cnt2, StrIntrinsicNode::ArgEnc ae) { 911 Node* result = NULL; 912 switch (opcode) { 913 case Op_StrIndexOf: 914 result = new StrIndexOfNode(control(), memory(TypeAryPtr::BYTES), 915 str1_start, cnt1, str2_start, cnt2, ae); 916 break; 917 case Op_StrComp: 918 result = new StrCompNode(control(), memory(TypeAryPtr::BYTES), 919 str1_start, cnt1, str2_start, cnt2, ae); 920 break; 921 case Op_StrEquals: 922 // We already know that cnt1 == cnt2 here (checked in 'inline_string_equals'). 923 // Use the constant length if there is one because optimized match rule may exist. 924 result = new StrEqualsNode(control(), memory(TypeAryPtr::BYTES), 925 str1_start, str2_start, cnt2->is_Con() ? cnt2 : cnt1, ae); 926 break; 927 default: 928 ShouldNotReachHere(); 929 return NULL; 930 } 931 932 // All these intrinsics have checks. 933 C->set_has_split_ifs(true); // Has chance for split-if optimization 934 clear_upper_avx(); 935 936 return _gvn.transform(result); 937 } 938 939 //------------------------------inline_string_compareTo------------------------ 940 bool LibraryCallKit::inline_string_compareTo(StrIntrinsicNode::ArgEnc ae) { 941 Node* arg1 = argument(0); 942 Node* arg2 = argument(1); 943 944 arg1 = must_be_not_null(arg1, true); 945 arg2 = must_be_not_null(arg2, true); 946 947 // Get start addr and length of first argument 948 Node* arg1_start = array_element_address(arg1, intcon(0), T_BYTE); 949 Node* arg1_cnt = load_array_length(arg1); 950 951 // Get start addr and length of second argument 952 Node* arg2_start = array_element_address(arg2, intcon(0), T_BYTE); 953 Node* arg2_cnt = load_array_length(arg2); 954 955 Node* result = make_string_method_node(Op_StrComp, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae); 956 set_result(result); 957 return true; 958 } 959 960 //------------------------------inline_string_equals------------------------ 961 bool LibraryCallKit::inline_string_equals(StrIntrinsicNode::ArgEnc ae) { 962 Node* arg1 = argument(0); 963 Node* arg2 = argument(1); 964 965 // paths (plus control) merge 966 RegionNode* region = new RegionNode(3); 967 Node* phi = new PhiNode(region, TypeInt::BOOL); 968 969 if (!stopped()) { 970 971 arg1 = must_be_not_null(arg1, true); 972 arg2 = must_be_not_null(arg2, true); 973 974 // Get start addr and length of first argument 975 Node* arg1_start = array_element_address(arg1, intcon(0), T_BYTE); 976 Node* arg1_cnt = load_array_length(arg1); 977 978 // Get start addr and length of second argument 979 Node* arg2_start = array_element_address(arg2, intcon(0), T_BYTE); 980 Node* arg2_cnt = load_array_length(arg2); 981 982 // Check for arg1_cnt != arg2_cnt 983 Node* cmp = _gvn.transform(new CmpINode(arg1_cnt, arg2_cnt)); 984 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne)); 985 Node* if_ne = generate_slow_guard(bol, NULL); 986 if (if_ne != NULL) { 987 phi->init_req(2, intcon(0)); 988 region->init_req(2, if_ne); 989 } 990 991 // Check for count == 0 is done by assembler code for StrEquals. 992 993 if (!stopped()) { 994 Node* equals = make_string_method_node(Op_StrEquals, arg1_start, arg1_cnt, arg2_start, arg2_cnt, ae); 995 phi->init_req(1, equals); 996 region->init_req(1, control()); 997 } 998 } 999 1000 // post merge 1001 set_control(_gvn.transform(region)); 1002 record_for_igvn(region); 1003 1004 set_result(_gvn.transform(phi)); 1005 return true; 1006 } 1007 1008 //------------------------------inline_array_equals---------------------------- 1009 bool LibraryCallKit::inline_array_equals(StrIntrinsicNode::ArgEnc ae) { 1010 assert(ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::LL, "unsupported array types"); 1011 Node* arg1 = argument(0); 1012 Node* arg2 = argument(1); 1013 1014 const TypeAryPtr* mtype = (ae == StrIntrinsicNode::UU) ? TypeAryPtr::CHARS : TypeAryPtr::BYTES; 1015 set_result(_gvn.transform(new AryEqNode(control(), memory(mtype), arg1, arg2, ae))); 1016 clear_upper_avx(); 1017 1018 return true; 1019 } 1020 1021 //------------------------------inline_countPositives------------------------------ 1022 bool LibraryCallKit::inline_countPositives() { 1023 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 1024 return false; 1025 } 1026 1027 assert(callee()->signature()->size() == 3, "countPositives has 3 parameters"); 1028 // no receiver since it is static method 1029 Node* ba = argument(0); 1030 Node* offset = argument(1); 1031 Node* len = argument(2); 1032 1033 ba = must_be_not_null(ba, true); 1034 1035 // Range checks 1036 generate_string_range_check(ba, offset, len, false); 1037 if (stopped()) { 1038 return true; 1039 } 1040 Node* ba_start = array_element_address(ba, offset, T_BYTE); 1041 Node* result = new CountPositivesNode(control(), memory(TypeAryPtr::BYTES), ba_start, len); 1042 set_result(_gvn.transform(result)); 1043 return true; 1044 } 1045 1046 bool LibraryCallKit::inline_preconditions_checkIndex(BasicType bt) { 1047 Node* index = argument(0); 1048 Node* length = bt == T_INT ? argument(1) : argument(2); 1049 if (too_many_traps(Deoptimization::Reason_intrinsic) || too_many_traps(Deoptimization::Reason_range_check)) { 1050 return false; 1051 } 1052 1053 // check that length is positive 1054 Node* len_pos_cmp = _gvn.transform(CmpNode::make(length, integercon(0, bt), bt)); 1055 Node* len_pos_bol = _gvn.transform(new BoolNode(len_pos_cmp, BoolTest::ge)); 1056 1057 { 1058 BuildCutout unless(this, len_pos_bol, PROB_MAX); 1059 uncommon_trap(Deoptimization::Reason_intrinsic, 1060 Deoptimization::Action_make_not_entrant); 1061 } 1062 1063 if (stopped()) { 1064 // Length is known to be always negative during compilation and the IR graph so far constructed is good so return success 1065 return true; 1066 } 1067 1068 // length is now known positive, add a cast node to make this explicit 1069 jlong upper_bound = _gvn.type(length)->is_integer(bt)->hi_as_long(); 1070 Node* casted_length = ConstraintCastNode::make(control(), length, TypeInteger::make(0, upper_bound, Type::WidenMax, bt), bt); 1071 casted_length = _gvn.transform(casted_length); 1072 replace_in_map(length, casted_length); 1073 length = casted_length; 1074 1075 // Use an unsigned comparison for the range check itself 1076 Node* rc_cmp = _gvn.transform(CmpNode::make(index, length, bt, true)); 1077 BoolTest::mask btest = BoolTest::lt; 1078 Node* rc_bool = _gvn.transform(new BoolNode(rc_cmp, btest)); 1079 RangeCheckNode* rc = new RangeCheckNode(control(), rc_bool, PROB_MAX, COUNT_UNKNOWN); 1080 _gvn.set_type(rc, rc->Value(&_gvn)); 1081 if (!rc_bool->is_Con()) { 1082 record_for_igvn(rc); 1083 } 1084 set_control(_gvn.transform(new IfTrueNode(rc))); 1085 { 1086 PreserveJVMState pjvms(this); 1087 set_control(_gvn.transform(new IfFalseNode(rc))); 1088 uncommon_trap(Deoptimization::Reason_range_check, 1089 Deoptimization::Action_make_not_entrant); 1090 } 1091 1092 if (stopped()) { 1093 // Range check is known to always fail during compilation and the IR graph so far constructed is good so return success 1094 return true; 1095 } 1096 1097 // index is now known to be >= 0 and < length, cast it 1098 Node* result = ConstraintCastNode::make(control(), index, TypeInteger::make(0, upper_bound, Type::WidenMax, bt), bt); 1099 result = _gvn.transform(result); 1100 set_result(result); 1101 replace_in_map(index, result); 1102 return true; 1103 } 1104 1105 //------------------------------inline_string_indexOf------------------------ 1106 bool LibraryCallKit::inline_string_indexOf(StrIntrinsicNode::ArgEnc ae) { 1107 if (!Matcher::match_rule_supported(Op_StrIndexOf)) { 1108 return false; 1109 } 1110 Node* src = argument(0); 1111 Node* tgt = argument(1); 1112 1113 // Make the merge point 1114 RegionNode* result_rgn = new RegionNode(4); 1115 Node* result_phi = new PhiNode(result_rgn, TypeInt::INT); 1116 1117 src = must_be_not_null(src, true); 1118 tgt = must_be_not_null(tgt, true); 1119 1120 // Get start addr and length of source string 1121 Node* src_start = array_element_address(src, intcon(0), T_BYTE); 1122 Node* src_count = load_array_length(src); 1123 1124 // Get start addr and length of substring 1125 Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE); 1126 Node* tgt_count = load_array_length(tgt); 1127 1128 if (ae == StrIntrinsicNode::UU || ae == StrIntrinsicNode::UL) { 1129 // Divide src size by 2 if String is UTF16 encoded 1130 src_count = _gvn.transform(new RShiftINode(src_count, intcon(1))); 1131 } 1132 if (ae == StrIntrinsicNode::UU) { 1133 // Divide substring size by 2 if String is UTF16 encoded 1134 tgt_count = _gvn.transform(new RShiftINode(tgt_count, intcon(1))); 1135 } 1136 1137 Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, result_rgn, result_phi, ae); 1138 if (result != NULL) { 1139 result_phi->init_req(3, result); 1140 result_rgn->init_req(3, control()); 1141 } 1142 set_control(_gvn.transform(result_rgn)); 1143 record_for_igvn(result_rgn); 1144 set_result(_gvn.transform(result_phi)); 1145 1146 return true; 1147 } 1148 1149 //-----------------------------inline_string_indexOf----------------------- 1150 bool LibraryCallKit::inline_string_indexOfI(StrIntrinsicNode::ArgEnc ae) { 1151 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 1152 return false; 1153 } 1154 if (!Matcher::match_rule_supported(Op_StrIndexOf)) { 1155 return false; 1156 } 1157 assert(callee()->signature()->size() == 5, "String.indexOf() has 5 arguments"); 1158 Node* src = argument(0); // byte[] 1159 Node* src_count = argument(1); // char count 1160 Node* tgt = argument(2); // byte[] 1161 Node* tgt_count = argument(3); // char count 1162 Node* from_index = argument(4); // char index 1163 1164 src = must_be_not_null(src, true); 1165 tgt = must_be_not_null(tgt, true); 1166 1167 // Multiply byte array index by 2 if String is UTF16 encoded 1168 Node* src_offset = (ae == StrIntrinsicNode::LL) ? from_index : _gvn.transform(new LShiftINode(from_index, intcon(1))); 1169 src_count = _gvn.transform(new SubINode(src_count, from_index)); 1170 Node* src_start = array_element_address(src, src_offset, T_BYTE); 1171 Node* tgt_start = array_element_address(tgt, intcon(0), T_BYTE); 1172 1173 // Range checks 1174 generate_string_range_check(src, src_offset, src_count, ae != StrIntrinsicNode::LL); 1175 generate_string_range_check(tgt, intcon(0), tgt_count, ae == StrIntrinsicNode::UU); 1176 if (stopped()) { 1177 return true; 1178 } 1179 1180 RegionNode* region = new RegionNode(5); 1181 Node* phi = new PhiNode(region, TypeInt::INT); 1182 1183 Node* result = make_indexOf_node(src_start, src_count, tgt_start, tgt_count, region, phi, ae); 1184 if (result != NULL) { 1185 // The result is index relative to from_index if substring was found, -1 otherwise. 1186 // Generate code which will fold into cmove. 1187 Node* cmp = _gvn.transform(new CmpINode(result, intcon(0))); 1188 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::lt)); 1189 1190 Node* if_lt = generate_slow_guard(bol, NULL); 1191 if (if_lt != NULL) { 1192 // result == -1 1193 phi->init_req(3, result); 1194 region->init_req(3, if_lt); 1195 } 1196 if (!stopped()) { 1197 result = _gvn.transform(new AddINode(result, from_index)); 1198 phi->init_req(4, result); 1199 region->init_req(4, control()); 1200 } 1201 } 1202 1203 set_control(_gvn.transform(region)); 1204 record_for_igvn(region); 1205 set_result(_gvn.transform(phi)); 1206 clear_upper_avx(); 1207 1208 return true; 1209 } 1210 1211 // Create StrIndexOfNode with fast path checks 1212 Node* LibraryCallKit::make_indexOf_node(Node* src_start, Node* src_count, Node* tgt_start, Node* tgt_count, 1213 RegionNode* region, Node* phi, StrIntrinsicNode::ArgEnc ae) { 1214 // Check for substr count > string count 1215 Node* cmp = _gvn.transform(new CmpINode(tgt_count, src_count)); 1216 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::gt)); 1217 Node* if_gt = generate_slow_guard(bol, NULL); 1218 if (if_gt != NULL) { 1219 phi->init_req(1, intcon(-1)); 1220 region->init_req(1, if_gt); 1221 } 1222 if (!stopped()) { 1223 // Check for substr count == 0 1224 cmp = _gvn.transform(new CmpINode(tgt_count, intcon(0))); 1225 bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq)); 1226 Node* if_zero = generate_slow_guard(bol, NULL); 1227 if (if_zero != NULL) { 1228 phi->init_req(2, intcon(0)); 1229 region->init_req(2, if_zero); 1230 } 1231 } 1232 if (!stopped()) { 1233 return make_string_method_node(Op_StrIndexOf, src_start, src_count, tgt_start, tgt_count, ae); 1234 } 1235 return NULL; 1236 } 1237 1238 //-----------------------------inline_string_indexOfChar----------------------- 1239 bool LibraryCallKit::inline_string_indexOfChar(StrIntrinsicNode::ArgEnc ae) { 1240 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 1241 return false; 1242 } 1243 if (!Matcher::match_rule_supported(Op_StrIndexOfChar)) { 1244 return false; 1245 } 1246 assert(callee()->signature()->size() == 4, "String.indexOfChar() has 4 arguments"); 1247 Node* src = argument(0); // byte[] 1248 Node* tgt = argument(1); // tgt is int ch 1249 Node* from_index = argument(2); 1250 Node* max = argument(3); 1251 1252 src = must_be_not_null(src, true); 1253 1254 Node* src_offset = ae == StrIntrinsicNode::L ? from_index : _gvn.transform(new LShiftINode(from_index, intcon(1))); 1255 Node* src_start = array_element_address(src, src_offset, T_BYTE); 1256 Node* src_count = _gvn.transform(new SubINode(max, from_index)); 1257 1258 // Range checks 1259 generate_string_range_check(src, src_offset, src_count, ae == StrIntrinsicNode::U); 1260 if (stopped()) { 1261 return true; 1262 } 1263 1264 RegionNode* region = new RegionNode(3); 1265 Node* phi = new PhiNode(region, TypeInt::INT); 1266 1267 Node* result = new StrIndexOfCharNode(control(), memory(TypeAryPtr::BYTES), src_start, src_count, tgt, ae); 1268 C->set_has_split_ifs(true); // Has chance for split-if optimization 1269 _gvn.transform(result); 1270 1271 Node* cmp = _gvn.transform(new CmpINode(result, intcon(0))); 1272 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::lt)); 1273 1274 Node* if_lt = generate_slow_guard(bol, NULL); 1275 if (if_lt != NULL) { 1276 // result == -1 1277 phi->init_req(2, result); 1278 region->init_req(2, if_lt); 1279 } 1280 if (!stopped()) { 1281 result = _gvn.transform(new AddINode(result, from_index)); 1282 phi->init_req(1, result); 1283 region->init_req(1, control()); 1284 } 1285 set_control(_gvn.transform(region)); 1286 record_for_igvn(region); 1287 set_result(_gvn.transform(phi)); 1288 1289 return true; 1290 } 1291 //---------------------------inline_string_copy--------------------- 1292 // compressIt == true --> generate a compressed copy operation (compress char[]/byte[] to byte[]) 1293 // int StringUTF16.compress(char[] src, int srcOff, byte[] dst, int dstOff, int len) 1294 // int StringUTF16.compress(byte[] src, int srcOff, byte[] dst, int dstOff, int len) 1295 // compressIt == false --> generate an inflated copy operation (inflate byte[] to char[]/byte[]) 1296 // void StringLatin1.inflate(byte[] src, int srcOff, char[] dst, int dstOff, int len) 1297 // void StringLatin1.inflate(byte[] src, int srcOff, byte[] dst, int dstOff, int len) 1298 bool LibraryCallKit::inline_string_copy(bool compress) { 1299 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 1300 return false; 1301 } 1302 int nargs = 5; // 2 oops, 3 ints 1303 assert(callee()->signature()->size() == nargs, "string copy has 5 arguments"); 1304 1305 Node* src = argument(0); 1306 Node* src_offset = argument(1); 1307 Node* dst = argument(2); 1308 Node* dst_offset = argument(3); 1309 Node* length = argument(4); 1310 1311 // Check for allocation before we add nodes that would confuse 1312 // tightly_coupled_allocation() 1313 AllocateArrayNode* alloc = tightly_coupled_allocation(dst); 1314 1315 // Figure out the size and type of the elements we will be copying. 1316 const Type* src_type = src->Value(&_gvn); 1317 const Type* dst_type = dst->Value(&_gvn); 1318 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 1319 BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 1320 assert((compress && dst_elem == T_BYTE && (src_elem == T_BYTE || src_elem == T_CHAR)) || 1321 (!compress && src_elem == T_BYTE && (dst_elem == T_BYTE || dst_elem == T_CHAR)), 1322 "Unsupported array types for inline_string_copy"); 1323 1324 src = must_be_not_null(src, true); 1325 dst = must_be_not_null(dst, true); 1326 1327 // Convert char[] offsets to byte[] offsets 1328 bool convert_src = (compress && src_elem == T_BYTE); 1329 bool convert_dst = (!compress && dst_elem == T_BYTE); 1330 if (convert_src) { 1331 src_offset = _gvn.transform(new LShiftINode(src_offset, intcon(1))); 1332 } else if (convert_dst) { 1333 dst_offset = _gvn.transform(new LShiftINode(dst_offset, intcon(1))); 1334 } 1335 1336 // Range checks 1337 generate_string_range_check(src, src_offset, length, convert_src); 1338 generate_string_range_check(dst, dst_offset, length, convert_dst); 1339 if (stopped()) { 1340 return true; 1341 } 1342 1343 Node* src_start = array_element_address(src, src_offset, src_elem); 1344 Node* dst_start = array_element_address(dst, dst_offset, dst_elem); 1345 // 'src_start' points to src array + scaled offset 1346 // 'dst_start' points to dst array + scaled offset 1347 Node* count = NULL; 1348 if (compress) { 1349 count = compress_string(src_start, TypeAryPtr::get_array_body_type(src_elem), dst_start, length); 1350 } else { 1351 inflate_string(src_start, dst_start, TypeAryPtr::get_array_body_type(dst_elem), length); 1352 } 1353 1354 if (alloc != NULL) { 1355 if (alloc->maybe_set_complete(&_gvn)) { 1356 // "You break it, you buy it." 1357 InitializeNode* init = alloc->initialization(); 1358 assert(init->is_complete(), "we just did this"); 1359 init->set_complete_with_arraycopy(); 1360 assert(dst->is_CheckCastPP(), "sanity"); 1361 assert(dst->in(0)->in(0) == init, "dest pinned"); 1362 } 1363 // Do not let stores that initialize this object be reordered with 1364 // a subsequent store that would make this object accessible by 1365 // other threads. 1366 // Record what AllocateNode this StoreStore protects so that 1367 // escape analysis can go from the MemBarStoreStoreNode to the 1368 // AllocateNode and eliminate the MemBarStoreStoreNode if possible 1369 // based on the escape status of the AllocateNode. 1370 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 1371 } 1372 if (compress) { 1373 set_result(_gvn.transform(count)); 1374 } 1375 clear_upper_avx(); 1376 1377 return true; 1378 } 1379 1380 #ifdef _LP64 1381 #define XTOP ,top() /*additional argument*/ 1382 #else //_LP64 1383 #define XTOP /*no additional argument*/ 1384 #endif //_LP64 1385 1386 //------------------------inline_string_toBytesU-------------------------- 1387 // public static byte[] StringUTF16.toBytes(char[] value, int off, int len) 1388 bool LibraryCallKit::inline_string_toBytesU() { 1389 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 1390 return false; 1391 } 1392 // Get the arguments. 1393 Node* value = argument(0); 1394 Node* offset = argument(1); 1395 Node* length = argument(2); 1396 1397 Node* newcopy = NULL; 1398 1399 // Set the original stack and the reexecute bit for the interpreter to reexecute 1400 // the bytecode that invokes StringUTF16.toBytes() if deoptimization happens. 1401 { PreserveReexecuteState preexecs(this); 1402 jvms()->set_should_reexecute(true); 1403 1404 // Check if a null path was taken unconditionally. 1405 value = null_check(value); 1406 1407 RegionNode* bailout = new RegionNode(1); 1408 record_for_igvn(bailout); 1409 1410 // Range checks 1411 generate_negative_guard(offset, bailout); 1412 generate_negative_guard(length, bailout); 1413 generate_limit_guard(offset, length, load_array_length(value), bailout); 1414 // Make sure that resulting byte[] length does not overflow Integer.MAX_VALUE 1415 generate_limit_guard(length, intcon(0), intcon(max_jint/2), bailout); 1416 1417 if (bailout->req() > 1) { 1418 PreserveJVMState pjvms(this); 1419 set_control(_gvn.transform(bailout)); 1420 uncommon_trap(Deoptimization::Reason_intrinsic, 1421 Deoptimization::Action_maybe_recompile); 1422 } 1423 if (stopped()) { 1424 return true; 1425 } 1426 1427 Node* size = _gvn.transform(new LShiftINode(length, intcon(1))); 1428 Node* klass_node = makecon(TypeKlassPtr::make(ciTypeArrayKlass::make(T_BYTE))); 1429 newcopy = new_array(klass_node, size, 0); // no arguments to push 1430 AllocateArrayNode* alloc = tightly_coupled_allocation(newcopy); 1431 guarantee(alloc != NULL, "created above"); 1432 1433 // Calculate starting addresses. 1434 Node* src_start = array_element_address(value, offset, T_CHAR); 1435 Node* dst_start = basic_plus_adr(newcopy, arrayOopDesc::base_offset_in_bytes(T_BYTE)); 1436 1437 // Check if src array address is aligned to HeapWordSize (dst is always aligned) 1438 const TypeInt* toffset = gvn().type(offset)->is_int(); 1439 bool aligned = toffset->is_con() && ((toffset->get_con() * type2aelembytes(T_CHAR)) % HeapWordSize == 0); 1440 1441 // Figure out which arraycopy runtime method to call (disjoint, uninitialized). 1442 const char* copyfunc_name = "arraycopy"; 1443 address copyfunc_addr = StubRoutines::select_arraycopy_function(T_CHAR, aligned, true, copyfunc_name, true); 1444 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, 1445 OptoRuntime::fast_arraycopy_Type(), 1446 copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM, 1447 src_start, dst_start, ConvI2X(length) XTOP); 1448 // Do not let reads from the cloned object float above the arraycopy. 1449 if (alloc->maybe_set_complete(&_gvn)) { 1450 // "You break it, you buy it." 1451 InitializeNode* init = alloc->initialization(); 1452 assert(init->is_complete(), "we just did this"); 1453 init->set_complete_with_arraycopy(); 1454 assert(newcopy->is_CheckCastPP(), "sanity"); 1455 assert(newcopy->in(0)->in(0) == init, "dest pinned"); 1456 } 1457 // Do not let stores that initialize this object be reordered with 1458 // a subsequent store that would make this object accessible by 1459 // other threads. 1460 // Record what AllocateNode this StoreStore protects so that 1461 // escape analysis can go from the MemBarStoreStoreNode to the 1462 // AllocateNode and eliminate the MemBarStoreStoreNode if possible 1463 // based on the escape status of the AllocateNode. 1464 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 1465 } // original reexecute is set back here 1466 1467 C->set_has_split_ifs(true); // Has chance for split-if optimization 1468 if (!stopped()) { 1469 set_result(newcopy); 1470 } 1471 clear_upper_avx(); 1472 1473 return true; 1474 } 1475 1476 //------------------------inline_string_getCharsU-------------------------- 1477 // public void StringUTF16.getChars(byte[] src, int srcBegin, int srcEnd, char dst[], int dstBegin) 1478 bool LibraryCallKit::inline_string_getCharsU() { 1479 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 1480 return false; 1481 } 1482 1483 // Get the arguments. 1484 Node* src = argument(0); 1485 Node* src_begin = argument(1); 1486 Node* src_end = argument(2); // exclusive offset (i < src_end) 1487 Node* dst = argument(3); 1488 Node* dst_begin = argument(4); 1489 1490 // Check for allocation before we add nodes that would confuse 1491 // tightly_coupled_allocation() 1492 AllocateArrayNode* alloc = tightly_coupled_allocation(dst); 1493 1494 // Check if a null path was taken unconditionally. 1495 src = null_check(src); 1496 dst = null_check(dst); 1497 if (stopped()) { 1498 return true; 1499 } 1500 1501 // Get length and convert char[] offset to byte[] offset 1502 Node* length = _gvn.transform(new SubINode(src_end, src_begin)); 1503 src_begin = _gvn.transform(new LShiftINode(src_begin, intcon(1))); 1504 1505 // Range checks 1506 generate_string_range_check(src, src_begin, length, true); 1507 generate_string_range_check(dst, dst_begin, length, false); 1508 if (stopped()) { 1509 return true; 1510 } 1511 1512 if (!stopped()) { 1513 // Calculate starting addresses. 1514 Node* src_start = array_element_address(src, src_begin, T_BYTE); 1515 Node* dst_start = array_element_address(dst, dst_begin, T_CHAR); 1516 1517 // Check if array addresses are aligned to HeapWordSize 1518 const TypeInt* tsrc = gvn().type(src_begin)->is_int(); 1519 const TypeInt* tdst = gvn().type(dst_begin)->is_int(); 1520 bool aligned = tsrc->is_con() && ((tsrc->get_con() * type2aelembytes(T_BYTE)) % HeapWordSize == 0) && 1521 tdst->is_con() && ((tdst->get_con() * type2aelembytes(T_CHAR)) % HeapWordSize == 0); 1522 1523 // Figure out which arraycopy runtime method to call (disjoint, uninitialized). 1524 const char* copyfunc_name = "arraycopy"; 1525 address copyfunc_addr = StubRoutines::select_arraycopy_function(T_CHAR, aligned, true, copyfunc_name, true); 1526 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, 1527 OptoRuntime::fast_arraycopy_Type(), 1528 copyfunc_addr, copyfunc_name, TypeRawPtr::BOTTOM, 1529 src_start, dst_start, ConvI2X(length) XTOP); 1530 // Do not let reads from the cloned object float above the arraycopy. 1531 if (alloc != NULL) { 1532 if (alloc->maybe_set_complete(&_gvn)) { 1533 // "You break it, you buy it." 1534 InitializeNode* init = alloc->initialization(); 1535 assert(init->is_complete(), "we just did this"); 1536 init->set_complete_with_arraycopy(); 1537 assert(dst->is_CheckCastPP(), "sanity"); 1538 assert(dst->in(0)->in(0) == init, "dest pinned"); 1539 } 1540 // Do not let stores that initialize this object be reordered with 1541 // a subsequent store that would make this object accessible by 1542 // other threads. 1543 // Record what AllocateNode this StoreStore protects so that 1544 // escape analysis can go from the MemBarStoreStoreNode to the 1545 // AllocateNode and eliminate the MemBarStoreStoreNode if possible 1546 // based on the escape status of the AllocateNode. 1547 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 1548 } else { 1549 insert_mem_bar(Op_MemBarCPUOrder); 1550 } 1551 } 1552 1553 C->set_has_split_ifs(true); // Has chance for split-if optimization 1554 return true; 1555 } 1556 1557 //----------------------inline_string_char_access---------------------------- 1558 // Store/Load char to/from byte[] array. 1559 // static void StringUTF16.putChar(byte[] val, int index, int c) 1560 // static char StringUTF16.getChar(byte[] val, int index) 1561 bool LibraryCallKit::inline_string_char_access(bool is_store) { 1562 Node* value = argument(0); 1563 Node* index = argument(1); 1564 Node* ch = is_store ? argument(2) : NULL; 1565 1566 // This intrinsic accesses byte[] array as char[] array. Computing the offsets 1567 // correctly requires matched array shapes. 1568 assert (arrayOopDesc::base_offset_in_bytes(T_CHAR) == arrayOopDesc::base_offset_in_bytes(T_BYTE), 1569 "sanity: byte[] and char[] bases agree"); 1570 assert (type2aelembytes(T_CHAR) == type2aelembytes(T_BYTE)*2, 1571 "sanity: byte[] and char[] scales agree"); 1572 1573 // Bail when getChar over constants is requested: constant folding would 1574 // reject folding mismatched char access over byte[]. A normal inlining for getChar 1575 // Java method would constant fold nicely instead. 1576 if (!is_store && value->is_Con() && index->is_Con()) { 1577 return false; 1578 } 1579 1580 value = must_be_not_null(value, true); 1581 1582 Node* adr = array_element_address(value, index, T_CHAR); 1583 if (adr->is_top()) { 1584 return false; 1585 } 1586 if (is_store) { 1587 access_store_at(value, adr, TypeAryPtr::BYTES, ch, TypeInt::CHAR, T_CHAR, IN_HEAP | MO_UNORDERED | C2_MISMATCHED); 1588 } else { 1589 ch = access_load_at(value, adr, TypeAryPtr::BYTES, TypeInt::CHAR, T_CHAR, IN_HEAP | MO_UNORDERED | C2_MISMATCHED | C2_CONTROL_DEPENDENT_LOAD | C2_UNKNOWN_CONTROL_LOAD); 1590 set_result(ch); 1591 } 1592 return true; 1593 } 1594 1595 //--------------------------round_double_node-------------------------------- 1596 // Round a double node if necessary. 1597 Node* LibraryCallKit::round_double_node(Node* n) { 1598 if (Matcher::strict_fp_requires_explicit_rounding) { 1599 #ifdef IA32 1600 if (UseSSE < 2) { 1601 n = _gvn.transform(new RoundDoubleNode(NULL, n)); 1602 } 1603 #else 1604 Unimplemented(); 1605 #endif // IA32 1606 } 1607 return n; 1608 } 1609 1610 //------------------------------inline_math----------------------------------- 1611 // public static double Math.abs(double) 1612 // public static double Math.sqrt(double) 1613 // public static double Math.log(double) 1614 // public static double Math.log10(double) 1615 // public static double Math.round(double) 1616 bool LibraryCallKit::inline_double_math(vmIntrinsics::ID id) { 1617 Node* arg = round_double_node(argument(0)); 1618 Node* n = NULL; 1619 switch (id) { 1620 case vmIntrinsics::_dabs: n = new AbsDNode( arg); break; 1621 case vmIntrinsics::_dsqrt: 1622 case vmIntrinsics::_dsqrt_strict: 1623 n = new SqrtDNode(C, control(), arg); break; 1624 case vmIntrinsics::_ceil: n = RoundDoubleModeNode::make(_gvn, arg, RoundDoubleModeNode::rmode_ceil); break; 1625 case vmIntrinsics::_floor: n = RoundDoubleModeNode::make(_gvn, arg, RoundDoubleModeNode::rmode_floor); break; 1626 case vmIntrinsics::_rint: n = RoundDoubleModeNode::make(_gvn, arg, RoundDoubleModeNode::rmode_rint); break; 1627 case vmIntrinsics::_roundD: n = new RoundDNode(arg); break; 1628 case vmIntrinsics::_dcopySign: n = CopySignDNode::make(_gvn, arg, round_double_node(argument(2))); break; 1629 case vmIntrinsics::_dsignum: n = SignumDNode::make(_gvn, arg); break; 1630 default: fatal_unexpected_iid(id); break; 1631 } 1632 set_result(_gvn.transform(n)); 1633 return true; 1634 } 1635 1636 //------------------------------inline_math----------------------------------- 1637 // public static float Math.abs(float) 1638 // public static int Math.abs(int) 1639 // public static long Math.abs(long) 1640 bool LibraryCallKit::inline_math(vmIntrinsics::ID id) { 1641 Node* arg = argument(0); 1642 Node* n = NULL; 1643 switch (id) { 1644 case vmIntrinsics::_fabs: n = new AbsFNode( arg); break; 1645 case vmIntrinsics::_iabs: n = new AbsINode( arg); break; 1646 case vmIntrinsics::_labs: n = new AbsLNode( arg); break; 1647 case vmIntrinsics::_fcopySign: n = new CopySignFNode(arg, argument(1)); break; 1648 case vmIntrinsics::_fsignum: n = SignumFNode::make(_gvn, arg); break; 1649 case vmIntrinsics::_roundF: n = new RoundFNode(arg); break; 1650 default: fatal_unexpected_iid(id); break; 1651 } 1652 set_result(_gvn.transform(n)); 1653 return true; 1654 } 1655 1656 //------------------------------runtime_math----------------------------- 1657 bool LibraryCallKit::runtime_math(const TypeFunc* call_type, address funcAddr, const char* funcName) { 1658 assert(call_type == OptoRuntime::Math_DD_D_Type() || call_type == OptoRuntime::Math_D_D_Type(), 1659 "must be (DD)D or (D)D type"); 1660 1661 // Inputs 1662 Node* a = round_double_node(argument(0)); 1663 Node* b = (call_type == OptoRuntime::Math_DD_D_Type()) ? round_double_node(argument(2)) : NULL; 1664 1665 const TypePtr* no_memory_effects = NULL; 1666 Node* trig = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName, 1667 no_memory_effects, 1668 a, top(), b, b ? top() : NULL); 1669 Node* value = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+0)); 1670 #ifdef ASSERT 1671 Node* value_top = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+1)); 1672 assert(value_top == top(), "second value must be top"); 1673 #endif 1674 1675 set_result(value); 1676 return true; 1677 } 1678 1679 //------------------------------inline_math_pow----------------------------- 1680 bool LibraryCallKit::inline_math_pow() { 1681 Node* exp = round_double_node(argument(2)); 1682 const TypeD* d = _gvn.type(exp)->isa_double_constant(); 1683 if (d != NULL) { 1684 if (d->getd() == 2.0) { 1685 // Special case: pow(x, 2.0) => x * x 1686 Node* base = round_double_node(argument(0)); 1687 set_result(_gvn.transform(new MulDNode(base, base))); 1688 return true; 1689 } else if (d->getd() == 0.5 && Matcher::match_rule_supported(Op_SqrtD)) { 1690 // Special case: pow(x, 0.5) => sqrt(x) 1691 Node* base = round_double_node(argument(0)); 1692 Node* zero = _gvn.zerocon(T_DOUBLE); 1693 1694 RegionNode* region = new RegionNode(3); 1695 Node* phi = new PhiNode(region, Type::DOUBLE); 1696 1697 Node* cmp = _gvn.transform(new CmpDNode(base, zero)); 1698 // According to the API specs, pow(-0.0, 0.5) = 0.0 and sqrt(-0.0) = -0.0. 1699 // So pow(-0.0, 0.5) shouldn't be replaced with sqrt(-0.0). 1700 // -0.0/+0.0 are both excluded since floating-point comparison doesn't distinguish -0.0 from +0.0. 1701 Node* test = _gvn.transform(new BoolNode(cmp, BoolTest::le)); 1702 1703 Node* if_pow = generate_slow_guard(test, NULL); 1704 Node* value_sqrt = _gvn.transform(new SqrtDNode(C, control(), base)); 1705 phi->init_req(1, value_sqrt); 1706 region->init_req(1, control()); 1707 1708 if (if_pow != NULL) { 1709 set_control(if_pow); 1710 address target = StubRoutines::dpow() != NULL ? StubRoutines::dpow() : 1711 CAST_FROM_FN_PTR(address, SharedRuntime::dpow); 1712 const TypePtr* no_memory_effects = NULL; 1713 Node* trig = make_runtime_call(RC_LEAF, OptoRuntime::Math_DD_D_Type(), target, "POW", 1714 no_memory_effects, base, top(), exp, top()); 1715 Node* value_pow = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+0)); 1716 #ifdef ASSERT 1717 Node* value_top = _gvn.transform(new ProjNode(trig, TypeFunc::Parms+1)); 1718 assert(value_top == top(), "second value must be top"); 1719 #endif 1720 phi->init_req(2, value_pow); 1721 region->init_req(2, _gvn.transform(new ProjNode(trig, TypeFunc::Control))); 1722 } 1723 1724 C->set_has_split_ifs(true); // Has chance for split-if optimization 1725 set_control(_gvn.transform(region)); 1726 record_for_igvn(region); 1727 set_result(_gvn.transform(phi)); 1728 1729 return true; 1730 } 1731 } 1732 1733 return StubRoutines::dpow() != NULL ? 1734 runtime_math(OptoRuntime::Math_DD_D_Type(), StubRoutines::dpow(), "dpow") : 1735 runtime_math(OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW"); 1736 } 1737 1738 //------------------------------inline_math_native----------------------------- 1739 bool LibraryCallKit::inline_math_native(vmIntrinsics::ID id) { 1740 #define FN_PTR(f) CAST_FROM_FN_PTR(address, f) 1741 switch (id) { 1742 // These intrinsics are not properly supported on all hardware 1743 case vmIntrinsics::_dsin: 1744 return StubRoutines::dsin() != NULL ? 1745 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dsin(), "dsin") : 1746 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dsin), "SIN"); 1747 case vmIntrinsics::_dcos: 1748 return StubRoutines::dcos() != NULL ? 1749 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dcos(), "dcos") : 1750 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dcos), "COS"); 1751 case vmIntrinsics::_dtan: 1752 return StubRoutines::dtan() != NULL ? 1753 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dtan(), "dtan") : 1754 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dtan), "TAN"); 1755 case vmIntrinsics::_dlog: 1756 return StubRoutines::dlog() != NULL ? 1757 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog(), "dlog") : 1758 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog), "LOG"); 1759 case vmIntrinsics::_dlog10: 1760 return StubRoutines::dlog10() != NULL ? 1761 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dlog10(), "dlog10") : 1762 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dlog10), "LOG10"); 1763 1764 // These intrinsics are supported on all hardware 1765 case vmIntrinsics::_roundD: return Matcher::match_rule_supported(Op_RoundD) ? inline_double_math(id) : false; 1766 case vmIntrinsics::_ceil: 1767 case vmIntrinsics::_floor: 1768 case vmIntrinsics::_rint: return Matcher::match_rule_supported(Op_RoundDoubleMode) ? inline_double_math(id) : false; 1769 1770 case vmIntrinsics::_dsqrt: 1771 case vmIntrinsics::_dsqrt_strict: 1772 return Matcher::match_rule_supported(Op_SqrtD) ? inline_double_math(id) : false; 1773 case vmIntrinsics::_dabs: return Matcher::has_match_rule(Op_AbsD) ? inline_double_math(id) : false; 1774 case vmIntrinsics::_fabs: return Matcher::match_rule_supported(Op_AbsF) ? inline_math(id) : false; 1775 case vmIntrinsics::_iabs: return Matcher::match_rule_supported(Op_AbsI) ? inline_math(id) : false; 1776 case vmIntrinsics::_labs: return Matcher::match_rule_supported(Op_AbsL) ? inline_math(id) : false; 1777 1778 case vmIntrinsics::_dexp: 1779 return StubRoutines::dexp() != NULL ? 1780 runtime_math(OptoRuntime::Math_D_D_Type(), StubRoutines::dexp(), "dexp") : 1781 runtime_math(OptoRuntime::Math_D_D_Type(), FN_PTR(SharedRuntime::dexp), "EXP"); 1782 #undef FN_PTR 1783 1784 case vmIntrinsics::_dpow: return inline_math_pow(); 1785 case vmIntrinsics::_dcopySign: return inline_double_math(id); 1786 case vmIntrinsics::_fcopySign: return inline_math(id); 1787 case vmIntrinsics::_dsignum: return Matcher::match_rule_supported(Op_SignumD) ? inline_double_math(id) : false; 1788 case vmIntrinsics::_fsignum: return Matcher::match_rule_supported(Op_SignumF) ? inline_math(id) : false; 1789 case vmIntrinsics::_roundF: return Matcher::match_rule_supported(Op_RoundF) ? inline_math(id) : false; 1790 1791 // These intrinsics are not yet correctly implemented 1792 case vmIntrinsics::_datan2: 1793 return false; 1794 1795 default: 1796 fatal_unexpected_iid(id); 1797 return false; 1798 } 1799 } 1800 1801 static bool is_simple_name(Node* n) { 1802 return (n->req() == 1 // constant 1803 || (n->is_Type() && n->as_Type()->type()->singleton()) 1804 || n->is_Proj() // parameter or return value 1805 || n->is_Phi() // local of some sort 1806 ); 1807 } 1808 1809 //----------------------------inline_notify-----------------------------------* 1810 bool LibraryCallKit::inline_notify(vmIntrinsics::ID id) { 1811 const TypeFunc* ftype = OptoRuntime::monitor_notify_Type(); 1812 address func; 1813 if (id == vmIntrinsics::_notify) { 1814 func = OptoRuntime::monitor_notify_Java(); 1815 } else { 1816 func = OptoRuntime::monitor_notifyAll_Java(); 1817 } 1818 Node* call = make_runtime_call(RC_NO_LEAF, ftype, func, NULL, TypeRawPtr::BOTTOM, argument(0)); 1819 make_slow_call_ex(call, env()->Throwable_klass(), false); 1820 return true; 1821 } 1822 1823 1824 //----------------------------inline_min_max----------------------------------- 1825 bool LibraryCallKit::inline_min_max(vmIntrinsics::ID id) { 1826 set_result(generate_min_max(id, argument(0), argument(1))); 1827 return true; 1828 } 1829 1830 void LibraryCallKit::inline_math_mathExact(Node* math, Node *test) { 1831 Node* bol = _gvn.transform( new BoolNode(test, BoolTest::overflow) ); 1832 IfNode* check = create_and_map_if(control(), bol, PROB_UNLIKELY_MAG(3), COUNT_UNKNOWN); 1833 Node* fast_path = _gvn.transform( new IfFalseNode(check)); 1834 Node* slow_path = _gvn.transform( new IfTrueNode(check) ); 1835 1836 { 1837 PreserveJVMState pjvms(this); 1838 PreserveReexecuteState preexecs(this); 1839 jvms()->set_should_reexecute(true); 1840 1841 set_control(slow_path); 1842 set_i_o(i_o()); 1843 1844 uncommon_trap(Deoptimization::Reason_intrinsic, 1845 Deoptimization::Action_none); 1846 } 1847 1848 set_control(fast_path); 1849 set_result(math); 1850 } 1851 1852 template <typename OverflowOp> 1853 bool LibraryCallKit::inline_math_overflow(Node* arg1, Node* arg2) { 1854 typedef typename OverflowOp::MathOp MathOp; 1855 1856 MathOp* mathOp = new MathOp(arg1, arg2); 1857 Node* operation = _gvn.transform( mathOp ); 1858 Node* ofcheck = _gvn.transform( new OverflowOp(arg1, arg2) ); 1859 inline_math_mathExact(operation, ofcheck); 1860 return true; 1861 } 1862 1863 bool LibraryCallKit::inline_math_addExactI(bool is_increment) { 1864 return inline_math_overflow<OverflowAddINode>(argument(0), is_increment ? intcon(1) : argument(1)); 1865 } 1866 1867 bool LibraryCallKit::inline_math_addExactL(bool is_increment) { 1868 return inline_math_overflow<OverflowAddLNode>(argument(0), is_increment ? longcon(1) : argument(2)); 1869 } 1870 1871 bool LibraryCallKit::inline_math_subtractExactI(bool is_decrement) { 1872 return inline_math_overflow<OverflowSubINode>(argument(0), is_decrement ? intcon(1) : argument(1)); 1873 } 1874 1875 bool LibraryCallKit::inline_math_subtractExactL(bool is_decrement) { 1876 return inline_math_overflow<OverflowSubLNode>(argument(0), is_decrement ? longcon(1) : argument(2)); 1877 } 1878 1879 bool LibraryCallKit::inline_math_negateExactI() { 1880 return inline_math_overflow<OverflowSubINode>(intcon(0), argument(0)); 1881 } 1882 1883 bool LibraryCallKit::inline_math_negateExactL() { 1884 return inline_math_overflow<OverflowSubLNode>(longcon(0), argument(0)); 1885 } 1886 1887 bool LibraryCallKit::inline_math_multiplyExactI() { 1888 return inline_math_overflow<OverflowMulINode>(argument(0), argument(1)); 1889 } 1890 1891 bool LibraryCallKit::inline_math_multiplyExactL() { 1892 return inline_math_overflow<OverflowMulLNode>(argument(0), argument(2)); 1893 } 1894 1895 bool LibraryCallKit::inline_math_multiplyHigh() { 1896 set_result(_gvn.transform(new MulHiLNode(argument(0), argument(2)))); 1897 return true; 1898 } 1899 1900 bool LibraryCallKit::inline_math_unsignedMultiplyHigh() { 1901 set_result(_gvn.transform(new UMulHiLNode(argument(0), argument(2)))); 1902 return true; 1903 } 1904 1905 Node* 1906 LibraryCallKit::generate_min_max(vmIntrinsics::ID id, Node* x0, Node* y0) { 1907 // These are the candidate return value: 1908 Node* xvalue = x0; 1909 Node* yvalue = y0; 1910 1911 if (xvalue == yvalue) { 1912 return xvalue; 1913 } 1914 1915 bool want_max = (id == vmIntrinsics::_max || id == vmIntrinsics::_max_strict); 1916 1917 const TypeInt* txvalue = _gvn.type(xvalue)->isa_int(); 1918 const TypeInt* tyvalue = _gvn.type(yvalue)->isa_int(); 1919 if (txvalue == NULL || tyvalue == NULL) return top(); 1920 // This is not really necessary, but it is consistent with a 1921 // hypothetical MaxINode::Value method: 1922 int widen = MAX2(txvalue->_widen, tyvalue->_widen); 1923 1924 // %%% This folding logic should (ideally) be in a different place. 1925 // Some should be inside IfNode, and there to be a more reliable 1926 // transformation of ?: style patterns into cmoves. We also want 1927 // more powerful optimizations around cmove and min/max. 1928 1929 // Try to find a dominating comparison of these guys. 1930 // It can simplify the index computation for Arrays.copyOf 1931 // and similar uses of System.arraycopy. 1932 // First, compute the normalized version of CmpI(x, y). 1933 int cmp_op = Op_CmpI; 1934 Node* xkey = xvalue; 1935 Node* ykey = yvalue; 1936 Node* ideal_cmpxy = _gvn.transform(new CmpINode(xkey, ykey)); 1937 if (ideal_cmpxy->is_Cmp()) { 1938 // E.g., if we have CmpI(length - offset, count), 1939 // it might idealize to CmpI(length, count + offset) 1940 cmp_op = ideal_cmpxy->Opcode(); 1941 xkey = ideal_cmpxy->in(1); 1942 ykey = ideal_cmpxy->in(2); 1943 } 1944 1945 // Start by locating any relevant comparisons. 1946 Node* start_from = (xkey->outcnt() < ykey->outcnt()) ? xkey : ykey; 1947 Node* cmpxy = NULL; 1948 Node* cmpyx = NULL; 1949 for (DUIterator_Fast kmax, k = start_from->fast_outs(kmax); k < kmax; k++) { 1950 Node* cmp = start_from->fast_out(k); 1951 if (cmp->outcnt() > 0 && // must have prior uses 1952 cmp->in(0) == NULL && // must be context-independent 1953 cmp->Opcode() == cmp_op) { // right kind of compare 1954 if (cmp->in(1) == xkey && cmp->in(2) == ykey) cmpxy = cmp; 1955 if (cmp->in(1) == ykey && cmp->in(2) == xkey) cmpyx = cmp; 1956 } 1957 } 1958 1959 const int NCMPS = 2; 1960 Node* cmps[NCMPS] = { cmpxy, cmpyx }; 1961 int cmpn; 1962 for (cmpn = 0; cmpn < NCMPS; cmpn++) { 1963 if (cmps[cmpn] != NULL) break; // find a result 1964 } 1965 if (cmpn < NCMPS) { 1966 // Look for a dominating test that tells us the min and max. 1967 int depth = 0; // Limit search depth for speed 1968 Node* dom = control(); 1969 for (; dom != NULL; dom = IfNode::up_one_dom(dom, true)) { 1970 if (++depth >= 100) break; 1971 Node* ifproj = dom; 1972 if (!ifproj->is_Proj()) continue; 1973 Node* iff = ifproj->in(0); 1974 if (!iff->is_If()) continue; 1975 Node* bol = iff->in(1); 1976 if (!bol->is_Bool()) continue; 1977 Node* cmp = bol->in(1); 1978 if (cmp == NULL) continue; 1979 for (cmpn = 0; cmpn < NCMPS; cmpn++) 1980 if (cmps[cmpn] == cmp) break; 1981 if (cmpn == NCMPS) continue; 1982 BoolTest::mask btest = bol->as_Bool()->_test._test; 1983 if (ifproj->is_IfFalse()) btest = BoolTest(btest).negate(); 1984 if (cmp->in(1) == ykey) btest = BoolTest(btest).commute(); 1985 // At this point, we know that 'x btest y' is true. 1986 switch (btest) { 1987 case BoolTest::eq: 1988 // They are proven equal, so we can collapse the min/max. 1989 // Either value is the answer. Choose the simpler. 1990 if (is_simple_name(yvalue) && !is_simple_name(xvalue)) 1991 return yvalue; 1992 return xvalue; 1993 case BoolTest::lt: // x < y 1994 case BoolTest::le: // x <= y 1995 return (want_max ? yvalue : xvalue); 1996 case BoolTest::gt: // x > y 1997 case BoolTest::ge: // x >= y 1998 return (want_max ? xvalue : yvalue); 1999 default: 2000 break; 2001 } 2002 } 2003 } 2004 2005 // We failed to find a dominating test. 2006 // Let's pick a test that might GVN with prior tests. 2007 Node* best_bol = NULL; 2008 BoolTest::mask best_btest = BoolTest::illegal; 2009 for (cmpn = 0; cmpn < NCMPS; cmpn++) { 2010 Node* cmp = cmps[cmpn]; 2011 if (cmp == NULL) continue; 2012 for (DUIterator_Fast jmax, j = cmp->fast_outs(jmax); j < jmax; j++) { 2013 Node* bol = cmp->fast_out(j); 2014 if (!bol->is_Bool()) continue; 2015 BoolTest::mask btest = bol->as_Bool()->_test._test; 2016 if (btest == BoolTest::eq || btest == BoolTest::ne) continue; 2017 if (cmp->in(1) == ykey) btest = BoolTest(btest).commute(); 2018 if (bol->outcnt() > (best_bol == NULL ? 0 : best_bol->outcnt())) { 2019 best_bol = bol->as_Bool(); 2020 best_btest = btest; 2021 } 2022 } 2023 } 2024 2025 Node* answer_if_true = NULL; 2026 Node* answer_if_false = NULL; 2027 switch (best_btest) { 2028 default: 2029 if (cmpxy == NULL) 2030 cmpxy = ideal_cmpxy; 2031 best_bol = _gvn.transform(new BoolNode(cmpxy, BoolTest::lt)); 2032 // and fall through: 2033 case BoolTest::lt: // x < y 2034 case BoolTest::le: // x <= y 2035 answer_if_true = (want_max ? yvalue : xvalue); 2036 answer_if_false = (want_max ? xvalue : yvalue); 2037 break; 2038 case BoolTest::gt: // x > y 2039 case BoolTest::ge: // x >= y 2040 answer_if_true = (want_max ? xvalue : yvalue); 2041 answer_if_false = (want_max ? yvalue : xvalue); 2042 break; 2043 } 2044 2045 jint hi, lo; 2046 if (want_max) { 2047 // We can sharpen the minimum. 2048 hi = MAX2(txvalue->_hi, tyvalue->_hi); 2049 lo = MAX2(txvalue->_lo, tyvalue->_lo); 2050 } else { 2051 // We can sharpen the maximum. 2052 hi = MIN2(txvalue->_hi, tyvalue->_hi); 2053 lo = MIN2(txvalue->_lo, tyvalue->_lo); 2054 } 2055 2056 // Use a flow-free graph structure, to avoid creating excess control edges 2057 // which could hinder other optimizations. 2058 // Since Math.min/max is often used with arraycopy, we want 2059 // tightly_coupled_allocation to be able to see beyond min/max expressions. 2060 Node* cmov = CMoveNode::make(NULL, best_bol, 2061 answer_if_false, answer_if_true, 2062 TypeInt::make(lo, hi, widen)); 2063 2064 return _gvn.transform(cmov); 2065 2066 /* 2067 // This is not as desirable as it may seem, since Min and Max 2068 // nodes do not have a full set of optimizations. 2069 // And they would interfere, anyway, with 'if' optimizations 2070 // and with CMoveI canonical forms. 2071 switch (id) { 2072 case vmIntrinsics::_min: 2073 result_val = _gvn.transform(new (C, 3) MinINode(x,y)); break; 2074 case vmIntrinsics::_max: 2075 result_val = _gvn.transform(new (C, 3) MaxINode(x,y)); break; 2076 default: 2077 ShouldNotReachHere(); 2078 } 2079 */ 2080 } 2081 2082 inline int 2083 LibraryCallKit::classify_unsafe_addr(Node* &base, Node* &offset, BasicType type) { 2084 const TypePtr* base_type = TypePtr::NULL_PTR; 2085 if (base != NULL) base_type = _gvn.type(base)->isa_ptr(); 2086 if (base_type == NULL) { 2087 // Unknown type. 2088 return Type::AnyPtr; 2089 } else if (base_type == TypePtr::NULL_PTR) { 2090 // Since this is a NULL+long form, we have to switch to a rawptr. 2091 base = _gvn.transform(new CastX2PNode(offset)); 2092 offset = MakeConX(0); 2093 return Type::RawPtr; 2094 } else if (base_type->base() == Type::RawPtr) { 2095 return Type::RawPtr; 2096 } else if (base_type->isa_oopptr()) { 2097 // Base is never null => always a heap address. 2098 if (!TypePtr::NULL_PTR->higher_equal(base_type)) { 2099 return Type::OopPtr; 2100 } 2101 // Offset is small => always a heap address. 2102 const TypeX* offset_type = _gvn.type(offset)->isa_intptr_t(); 2103 if (offset_type != NULL && 2104 base_type->offset() == 0 && // (should always be?) 2105 offset_type->_lo >= 0 && 2106 !MacroAssembler::needs_explicit_null_check(offset_type->_hi)) { 2107 return Type::OopPtr; 2108 } else if (type == T_OBJECT) { 2109 // off heap access to an oop doesn't make any sense. Has to be on 2110 // heap. 2111 return Type::OopPtr; 2112 } 2113 // Otherwise, it might either be oop+off or NULL+addr. 2114 return Type::AnyPtr; 2115 } else { 2116 // No information: 2117 return Type::AnyPtr; 2118 } 2119 } 2120 2121 Node* LibraryCallKit::make_unsafe_address(Node*& base, Node* offset, BasicType type, bool can_cast) { 2122 Node* uncasted_base = base; 2123 int kind = classify_unsafe_addr(uncasted_base, offset, type); 2124 if (kind == Type::RawPtr) { 2125 return basic_plus_adr(top(), uncasted_base, offset); 2126 } else if (kind == Type::AnyPtr) { 2127 assert(base == uncasted_base, "unexpected base change"); 2128 if (can_cast) { 2129 if (!_gvn.type(base)->speculative_maybe_null() && 2130 !too_many_traps(Deoptimization::Reason_speculate_null_check)) { 2131 // According to profiling, this access is always on 2132 // heap. Casting the base to not null and thus avoiding membars 2133 // around the access should allow better optimizations 2134 Node* null_ctl = top(); 2135 base = null_check_oop(base, &null_ctl, true, true, true); 2136 assert(null_ctl->is_top(), "no null control here"); 2137 return basic_plus_adr(base, offset); 2138 } else if (_gvn.type(base)->speculative_always_null() && 2139 !too_many_traps(Deoptimization::Reason_speculate_null_assert)) { 2140 // According to profiling, this access is always off 2141 // heap. 2142 base = null_assert(base); 2143 Node* raw_base = _gvn.transform(new CastX2PNode(offset)); 2144 offset = MakeConX(0); 2145 return basic_plus_adr(top(), raw_base, offset); 2146 } 2147 } 2148 // We don't know if it's an on heap or off heap access. Fall back 2149 // to raw memory access. 2150 Node* raw = _gvn.transform(new CheckCastPPNode(control(), base, TypeRawPtr::BOTTOM)); 2151 return basic_plus_adr(top(), raw, offset); 2152 } else { 2153 assert(base == uncasted_base, "unexpected base change"); 2154 // We know it's an on heap access so base can't be null 2155 if (TypePtr::NULL_PTR->higher_equal(_gvn.type(base))) { 2156 base = must_be_not_null(base, true); 2157 } 2158 return basic_plus_adr(base, offset); 2159 } 2160 } 2161 2162 //--------------------------inline_number_methods----------------------------- 2163 // inline int Integer.numberOfLeadingZeros(int) 2164 // inline int Long.numberOfLeadingZeros(long) 2165 // 2166 // inline int Integer.numberOfTrailingZeros(int) 2167 // inline int Long.numberOfTrailingZeros(long) 2168 // 2169 // inline int Integer.bitCount(int) 2170 // inline int Long.bitCount(long) 2171 // 2172 // inline char Character.reverseBytes(char) 2173 // inline short Short.reverseBytes(short) 2174 // inline int Integer.reverseBytes(int) 2175 // inline long Long.reverseBytes(long) 2176 bool LibraryCallKit::inline_number_methods(vmIntrinsics::ID id) { 2177 Node* arg = argument(0); 2178 Node* n = NULL; 2179 switch (id) { 2180 case vmIntrinsics::_numberOfLeadingZeros_i: n = new CountLeadingZerosINode( arg); break; 2181 case vmIntrinsics::_numberOfLeadingZeros_l: n = new CountLeadingZerosLNode( arg); break; 2182 case vmIntrinsics::_numberOfTrailingZeros_i: n = new CountTrailingZerosINode(arg); break; 2183 case vmIntrinsics::_numberOfTrailingZeros_l: n = new CountTrailingZerosLNode(arg); break; 2184 case vmIntrinsics::_bitCount_i: n = new PopCountINode( arg); break; 2185 case vmIntrinsics::_bitCount_l: n = new PopCountLNode( arg); break; 2186 case vmIntrinsics::_reverseBytes_c: n = new ReverseBytesUSNode(0, arg); break; 2187 case vmIntrinsics::_reverseBytes_s: n = new ReverseBytesSNode( 0, arg); break; 2188 case vmIntrinsics::_reverseBytes_i: n = new ReverseBytesINode( 0, arg); break; 2189 case vmIntrinsics::_reverseBytes_l: n = new ReverseBytesLNode( 0, arg); break; 2190 default: fatal_unexpected_iid(id); break; 2191 } 2192 set_result(_gvn.transform(n)); 2193 return true; 2194 } 2195 2196 //--------------------------inline_unsigned_divmod_methods----------------------------- 2197 // inline int Integer.divideUnsigned(int, int) 2198 // inline int Integer.remainderUnsigned(int, int) 2199 // inline long Long.divideUnsigned(long, long) 2200 // inline long Long.remainderUnsigned(long, long) 2201 bool LibraryCallKit::inline_divmod_methods(vmIntrinsics::ID id) { 2202 Node* n = NULL; 2203 switch (id) { 2204 case vmIntrinsics::_divideUnsigned_i: { 2205 zero_check_int(argument(1)); 2206 // Compile-time detect of null-exception 2207 if (stopped()) { 2208 return true; // keep the graph constructed so far 2209 } 2210 n = new UDivINode(control(), argument(0), argument(1)); 2211 break; 2212 } 2213 case vmIntrinsics::_divideUnsigned_l: { 2214 zero_check_long(argument(2)); 2215 // Compile-time detect of null-exception 2216 if (stopped()) { 2217 return true; // keep the graph constructed so far 2218 } 2219 n = new UDivLNode(control(), argument(0), argument(2)); 2220 break; 2221 } 2222 case vmIntrinsics::_remainderUnsigned_i: { 2223 zero_check_int(argument(1)); 2224 // Compile-time detect of null-exception 2225 if (stopped()) { 2226 return true; // keep the graph constructed so far 2227 } 2228 n = new UModINode(control(), argument(0), argument(1)); 2229 break; 2230 } 2231 case vmIntrinsics::_remainderUnsigned_l: { 2232 zero_check_long(argument(2)); 2233 // Compile-time detect of null-exception 2234 if (stopped()) { 2235 return true; // keep the graph constructed so far 2236 } 2237 n = new UModLNode(control(), argument(0), argument(2)); 2238 break; 2239 } 2240 default: fatal_unexpected_iid(id); break; 2241 } 2242 set_result(_gvn.transform(n)); 2243 return true; 2244 } 2245 2246 //----------------------------inline_unsafe_access---------------------------- 2247 2248 const TypeOopPtr* LibraryCallKit::sharpen_unsafe_type(Compile::AliasType* alias_type, const TypePtr *adr_type) { 2249 // Attempt to infer a sharper value type from the offset and base type. 2250 ciKlass* sharpened_klass = NULL; 2251 2252 // See if it is an instance field, with an object type. 2253 if (alias_type->field() != NULL) { 2254 if (alias_type->field()->type()->is_klass()) { 2255 sharpened_klass = alias_type->field()->type()->as_klass(); 2256 } 2257 } 2258 2259 // See if it is a narrow oop array. 2260 if (adr_type->isa_aryptr()) { 2261 if (adr_type->offset() >= objArrayOopDesc::base_offset_in_bytes()) { 2262 const TypeOopPtr* elem_type = adr_type->is_aryptr()->elem()->make_oopptr(); 2263 if (elem_type != NULL) { 2264 sharpened_klass = elem_type->klass(); 2265 } 2266 } 2267 } 2268 2269 // The sharpened class might be unloaded if there is no class loader 2270 // constraint in place. 2271 if (sharpened_klass != NULL && sharpened_klass->is_loaded()) { 2272 const TypeOopPtr* tjp = TypeOopPtr::make_from_klass(sharpened_klass); 2273 2274 #ifndef PRODUCT 2275 if (C->print_intrinsics() || C->print_inlining()) { 2276 tty->print(" from base type: "); adr_type->dump(); tty->cr(); 2277 tty->print(" sharpened value: "); tjp->dump(); tty->cr(); 2278 } 2279 #endif 2280 // Sharpen the value type. 2281 return tjp; 2282 } 2283 return NULL; 2284 } 2285 2286 DecoratorSet LibraryCallKit::mo_decorator_for_access_kind(AccessKind kind) { 2287 switch (kind) { 2288 case Relaxed: 2289 return MO_UNORDERED; 2290 case Opaque: 2291 return MO_RELAXED; 2292 case Acquire: 2293 return MO_ACQUIRE; 2294 case Release: 2295 return MO_RELEASE; 2296 case Volatile: 2297 return MO_SEQ_CST; 2298 default: 2299 ShouldNotReachHere(); 2300 return 0; 2301 } 2302 } 2303 2304 bool LibraryCallKit::inline_unsafe_access(bool is_store, const BasicType type, const AccessKind kind, const bool unaligned) { 2305 if (callee()->is_static()) return false; // caller must have the capability! 2306 DecoratorSet decorators = C2_UNSAFE_ACCESS; 2307 guarantee(!is_store || kind != Acquire, "Acquire accesses can be produced only for loads"); 2308 guarantee( is_store || kind != Release, "Release accesses can be produced only for stores"); 2309 assert(type != T_OBJECT || !unaligned, "unaligned access not supported with object type"); 2310 2311 if (is_reference_type(type)) { 2312 decorators |= ON_UNKNOWN_OOP_REF; 2313 } 2314 2315 if (unaligned) { 2316 decorators |= C2_UNALIGNED; 2317 } 2318 2319 #ifndef PRODUCT 2320 { 2321 ResourceMark rm; 2322 // Check the signatures. 2323 ciSignature* sig = callee()->signature(); 2324 #ifdef ASSERT 2325 if (!is_store) { 2326 // Object getReference(Object base, int/long offset), etc. 2327 BasicType rtype = sig->return_type()->basic_type(); 2328 assert(rtype == type, "getter must return the expected value"); 2329 assert(sig->count() == 2, "oop getter has 2 arguments"); 2330 assert(sig->type_at(0)->basic_type() == T_OBJECT, "getter base is object"); 2331 assert(sig->type_at(1)->basic_type() == T_LONG, "getter offset is correct"); 2332 } else { 2333 // void putReference(Object base, int/long offset, Object x), etc. 2334 assert(sig->return_type()->basic_type() == T_VOID, "putter must not return a value"); 2335 assert(sig->count() == 3, "oop putter has 3 arguments"); 2336 assert(sig->type_at(0)->basic_type() == T_OBJECT, "putter base is object"); 2337 assert(sig->type_at(1)->basic_type() == T_LONG, "putter offset is correct"); 2338 BasicType vtype = sig->type_at(sig->count()-1)->basic_type(); 2339 assert(vtype == type, "putter must accept the expected value"); 2340 } 2341 #endif // ASSERT 2342 } 2343 #endif //PRODUCT 2344 2345 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". 2346 2347 Node* receiver = argument(0); // type: oop 2348 2349 // Build address expression. 2350 Node* heap_base_oop = top(); 2351 2352 // The base is either a Java object or a value produced by Unsafe.staticFieldBase 2353 Node* base = argument(1); // type: oop 2354 // The offset is a value produced by Unsafe.staticFieldOffset or Unsafe.objectFieldOffset 2355 Node* offset = argument(2); // type: long 2356 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset 2357 // to be plain byte offsets, which are also the same as those accepted 2358 // by oopDesc::field_addr. 2359 assert(Unsafe_field_offset_to_byte_offset(11) == 11, 2360 "fieldOffset must be byte-scaled"); 2361 // 32-bit machines ignore the high half! 2362 offset = ConvL2X(offset); 2363 2364 // Save state and restore on bailout 2365 uint old_sp = sp(); 2366 SafePointNode* old_map = clone_map(); 2367 2368 Node* adr = make_unsafe_address(base, offset, type, kind == Relaxed); 2369 2370 if (_gvn.type(base)->isa_ptr() == TypePtr::NULL_PTR) { 2371 if (type != T_OBJECT) { 2372 decorators |= IN_NATIVE; // off-heap primitive access 2373 } else { 2374 set_map(old_map); 2375 set_sp(old_sp); 2376 return false; // off-heap oop accesses are not supported 2377 } 2378 } else { 2379 heap_base_oop = base; // on-heap or mixed access 2380 } 2381 2382 // Can base be NULL? Otherwise, always on-heap access. 2383 bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(_gvn.type(base)); 2384 2385 if (!can_access_non_heap) { 2386 decorators |= IN_HEAP; 2387 } 2388 2389 Node* val = is_store ? argument(4) : NULL; 2390 2391 const TypePtr* adr_type = _gvn.type(adr)->isa_ptr(); 2392 if (adr_type == TypePtr::NULL_PTR) { 2393 set_map(old_map); 2394 set_sp(old_sp); 2395 return false; // off-heap access with zero address 2396 } 2397 2398 // Try to categorize the address. 2399 Compile::AliasType* alias_type = C->alias_type(adr_type); 2400 assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here"); 2401 2402 if (alias_type->adr_type() == TypeInstPtr::KLASS || 2403 alias_type->adr_type() == TypeAryPtr::RANGE) { 2404 set_map(old_map); 2405 set_sp(old_sp); 2406 return false; // not supported 2407 } 2408 2409 bool mismatched = false; 2410 BasicType bt = alias_type->basic_type(); 2411 if (bt != T_ILLEGAL) { 2412 assert(alias_type->adr_type()->is_oopptr(), "should be on-heap access"); 2413 if (bt == T_BYTE && adr_type->isa_aryptr()) { 2414 // Alias type doesn't differentiate between byte[] and boolean[]). 2415 // Use address type to get the element type. 2416 bt = adr_type->is_aryptr()->elem()->array_element_basic_type(); 2417 } 2418 if (bt == T_ARRAY || bt == T_NARROWOOP) { 2419 // accessing an array field with getReference is not a mismatch 2420 bt = T_OBJECT; 2421 } 2422 if ((bt == T_OBJECT) != (type == T_OBJECT)) { 2423 // Don't intrinsify mismatched object accesses 2424 set_map(old_map); 2425 set_sp(old_sp); 2426 return false; 2427 } 2428 mismatched = (bt != type); 2429 } else if (alias_type->adr_type()->isa_oopptr()) { 2430 mismatched = true; // conservatively mark all "wide" on-heap accesses as mismatched 2431 } 2432 2433 old_map->destruct(&_gvn); 2434 assert(!mismatched || alias_type->adr_type()->is_oopptr(), "off-heap access can't be mismatched"); 2435 2436 if (mismatched) { 2437 decorators |= C2_MISMATCHED; 2438 } 2439 2440 // First guess at the value type. 2441 const Type *value_type = Type::get_const_basic_type(type); 2442 2443 // Figure out the memory ordering. 2444 decorators |= mo_decorator_for_access_kind(kind); 2445 2446 if (!is_store && type == T_OBJECT) { 2447 const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type); 2448 if (tjp != NULL) { 2449 value_type = tjp; 2450 } 2451 } 2452 2453 receiver = null_check(receiver); 2454 if (stopped()) { 2455 return true; 2456 } 2457 // Heap pointers get a null-check from the interpreter, 2458 // as a courtesy. However, this is not guaranteed by Unsafe, 2459 // and it is not possible to fully distinguish unintended nulls 2460 // from intended ones in this API. 2461 2462 if (!is_store) { 2463 Node* p = NULL; 2464 // Try to constant fold a load from a constant field 2465 ciField* field = alias_type->field(); 2466 if (heap_base_oop != top() && field != NULL && field->is_constant() && !mismatched) { 2467 // final or stable field 2468 p = make_constant_from_field(field, heap_base_oop); 2469 } 2470 2471 if (p == NULL) { // Could not constant fold the load 2472 p = access_load_at(heap_base_oop, adr, adr_type, value_type, type, decorators); 2473 // Normalize the value returned by getBoolean in the following cases 2474 if (type == T_BOOLEAN && 2475 (mismatched || 2476 heap_base_oop == top() || // - heap_base_oop is NULL or 2477 (can_access_non_heap && field == NULL)) // - heap_base_oop is potentially NULL 2478 // and the unsafe access is made to large offset 2479 // (i.e., larger than the maximum offset necessary for any 2480 // field access) 2481 ) { 2482 IdealKit ideal = IdealKit(this); 2483 #define __ ideal. 2484 IdealVariable normalized_result(ideal); 2485 __ declarations_done(); 2486 __ set(normalized_result, p); 2487 __ if_then(p, BoolTest::ne, ideal.ConI(0)); 2488 __ set(normalized_result, ideal.ConI(1)); 2489 ideal.end_if(); 2490 final_sync(ideal); 2491 p = __ value(normalized_result); 2492 #undef __ 2493 } 2494 } 2495 if (type == T_ADDRESS) { 2496 p = gvn().transform(new CastP2XNode(NULL, p)); 2497 p = ConvX2UL(p); 2498 } 2499 // The load node has the control of the preceding MemBarCPUOrder. All 2500 // following nodes will have the control of the MemBarCPUOrder inserted at 2501 // the end of this method. So, pushing the load onto the stack at a later 2502 // point is fine. 2503 set_result(p); 2504 } else { 2505 if (bt == T_ADDRESS) { 2506 // Repackage the long as a pointer. 2507 val = ConvL2X(val); 2508 val = gvn().transform(new CastX2PNode(val)); 2509 } 2510 access_store_at(heap_base_oop, adr, adr_type, val, value_type, type, decorators); 2511 } 2512 2513 return true; 2514 } 2515 2516 //----------------------------inline_unsafe_load_store---------------------------- 2517 // This method serves a couple of different customers (depending on LoadStoreKind): 2518 // 2519 // LS_cmp_swap: 2520 // 2521 // boolean compareAndSetReference(Object o, long offset, Object expected, Object x); 2522 // boolean compareAndSetInt( Object o, long offset, int expected, int x); 2523 // boolean compareAndSetLong( Object o, long offset, long expected, long x); 2524 // 2525 // LS_cmp_swap_weak: 2526 // 2527 // boolean weakCompareAndSetReference( Object o, long offset, Object expected, Object x); 2528 // boolean weakCompareAndSetReferencePlain( Object o, long offset, Object expected, Object x); 2529 // boolean weakCompareAndSetReferenceAcquire(Object o, long offset, Object expected, Object x); 2530 // boolean weakCompareAndSetReferenceRelease(Object o, long offset, Object expected, Object x); 2531 // 2532 // boolean weakCompareAndSetInt( Object o, long offset, int expected, int x); 2533 // boolean weakCompareAndSetIntPlain( Object o, long offset, int expected, int x); 2534 // boolean weakCompareAndSetIntAcquire( Object o, long offset, int expected, int x); 2535 // boolean weakCompareAndSetIntRelease( Object o, long offset, int expected, int x); 2536 // 2537 // boolean weakCompareAndSetLong( Object o, long offset, long expected, long x); 2538 // boolean weakCompareAndSetLongPlain( Object o, long offset, long expected, long x); 2539 // boolean weakCompareAndSetLongAcquire( Object o, long offset, long expected, long x); 2540 // boolean weakCompareAndSetLongRelease( Object o, long offset, long expected, long x); 2541 // 2542 // LS_cmp_exchange: 2543 // 2544 // Object compareAndExchangeReferenceVolatile(Object o, long offset, Object expected, Object x); 2545 // Object compareAndExchangeReferenceAcquire( Object o, long offset, Object expected, Object x); 2546 // Object compareAndExchangeReferenceRelease( Object o, long offset, Object expected, Object x); 2547 // 2548 // Object compareAndExchangeIntVolatile( Object o, long offset, Object expected, Object x); 2549 // Object compareAndExchangeIntAcquire( Object o, long offset, Object expected, Object x); 2550 // Object compareAndExchangeIntRelease( Object o, long offset, Object expected, Object x); 2551 // 2552 // Object compareAndExchangeLongVolatile( Object o, long offset, Object expected, Object x); 2553 // Object compareAndExchangeLongAcquire( Object o, long offset, Object expected, Object x); 2554 // Object compareAndExchangeLongRelease( Object o, long offset, Object expected, Object x); 2555 // 2556 // LS_get_add: 2557 // 2558 // int getAndAddInt( Object o, long offset, int delta) 2559 // long getAndAddLong(Object o, long offset, long delta) 2560 // 2561 // LS_get_set: 2562 // 2563 // int getAndSet(Object o, long offset, int newValue) 2564 // long getAndSet(Object o, long offset, long newValue) 2565 // Object getAndSet(Object o, long offset, Object newValue) 2566 // 2567 bool LibraryCallKit::inline_unsafe_load_store(const BasicType type, const LoadStoreKind kind, const AccessKind access_kind) { 2568 // This basic scheme here is the same as inline_unsafe_access, but 2569 // differs in enough details that combining them would make the code 2570 // overly confusing. (This is a true fact! I originally combined 2571 // them, but even I was confused by it!) As much code/comments as 2572 // possible are retained from inline_unsafe_access though to make 2573 // the correspondences clearer. - dl 2574 2575 if (callee()->is_static()) return false; // caller must have the capability! 2576 2577 DecoratorSet decorators = C2_UNSAFE_ACCESS; 2578 decorators |= mo_decorator_for_access_kind(access_kind); 2579 2580 #ifndef PRODUCT 2581 BasicType rtype; 2582 { 2583 ResourceMark rm; 2584 // Check the signatures. 2585 ciSignature* sig = callee()->signature(); 2586 rtype = sig->return_type()->basic_type(); 2587 switch(kind) { 2588 case LS_get_add: 2589 case LS_get_set: { 2590 // Check the signatures. 2591 #ifdef ASSERT 2592 assert(rtype == type, "get and set must return the expected type"); 2593 assert(sig->count() == 3, "get and set has 3 arguments"); 2594 assert(sig->type_at(0)->basic_type() == T_OBJECT, "get and set base is object"); 2595 assert(sig->type_at(1)->basic_type() == T_LONG, "get and set offset is long"); 2596 assert(sig->type_at(2)->basic_type() == type, "get and set must take expected type as new value/delta"); 2597 assert(access_kind == Volatile, "mo is not passed to intrinsic nodes in current implementation"); 2598 #endif // ASSERT 2599 break; 2600 } 2601 case LS_cmp_swap: 2602 case LS_cmp_swap_weak: { 2603 // Check the signatures. 2604 #ifdef ASSERT 2605 assert(rtype == T_BOOLEAN, "CAS must return boolean"); 2606 assert(sig->count() == 4, "CAS has 4 arguments"); 2607 assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object"); 2608 assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long"); 2609 #endif // ASSERT 2610 break; 2611 } 2612 case LS_cmp_exchange: { 2613 // Check the signatures. 2614 #ifdef ASSERT 2615 assert(rtype == type, "CAS must return the expected type"); 2616 assert(sig->count() == 4, "CAS has 4 arguments"); 2617 assert(sig->type_at(0)->basic_type() == T_OBJECT, "CAS base is object"); 2618 assert(sig->type_at(1)->basic_type() == T_LONG, "CAS offset is long"); 2619 #endif // ASSERT 2620 break; 2621 } 2622 default: 2623 ShouldNotReachHere(); 2624 } 2625 } 2626 #endif //PRODUCT 2627 2628 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". 2629 2630 // Get arguments: 2631 Node* receiver = NULL; 2632 Node* base = NULL; 2633 Node* offset = NULL; 2634 Node* oldval = NULL; 2635 Node* newval = NULL; 2636 switch(kind) { 2637 case LS_cmp_swap: 2638 case LS_cmp_swap_weak: 2639 case LS_cmp_exchange: { 2640 const bool two_slot_type = type2size[type] == 2; 2641 receiver = argument(0); // type: oop 2642 base = argument(1); // type: oop 2643 offset = argument(2); // type: long 2644 oldval = argument(4); // type: oop, int, or long 2645 newval = argument(two_slot_type ? 6 : 5); // type: oop, int, or long 2646 break; 2647 } 2648 case LS_get_add: 2649 case LS_get_set: { 2650 receiver = argument(0); // type: oop 2651 base = argument(1); // type: oop 2652 offset = argument(2); // type: long 2653 oldval = NULL; 2654 newval = argument(4); // type: oop, int, or long 2655 break; 2656 } 2657 default: 2658 ShouldNotReachHere(); 2659 } 2660 2661 // Build field offset expression. 2662 // We currently rely on the cookies produced by Unsafe.xxxFieldOffset 2663 // to be plain byte offsets, which are also the same as those accepted 2664 // by oopDesc::field_addr. 2665 assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); 2666 // 32-bit machines ignore the high half of long offsets 2667 offset = ConvL2X(offset); 2668 // Save state and restore on bailout 2669 uint old_sp = sp(); 2670 SafePointNode* old_map = clone_map(); 2671 Node* adr = make_unsafe_address(base, offset,type, false); 2672 const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); 2673 2674 Compile::AliasType* alias_type = C->alias_type(adr_type); 2675 BasicType bt = alias_type->basic_type(); 2676 if (bt != T_ILLEGAL && 2677 (is_reference_type(bt) != (type == T_OBJECT))) { 2678 // Don't intrinsify mismatched object accesses. 2679 set_map(old_map); 2680 set_sp(old_sp); 2681 return false; 2682 } 2683 2684 old_map->destruct(&_gvn); 2685 2686 // For CAS, unlike inline_unsafe_access, there seems no point in 2687 // trying to refine types. Just use the coarse types here. 2688 assert(alias_type->index() != Compile::AliasIdxBot, "no bare pointers here"); 2689 const Type *value_type = Type::get_const_basic_type(type); 2690 2691 switch (kind) { 2692 case LS_get_set: 2693 case LS_cmp_exchange: { 2694 if (type == T_OBJECT) { 2695 const TypeOopPtr* tjp = sharpen_unsafe_type(alias_type, adr_type); 2696 if (tjp != NULL) { 2697 value_type = tjp; 2698 } 2699 } 2700 break; 2701 } 2702 case LS_cmp_swap: 2703 case LS_cmp_swap_weak: 2704 case LS_get_add: 2705 break; 2706 default: 2707 ShouldNotReachHere(); 2708 } 2709 2710 // Null check receiver. 2711 receiver = null_check(receiver); 2712 if (stopped()) { 2713 return true; 2714 } 2715 2716 int alias_idx = C->get_alias_index(adr_type); 2717 2718 if (is_reference_type(type)) { 2719 decorators |= IN_HEAP | ON_UNKNOWN_OOP_REF; 2720 2721 // Transformation of a value which could be NULL pointer (CastPP #NULL) 2722 // could be delayed during Parse (for example, in adjust_map_after_if()). 2723 // Execute transformation here to avoid barrier generation in such case. 2724 if (_gvn.type(newval) == TypePtr::NULL_PTR) 2725 newval = _gvn.makecon(TypePtr::NULL_PTR); 2726 2727 if (oldval != NULL && _gvn.type(oldval) == TypePtr::NULL_PTR) { 2728 // Refine the value to a null constant, when it is known to be null 2729 oldval = _gvn.makecon(TypePtr::NULL_PTR); 2730 } 2731 } 2732 2733 Node* result = NULL; 2734 switch (kind) { 2735 case LS_cmp_exchange: { 2736 result = access_atomic_cmpxchg_val_at(base, adr, adr_type, alias_idx, 2737 oldval, newval, value_type, type, decorators); 2738 break; 2739 } 2740 case LS_cmp_swap_weak: 2741 decorators |= C2_WEAK_CMPXCHG; 2742 case LS_cmp_swap: { 2743 result = access_atomic_cmpxchg_bool_at(base, adr, adr_type, alias_idx, 2744 oldval, newval, value_type, type, decorators); 2745 break; 2746 } 2747 case LS_get_set: { 2748 result = access_atomic_xchg_at(base, adr, adr_type, alias_idx, 2749 newval, value_type, type, decorators); 2750 break; 2751 } 2752 case LS_get_add: { 2753 result = access_atomic_add_at(base, adr, adr_type, alias_idx, 2754 newval, value_type, type, decorators); 2755 break; 2756 } 2757 default: 2758 ShouldNotReachHere(); 2759 } 2760 2761 assert(type2size[result->bottom_type()->basic_type()] == type2size[rtype], "result type should match"); 2762 set_result(result); 2763 return true; 2764 } 2765 2766 bool LibraryCallKit::inline_unsafe_fence(vmIntrinsics::ID id) { 2767 // Regardless of form, don't allow previous ld/st to move down, 2768 // then issue acquire, release, or volatile mem_bar. 2769 insert_mem_bar(Op_MemBarCPUOrder); 2770 switch(id) { 2771 case vmIntrinsics::_loadFence: 2772 insert_mem_bar(Op_LoadFence); 2773 return true; 2774 case vmIntrinsics::_storeFence: 2775 insert_mem_bar(Op_StoreFence); 2776 return true; 2777 case vmIntrinsics::_storeStoreFence: 2778 insert_mem_bar(Op_StoreStoreFence); 2779 return true; 2780 case vmIntrinsics::_fullFence: 2781 insert_mem_bar(Op_MemBarVolatile); 2782 return true; 2783 default: 2784 fatal_unexpected_iid(id); 2785 return false; 2786 } 2787 } 2788 2789 bool LibraryCallKit::inline_onspinwait() { 2790 insert_mem_bar(Op_OnSpinWait); 2791 return true; 2792 } 2793 2794 bool LibraryCallKit::klass_needs_init_guard(Node* kls) { 2795 if (!kls->is_Con()) { 2796 return true; 2797 } 2798 const TypeKlassPtr* klsptr = kls->bottom_type()->isa_klassptr(); 2799 if (klsptr == NULL) { 2800 return true; 2801 } 2802 ciInstanceKlass* ik = klsptr->klass()->as_instance_klass(); 2803 // don't need a guard for a klass that is already initialized 2804 return !ik->is_initialized(); 2805 } 2806 2807 //----------------------------inline_unsafe_writeback0------------------------- 2808 // public native void Unsafe.writeback0(long address) 2809 bool LibraryCallKit::inline_unsafe_writeback0() { 2810 if (!Matcher::has_match_rule(Op_CacheWB)) { 2811 return false; 2812 } 2813 #ifndef PRODUCT 2814 assert(Matcher::has_match_rule(Op_CacheWBPreSync), "found match rule for CacheWB but not CacheWBPreSync"); 2815 assert(Matcher::has_match_rule(Op_CacheWBPostSync), "found match rule for CacheWB but not CacheWBPostSync"); 2816 ciSignature* sig = callee()->signature(); 2817 assert(sig->type_at(0)->basic_type() == T_LONG, "Unsafe_writeback0 address is long!"); 2818 #endif 2819 null_check_receiver(); // null-check, then ignore 2820 Node *addr = argument(1); 2821 addr = new CastX2PNode(addr); 2822 addr = _gvn.transform(addr); 2823 Node *flush = new CacheWBNode(control(), memory(TypeRawPtr::BOTTOM), addr); 2824 flush = _gvn.transform(flush); 2825 set_memory(flush, TypeRawPtr::BOTTOM); 2826 return true; 2827 } 2828 2829 //----------------------------inline_unsafe_writeback0------------------------- 2830 // public native void Unsafe.writeback0(long address) 2831 bool LibraryCallKit::inline_unsafe_writebackSync0(bool is_pre) { 2832 if (is_pre && !Matcher::has_match_rule(Op_CacheWBPreSync)) { 2833 return false; 2834 } 2835 if (!is_pre && !Matcher::has_match_rule(Op_CacheWBPostSync)) { 2836 return false; 2837 } 2838 #ifndef PRODUCT 2839 assert(Matcher::has_match_rule(Op_CacheWB), 2840 (is_pre ? "found match rule for CacheWBPreSync but not CacheWB" 2841 : "found match rule for CacheWBPostSync but not CacheWB")); 2842 2843 #endif 2844 null_check_receiver(); // null-check, then ignore 2845 Node *sync; 2846 if (is_pre) { 2847 sync = new CacheWBPreSyncNode(control(), memory(TypeRawPtr::BOTTOM)); 2848 } else { 2849 sync = new CacheWBPostSyncNode(control(), memory(TypeRawPtr::BOTTOM)); 2850 } 2851 sync = _gvn.transform(sync); 2852 set_memory(sync, TypeRawPtr::BOTTOM); 2853 return true; 2854 } 2855 2856 //----------------------------inline_unsafe_allocate--------------------------- 2857 // public native Object Unsafe.allocateInstance(Class<?> cls); 2858 bool LibraryCallKit::inline_unsafe_allocate() { 2859 if (callee()->is_static()) return false; // caller must have the capability! 2860 2861 null_check_receiver(); // null-check, then ignore 2862 Node* cls = null_check(argument(1)); 2863 if (stopped()) return true; 2864 2865 Node* kls = load_klass_from_mirror(cls, false, NULL, 0); 2866 kls = null_check(kls); 2867 if (stopped()) return true; // argument was like int.class 2868 2869 Node* test = NULL; 2870 if (LibraryCallKit::klass_needs_init_guard(kls)) { 2871 // Note: The argument might still be an illegal value like 2872 // Serializable.class or Object[].class. The runtime will handle it. 2873 // But we must make an explicit check for initialization. 2874 Node* insp = basic_plus_adr(kls, in_bytes(InstanceKlass::init_state_offset())); 2875 // Use T_BOOLEAN for InstanceKlass::_init_state so the compiler 2876 // can generate code to load it as unsigned byte. 2877 Node* inst = make_load(NULL, insp, TypeInt::UBYTE, T_BOOLEAN, MemNode::unordered); 2878 Node* bits = intcon(InstanceKlass::fully_initialized); 2879 test = _gvn.transform(new SubINode(inst, bits)); 2880 // The 'test' is non-zero if we need to take a slow path. 2881 } 2882 2883 Node* obj = new_instance(kls, test); 2884 set_result(obj); 2885 return true; 2886 } 2887 2888 //------------------------inline_native_time_funcs-------------- 2889 // inline code for System.currentTimeMillis() and System.nanoTime() 2890 // these have the same type and signature 2891 bool LibraryCallKit::inline_native_time_funcs(address funcAddr, const char* funcName) { 2892 const TypeFunc* tf = OptoRuntime::void_long_Type(); 2893 const TypePtr* no_memory_effects = NULL; 2894 Node* time = make_runtime_call(RC_LEAF, tf, funcAddr, funcName, no_memory_effects); 2895 Node* value = _gvn.transform(new ProjNode(time, TypeFunc::Parms+0)); 2896 #ifdef ASSERT 2897 Node* value_top = _gvn.transform(new ProjNode(time, TypeFunc::Parms+1)); 2898 assert(value_top == top(), "second value must be top"); 2899 #endif 2900 set_result(value); 2901 return true; 2902 } 2903 2904 #ifdef JFR_HAVE_INTRINSICS 2905 2906 /** 2907 * if oop->klass != null 2908 * // normal class 2909 * epoch = _epoch_state ? 2 : 1 2910 * if oop->klass->trace_id & ((epoch << META_SHIFT) | epoch)) != epoch { 2911 * ... // enter slow path when the klass is first recorded or the epoch of JFR shifts 2912 * } 2913 * id = oop->klass->trace_id >> TRACE_ID_SHIFT // normal class path 2914 * else 2915 * // primitive class 2916 * if oop->array_klass != null 2917 * id = (oop->array_klass->trace_id >> TRACE_ID_SHIFT) + 1 // primitive class path 2918 * else 2919 * id = LAST_TYPE_ID + 1 // void class path 2920 * if (!signaled) 2921 * signaled = true 2922 */ 2923 bool LibraryCallKit::inline_native_classID() { 2924 Node* cls = argument(0); 2925 2926 IdealKit ideal(this); 2927 #define __ ideal. 2928 IdealVariable result(ideal); __ declarations_done(); 2929 Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), 2930 basic_plus_adr(cls, java_lang_Class::klass_offset()), 2931 TypeRawPtr::BOTTOM, TypeInstKlassPtr::OBJECT_OR_NULL)); 2932 2933 2934 __ if_then(kls, BoolTest::ne, null()); { 2935 Node* kls_trace_id_addr = basic_plus_adr(kls, in_bytes(KLASS_TRACE_ID_OFFSET)); 2936 Node* kls_trace_id_raw = ideal.load(ideal.ctrl(), kls_trace_id_addr,TypeLong::LONG, T_LONG, Compile::AliasIdxRaw); 2937 2938 Node* epoch_address = makecon(TypeRawPtr::make(Jfr::epoch_address())); 2939 Node* epoch = ideal.load(ideal.ctrl(), epoch_address, TypeInt::BOOL, T_BOOLEAN, Compile::AliasIdxRaw); 2940 epoch = _gvn.transform(new LShiftLNode(longcon(1), epoch)); 2941 Node* mask = _gvn.transform(new LShiftLNode(epoch, intcon(META_SHIFT))); 2942 mask = _gvn.transform(new OrLNode(mask, epoch)); 2943 Node* kls_trace_id_raw_and_mask = _gvn.transform(new AndLNode(kls_trace_id_raw, mask)); 2944 2945 float unlikely = PROB_UNLIKELY(0.999); 2946 __ if_then(kls_trace_id_raw_and_mask, BoolTest::ne, epoch, unlikely); { 2947 sync_kit(ideal); 2948 make_runtime_call(RC_LEAF, 2949 OptoRuntime::get_class_id_intrinsic_Type(), 2950 CAST_FROM_FN_PTR(address, Jfr::get_class_id_intrinsic), 2951 "get_class_id_intrinsic", 2952 TypePtr::BOTTOM, 2953 kls); 2954 ideal.sync_kit(this); 2955 } __ end_if(); 2956 2957 ideal.set(result, _gvn.transform(new URShiftLNode(kls_trace_id_raw, ideal.ConI(TRACE_ID_SHIFT)))); 2958 } __ else_(); { 2959 Node* array_kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), 2960 basic_plus_adr(cls, java_lang_Class::array_klass_offset()), 2961 TypeRawPtr::BOTTOM, TypeInstKlassPtr::OBJECT_OR_NULL)); 2962 __ if_then(array_kls, BoolTest::ne, null()); { 2963 Node* array_kls_trace_id_addr = basic_plus_adr(array_kls, in_bytes(KLASS_TRACE_ID_OFFSET)); 2964 Node* array_kls_trace_id_raw = ideal.load(ideal.ctrl(), array_kls_trace_id_addr, TypeLong::LONG, T_LONG, Compile::AliasIdxRaw); 2965 Node* array_kls_trace_id = _gvn.transform(new URShiftLNode(array_kls_trace_id_raw, ideal.ConI(TRACE_ID_SHIFT))); 2966 ideal.set(result, _gvn.transform(new AddLNode(array_kls_trace_id, longcon(1)))); 2967 } __ else_(); { 2968 // void class case 2969 ideal.set(result, _gvn.transform(longcon(LAST_TYPE_ID + 1))); 2970 } __ end_if(); 2971 2972 Node* signaled_flag_address = makecon(TypeRawPtr::make(Jfr::signal_address())); 2973 Node* signaled = ideal.load(ideal.ctrl(), signaled_flag_address, TypeInt::BOOL, T_BOOLEAN, Compile::AliasIdxRaw, true, MemNode::acquire); 2974 __ if_then(signaled, BoolTest::ne, ideal.ConI(1)); { 2975 ideal.store(ideal.ctrl(), signaled_flag_address, ideal.ConI(1), T_BOOLEAN, Compile::AliasIdxRaw, MemNode::release, true); 2976 } __ end_if(); 2977 } __ end_if(); 2978 2979 final_sync(ideal); 2980 set_result(ideal.value(result)); 2981 #undef __ 2982 return true; 2983 } 2984 2985 bool LibraryCallKit::inline_native_getEventWriter() { 2986 Node* tls_ptr = _gvn.transform(new ThreadLocalNode()); 2987 2988 Node* jobj_ptr = basic_plus_adr(top(), tls_ptr, 2989 in_bytes(THREAD_LOCAL_WRITER_OFFSET_JFR)); 2990 2991 Node* jobj = make_load(control(), jobj_ptr, TypeRawPtr::BOTTOM, T_ADDRESS, MemNode::unordered); 2992 2993 Node* jobj_cmp_null = _gvn.transform( new CmpPNode(jobj, null()) ); 2994 Node* test_jobj_eq_null = _gvn.transform( new BoolNode(jobj_cmp_null, BoolTest::eq) ); 2995 2996 IfNode* iff_jobj_null = 2997 create_and_map_if(control(), test_jobj_eq_null, PROB_MIN, COUNT_UNKNOWN); 2998 2999 enum { _normal_path = 1, 3000 _null_path = 2, 3001 PATH_LIMIT }; 3002 3003 RegionNode* result_rgn = new RegionNode(PATH_LIMIT); 3004 PhiNode* result_val = new PhiNode(result_rgn, TypeInstPtr::BOTTOM); 3005 3006 Node* jobj_is_null = _gvn.transform(new IfTrueNode(iff_jobj_null)); 3007 result_rgn->init_req(_null_path, jobj_is_null); 3008 result_val->init_req(_null_path, null()); 3009 3010 Node* jobj_is_not_null = _gvn.transform(new IfFalseNode(iff_jobj_null)); 3011 set_control(jobj_is_not_null); 3012 Node* res = access_load(jobj, TypeInstPtr::NOTNULL, T_OBJECT, 3013 IN_NATIVE | C2_CONTROL_DEPENDENT_LOAD); 3014 result_rgn->init_req(_normal_path, control()); 3015 result_val->init_req(_normal_path, res); 3016 3017 set_result(result_rgn, result_val); 3018 3019 return true; 3020 } 3021 3022 #endif // JFR_HAVE_INTRINSICS 3023 3024 //------------------------inline_native_currentThread------------------ 3025 bool LibraryCallKit::inline_native_currentThread() { 3026 Node* junk = NULL; 3027 set_result(generate_current_thread(junk)); 3028 return true; 3029 } 3030 3031 //---------------------------load_mirror_from_klass---------------------------- 3032 // Given a klass oop, load its java mirror (a java.lang.Class oop). 3033 Node* LibraryCallKit::load_mirror_from_klass(Node* klass) { 3034 Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset())); 3035 Node* load = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered); 3036 // mirror = ((OopHandle)mirror)->resolve(); 3037 return access_load(load, TypeInstPtr::MIRROR, T_OBJECT, IN_NATIVE); 3038 } 3039 3040 //-----------------------load_klass_from_mirror_common------------------------- 3041 // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop. 3042 // Test the klass oop for null (signifying a primitive Class like Integer.TYPE), 3043 // and branch to the given path on the region. 3044 // If never_see_null, take an uncommon trap on null, so we can optimistically 3045 // compile for the non-null case. 3046 // If the region is NULL, force never_see_null = true. 3047 Node* LibraryCallKit::load_klass_from_mirror_common(Node* mirror, 3048 bool never_see_null, 3049 RegionNode* region, 3050 int null_path, 3051 int offset) { 3052 if (region == NULL) never_see_null = true; 3053 Node* p = basic_plus_adr(mirror, offset); 3054 const TypeKlassPtr* kls_type = TypeInstKlassPtr::OBJECT_OR_NULL; 3055 Node* kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, kls_type)); 3056 Node* null_ctl = top(); 3057 kls = null_check_oop(kls, &null_ctl, never_see_null); 3058 if (region != NULL) { 3059 // Set region->in(null_path) if the mirror is a primitive (e.g, int.class). 3060 region->init_req(null_path, null_ctl); 3061 } else { 3062 assert(null_ctl == top(), "no loose ends"); 3063 } 3064 return kls; 3065 } 3066 3067 //--------------------(inline_native_Class_query helpers)--------------------- 3068 // Use this for JVM_ACC_INTERFACE, JVM_ACC_IS_CLONEABLE_FAST, JVM_ACC_HAS_FINALIZER. 3069 // Fall through if (mods & mask) == bits, take the guard otherwise. 3070 Node* LibraryCallKit::generate_access_flags_guard(Node* kls, int modifier_mask, int modifier_bits, RegionNode* region) { 3071 // Branch around if the given klass has the given modifier bit set. 3072 // Like generate_guard, adds a new path onto the region. 3073 Node* modp = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset())); 3074 Node* mods = make_load(NULL, modp, TypeInt::INT, T_INT, MemNode::unordered); 3075 Node* mask = intcon(modifier_mask); 3076 Node* bits = intcon(modifier_bits); 3077 Node* mbit = _gvn.transform(new AndINode(mods, mask)); 3078 Node* cmp = _gvn.transform(new CmpINode(mbit, bits)); 3079 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::ne)); 3080 return generate_fair_guard(bol, region); 3081 } 3082 Node* LibraryCallKit::generate_interface_guard(Node* kls, RegionNode* region) { 3083 return generate_access_flags_guard(kls, JVM_ACC_INTERFACE, 0, region); 3084 } 3085 Node* LibraryCallKit::generate_hidden_class_guard(Node* kls, RegionNode* region) { 3086 return generate_access_flags_guard(kls, JVM_ACC_IS_HIDDEN_CLASS, 0, region); 3087 } 3088 3089 //-------------------------inline_native_Class_query------------------- 3090 bool LibraryCallKit::inline_native_Class_query(vmIntrinsics::ID id) { 3091 const Type* return_type = TypeInt::BOOL; 3092 Node* prim_return_value = top(); // what happens if it's a primitive class? 3093 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); 3094 bool expect_prim = false; // most of these guys expect to work on refs 3095 3096 enum { _normal_path = 1, _prim_path = 2, PATH_LIMIT }; 3097 3098 Node* mirror = argument(0); 3099 Node* obj = top(); 3100 3101 switch (id) { 3102 case vmIntrinsics::_isInstance: 3103 // nothing is an instance of a primitive type 3104 prim_return_value = intcon(0); 3105 obj = argument(1); 3106 break; 3107 case vmIntrinsics::_getModifiers: 3108 prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC); 3109 assert(is_power_of_2((int)JVM_ACC_WRITTEN_FLAGS+1), "change next line"); 3110 return_type = TypeInt::make(0, JVM_ACC_WRITTEN_FLAGS, Type::WidenMin); 3111 break; 3112 case vmIntrinsics::_isInterface: 3113 prim_return_value = intcon(0); 3114 break; 3115 case vmIntrinsics::_isArray: 3116 prim_return_value = intcon(0); 3117 expect_prim = true; // cf. ObjectStreamClass.getClassSignature 3118 break; 3119 case vmIntrinsics::_isPrimitive: 3120 prim_return_value = intcon(1); 3121 expect_prim = true; // obviously 3122 break; 3123 case vmIntrinsics::_isHidden: 3124 prim_return_value = intcon(0); 3125 break; 3126 case vmIntrinsics::_getSuperclass: 3127 prim_return_value = null(); 3128 return_type = TypeInstPtr::MIRROR->cast_to_ptr_type(TypePtr::BotPTR); 3129 break; 3130 case vmIntrinsics::_getClassAccessFlags: 3131 prim_return_value = intcon(JVM_ACC_ABSTRACT | JVM_ACC_FINAL | JVM_ACC_PUBLIC); 3132 return_type = TypeInt::INT; // not bool! 6297094 3133 break; 3134 default: 3135 fatal_unexpected_iid(id); 3136 break; 3137 } 3138 3139 const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr(); 3140 if (mirror_con == NULL) return false; // cannot happen? 3141 3142 #ifndef PRODUCT 3143 if (C->print_intrinsics() || C->print_inlining()) { 3144 ciType* k = mirror_con->java_mirror_type(); 3145 if (k) { 3146 tty->print("Inlining %s on constant Class ", vmIntrinsics::name_at(intrinsic_id())); 3147 k->print_name(); 3148 tty->cr(); 3149 } 3150 } 3151 #endif 3152 3153 // Null-check the mirror, and the mirror's klass ptr (in case it is a primitive). 3154 RegionNode* region = new RegionNode(PATH_LIMIT); 3155 record_for_igvn(region); 3156 PhiNode* phi = new PhiNode(region, return_type); 3157 3158 // The mirror will never be null of Reflection.getClassAccessFlags, however 3159 // it may be null for Class.isInstance or Class.getModifiers. Throw a NPE 3160 // if it is. See bug 4774291. 3161 3162 // For Reflection.getClassAccessFlags(), the null check occurs in 3163 // the wrong place; see inline_unsafe_access(), above, for a similar 3164 // situation. 3165 mirror = null_check(mirror); 3166 // If mirror or obj is dead, only null-path is taken. 3167 if (stopped()) return true; 3168 3169 if (expect_prim) never_see_null = false; // expect nulls (meaning prims) 3170 3171 // Now load the mirror's klass metaobject, and null-check it. 3172 // Side-effects region with the control path if the klass is null. 3173 Node* kls = load_klass_from_mirror(mirror, never_see_null, region, _prim_path); 3174 // If kls is null, we have a primitive mirror. 3175 phi->init_req(_prim_path, prim_return_value); 3176 if (stopped()) { set_result(region, phi); return true; } 3177 bool safe_for_replace = (region->in(_prim_path) == top()); 3178 3179 Node* p; // handy temp 3180 Node* null_ctl; 3181 3182 // Now that we have the non-null klass, we can perform the real query. 3183 // For constant classes, the query will constant-fold in LoadNode::Value. 3184 Node* query_value = top(); 3185 switch (id) { 3186 case vmIntrinsics::_isInstance: 3187 // nothing is an instance of a primitive type 3188 query_value = gen_instanceof(obj, kls, safe_for_replace); 3189 break; 3190 3191 case vmIntrinsics::_getModifiers: 3192 p = basic_plus_adr(kls, in_bytes(Klass::modifier_flags_offset())); 3193 query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered); 3194 break; 3195 3196 case vmIntrinsics::_isInterface: 3197 // (To verify this code sequence, check the asserts in JVM_IsInterface.) 3198 if (generate_interface_guard(kls, region) != NULL) 3199 // A guard was added. If the guard is taken, it was an interface. 3200 phi->add_req(intcon(1)); 3201 // If we fall through, it's a plain class. 3202 query_value = intcon(0); 3203 break; 3204 3205 case vmIntrinsics::_isArray: 3206 // (To verify this code sequence, check the asserts in JVM_IsArrayClass.) 3207 if (generate_array_guard(kls, region) != NULL) 3208 // A guard was added. If the guard is taken, it was an array. 3209 phi->add_req(intcon(1)); 3210 // If we fall through, it's a plain class. 3211 query_value = intcon(0); 3212 break; 3213 3214 case vmIntrinsics::_isPrimitive: 3215 query_value = intcon(0); // "normal" path produces false 3216 break; 3217 3218 case vmIntrinsics::_isHidden: 3219 // (To verify this code sequence, check the asserts in JVM_IsHiddenClass.) 3220 if (generate_hidden_class_guard(kls, region) != NULL) 3221 // A guard was added. If the guard is taken, it was an hidden class. 3222 phi->add_req(intcon(1)); 3223 // If we fall through, it's a plain class. 3224 query_value = intcon(0); 3225 break; 3226 3227 3228 case vmIntrinsics::_getSuperclass: 3229 // The rules here are somewhat unfortunate, but we can still do better 3230 // with random logic than with a JNI call. 3231 // Interfaces store null or Object as _super, but must report null. 3232 // Arrays store an intermediate super as _super, but must report Object. 3233 // Other types can report the actual _super. 3234 // (To verify this code sequence, check the asserts in JVM_IsInterface.) 3235 if (generate_interface_guard(kls, region) != NULL) 3236 // A guard was added. If the guard is taken, it was an interface. 3237 phi->add_req(null()); 3238 if (generate_array_guard(kls, region) != NULL) 3239 // A guard was added. If the guard is taken, it was an array. 3240 phi->add_req(makecon(TypeInstPtr::make(env()->Object_klass()->java_mirror()))); 3241 // If we fall through, it's a plain class. Get its _super. 3242 p = basic_plus_adr(kls, in_bytes(Klass::super_offset())); 3243 kls = _gvn.transform(LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, TypeRawPtr::BOTTOM, TypeInstKlassPtr::OBJECT_OR_NULL)); 3244 null_ctl = top(); 3245 kls = null_check_oop(kls, &null_ctl); 3246 if (null_ctl != top()) { 3247 // If the guard is taken, Object.superClass is null (both klass and mirror). 3248 region->add_req(null_ctl); 3249 phi ->add_req(null()); 3250 } 3251 if (!stopped()) { 3252 query_value = load_mirror_from_klass(kls); 3253 } 3254 break; 3255 3256 case vmIntrinsics::_getClassAccessFlags: 3257 p = basic_plus_adr(kls, in_bytes(Klass::access_flags_offset())); 3258 query_value = make_load(NULL, p, TypeInt::INT, T_INT, MemNode::unordered); 3259 break; 3260 3261 default: 3262 fatal_unexpected_iid(id); 3263 break; 3264 } 3265 3266 // Fall-through is the normal case of a query to a real class. 3267 phi->init_req(1, query_value); 3268 region->init_req(1, control()); 3269 3270 C->set_has_split_ifs(true); // Has chance for split-if optimization 3271 set_result(region, phi); 3272 return true; 3273 } 3274 3275 //-------------------------inline_Class_cast------------------- 3276 bool LibraryCallKit::inline_Class_cast() { 3277 Node* mirror = argument(0); // Class 3278 Node* obj = argument(1); 3279 const TypeInstPtr* mirror_con = _gvn.type(mirror)->isa_instptr(); 3280 if (mirror_con == NULL) { 3281 return false; // dead path (mirror->is_top()). 3282 } 3283 if (obj == NULL || obj->is_top()) { 3284 return false; // dead path 3285 } 3286 const TypeOopPtr* tp = _gvn.type(obj)->isa_oopptr(); 3287 3288 // First, see if Class.cast() can be folded statically. 3289 // java_mirror_type() returns non-null for compile-time Class constants. 3290 ciType* tm = mirror_con->java_mirror_type(); 3291 if (tm != NULL && tm->is_klass() && 3292 tp != NULL && tp->klass() != NULL) { 3293 if (!tp->klass()->is_loaded()) { 3294 // Don't use intrinsic when class is not loaded. 3295 return false; 3296 } else { 3297 int static_res = C->static_subtype_check(tm->as_klass(), tp->klass()); 3298 if (static_res == Compile::SSC_always_true) { 3299 // isInstance() is true - fold the code. 3300 set_result(obj); 3301 return true; 3302 } else if (static_res == Compile::SSC_always_false) { 3303 // Don't use intrinsic, have to throw ClassCastException. 3304 // If the reference is null, the non-intrinsic bytecode will 3305 // be optimized appropriately. 3306 return false; 3307 } 3308 } 3309 } 3310 3311 // Bailout intrinsic and do normal inlining if exception path is frequent. 3312 if (too_many_traps(Deoptimization::Reason_intrinsic)) { 3313 return false; 3314 } 3315 3316 // Generate dynamic checks. 3317 // Class.cast() is java implementation of _checkcast bytecode. 3318 // Do checkcast (Parse::do_checkcast()) optimizations here. 3319 3320 mirror = null_check(mirror); 3321 // If mirror is dead, only null-path is taken. 3322 if (stopped()) { 3323 return true; 3324 } 3325 3326 // Not-subtype or the mirror's klass ptr is NULL (in case it is a primitive). 3327 enum { _bad_type_path = 1, _prim_path = 2, PATH_LIMIT }; 3328 RegionNode* region = new RegionNode(PATH_LIMIT); 3329 record_for_igvn(region); 3330 3331 // Now load the mirror's klass metaobject, and null-check it. 3332 // If kls is null, we have a primitive mirror and 3333 // nothing is an instance of a primitive type. 3334 Node* kls = load_klass_from_mirror(mirror, false, region, _prim_path); 3335 3336 Node* res = top(); 3337 if (!stopped()) { 3338 Node* bad_type_ctrl = top(); 3339 // Do checkcast optimizations. 3340 res = gen_checkcast(obj, kls, &bad_type_ctrl); 3341 region->init_req(_bad_type_path, bad_type_ctrl); 3342 } 3343 if (region->in(_prim_path) != top() || 3344 region->in(_bad_type_path) != top()) { 3345 // Let Interpreter throw ClassCastException. 3346 PreserveJVMState pjvms(this); 3347 set_control(_gvn.transform(region)); 3348 uncommon_trap(Deoptimization::Reason_intrinsic, 3349 Deoptimization::Action_maybe_recompile); 3350 } 3351 if (!stopped()) { 3352 set_result(res); 3353 } 3354 return true; 3355 } 3356 3357 3358 //--------------------------inline_native_subtype_check------------------------ 3359 // This intrinsic takes the JNI calls out of the heart of 3360 // UnsafeFieldAccessorImpl.set, which improves Field.set, readObject, etc. 3361 bool LibraryCallKit::inline_native_subtype_check() { 3362 // Pull both arguments off the stack. 3363 Node* args[2]; // two java.lang.Class mirrors: superc, subc 3364 args[0] = argument(0); 3365 args[1] = argument(1); 3366 Node* klasses[2]; // corresponding Klasses: superk, subk 3367 klasses[0] = klasses[1] = top(); 3368 3369 enum { 3370 // A full decision tree on {superc is prim, subc is prim}: 3371 _prim_0_path = 1, // {P,N} => false 3372 // {P,P} & superc!=subc => false 3373 _prim_same_path, // {P,P} & superc==subc => true 3374 _prim_1_path, // {N,P} => false 3375 _ref_subtype_path, // {N,N} & subtype check wins => true 3376 _both_ref_path, // {N,N} & subtype check loses => false 3377 PATH_LIMIT 3378 }; 3379 3380 RegionNode* region = new RegionNode(PATH_LIMIT); 3381 Node* phi = new PhiNode(region, TypeInt::BOOL); 3382 record_for_igvn(region); 3383 3384 const TypePtr* adr_type = TypeRawPtr::BOTTOM; // memory type of loads 3385 const TypeKlassPtr* kls_type = TypeInstKlassPtr::OBJECT_OR_NULL; 3386 int class_klass_offset = java_lang_Class::klass_offset(); 3387 3388 // First null-check both mirrors and load each mirror's klass metaobject. 3389 int which_arg; 3390 for (which_arg = 0; which_arg <= 1; which_arg++) { 3391 Node* arg = args[which_arg]; 3392 arg = null_check(arg); 3393 if (stopped()) break; 3394 args[which_arg] = arg; 3395 3396 Node* p = basic_plus_adr(arg, class_klass_offset); 3397 Node* kls = LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, adr_type, kls_type); 3398 klasses[which_arg] = _gvn.transform(kls); 3399 } 3400 3401 // Having loaded both klasses, test each for null. 3402 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); 3403 for (which_arg = 0; which_arg <= 1; which_arg++) { 3404 Node* kls = klasses[which_arg]; 3405 Node* null_ctl = top(); 3406 kls = null_check_oop(kls, &null_ctl, never_see_null); 3407 int prim_path = (which_arg == 0 ? _prim_0_path : _prim_1_path); 3408 region->init_req(prim_path, null_ctl); 3409 if (stopped()) break; 3410 klasses[which_arg] = kls; 3411 } 3412 3413 if (!stopped()) { 3414 // now we have two reference types, in klasses[0..1] 3415 Node* subk = klasses[1]; // the argument to isAssignableFrom 3416 Node* superk = klasses[0]; // the receiver 3417 region->set_req(_both_ref_path, gen_subtype_check(subk, superk)); 3418 // now we have a successful reference subtype check 3419 region->set_req(_ref_subtype_path, control()); 3420 } 3421 3422 // If both operands are primitive (both klasses null), then 3423 // we must return true when they are identical primitives. 3424 // It is convenient to test this after the first null klass check. 3425 set_control(region->in(_prim_0_path)); // go back to first null check 3426 if (!stopped()) { 3427 // Since superc is primitive, make a guard for the superc==subc case. 3428 Node* cmp_eq = _gvn.transform(new CmpPNode(args[0], args[1])); 3429 Node* bol_eq = _gvn.transform(new BoolNode(cmp_eq, BoolTest::eq)); 3430 generate_guard(bol_eq, region, PROB_FAIR); 3431 if (region->req() == PATH_LIMIT+1) { 3432 // A guard was added. If the added guard is taken, superc==subc. 3433 region->swap_edges(PATH_LIMIT, _prim_same_path); 3434 region->del_req(PATH_LIMIT); 3435 } 3436 region->set_req(_prim_0_path, control()); // Not equal after all. 3437 } 3438 3439 // these are the only paths that produce 'true': 3440 phi->set_req(_prim_same_path, intcon(1)); 3441 phi->set_req(_ref_subtype_path, intcon(1)); 3442 3443 // pull together the cases: 3444 assert(region->req() == PATH_LIMIT, "sane region"); 3445 for (uint i = 1; i < region->req(); i++) { 3446 Node* ctl = region->in(i); 3447 if (ctl == NULL || ctl == top()) { 3448 region->set_req(i, top()); 3449 phi ->set_req(i, top()); 3450 } else if (phi->in(i) == NULL) { 3451 phi->set_req(i, intcon(0)); // all other paths produce 'false' 3452 } 3453 } 3454 3455 set_control(_gvn.transform(region)); 3456 set_result(_gvn.transform(phi)); 3457 return true; 3458 } 3459 3460 //---------------------generate_array_guard_common------------------------ 3461 Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region, 3462 bool obj_array, bool not_array) { 3463 3464 if (stopped()) { 3465 return NULL; 3466 } 3467 3468 // If obj_array/non_array==false/false: 3469 // Branch around if the given klass is in fact an array (either obj or prim). 3470 // If obj_array/non_array==false/true: 3471 // Branch around if the given klass is not an array klass of any kind. 3472 // If obj_array/non_array==true/true: 3473 // Branch around if the kls is not an oop array (kls is int[], String, etc.) 3474 // If obj_array/non_array==true/false: 3475 // Branch around if the kls is an oop array (Object[] or subtype) 3476 // 3477 // Like generate_guard, adds a new path onto the region. 3478 jint layout_con = 0; 3479 Node* layout_val = get_layout_helper(kls, layout_con); 3480 if (layout_val == NULL) { 3481 bool query = (obj_array 3482 ? Klass::layout_helper_is_objArray(layout_con) 3483 : Klass::layout_helper_is_array(layout_con)); 3484 if (query == not_array) { 3485 return NULL; // never a branch 3486 } else { // always a branch 3487 Node* always_branch = control(); 3488 if (region != NULL) 3489 region->add_req(always_branch); 3490 set_control(top()); 3491 return always_branch; 3492 } 3493 } 3494 // Now test the correct condition. 3495 jint nval = (obj_array 3496 ? (jint)(Klass::_lh_array_tag_type_value 3497 << Klass::_lh_array_tag_shift) 3498 : Klass::_lh_neutral_value); 3499 Node* cmp = _gvn.transform(new CmpINode(layout_val, intcon(nval))); 3500 BoolTest::mask btest = BoolTest::lt; // correct for testing is_[obj]array 3501 // invert the test if we are looking for a non-array 3502 if (not_array) btest = BoolTest(btest).negate(); 3503 Node* bol = _gvn.transform(new BoolNode(cmp, btest)); 3504 return generate_fair_guard(bol, region); 3505 } 3506 3507 3508 //-----------------------inline_native_newArray-------------------------- 3509 // private static native Object java.lang.reflect.newArray(Class<?> componentType, int length); 3510 // private native Object Unsafe.allocateUninitializedArray0(Class<?> cls, int size); 3511 bool LibraryCallKit::inline_unsafe_newArray(bool uninitialized) { 3512 Node* mirror; 3513 Node* count_val; 3514 if (uninitialized) { 3515 mirror = argument(1); 3516 count_val = argument(2); 3517 } else { 3518 mirror = argument(0); 3519 count_val = argument(1); 3520 } 3521 3522 mirror = null_check(mirror); 3523 // If mirror or obj is dead, only null-path is taken. 3524 if (stopped()) return true; 3525 3526 enum { _normal_path = 1, _slow_path = 2, PATH_LIMIT }; 3527 RegionNode* result_reg = new RegionNode(PATH_LIMIT); 3528 PhiNode* result_val = new PhiNode(result_reg, TypeInstPtr::NOTNULL); 3529 PhiNode* result_io = new PhiNode(result_reg, Type::ABIO); 3530 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM); 3531 3532 bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); 3533 Node* klass_node = load_array_klass_from_mirror(mirror, never_see_null, 3534 result_reg, _slow_path); 3535 Node* normal_ctl = control(); 3536 Node* no_array_ctl = result_reg->in(_slow_path); 3537 3538 // Generate code for the slow case. We make a call to newArray(). 3539 set_control(no_array_ctl); 3540 if (!stopped()) { 3541 // Either the input type is void.class, or else the 3542 // array klass has not yet been cached. Either the 3543 // ensuing call will throw an exception, or else it 3544 // will cache the array klass for next time. 3545 PreserveJVMState pjvms(this); 3546 CallJavaNode* slow_call = NULL; 3547 if (uninitialized) { 3548 // Generate optimized virtual call (holder class 'Unsafe' is final) 3549 slow_call = generate_method_call(vmIntrinsics::_allocateUninitializedArray, false, false); 3550 } else { 3551 slow_call = generate_method_call_static(vmIntrinsics::_newArray); 3552 } 3553 Node* slow_result = set_results_for_java_call(slow_call); 3554 // this->control() comes from set_results_for_java_call 3555 result_reg->set_req(_slow_path, control()); 3556 result_val->set_req(_slow_path, slow_result); 3557 result_io ->set_req(_slow_path, i_o()); 3558 result_mem->set_req(_slow_path, reset_memory()); 3559 } 3560 3561 set_control(normal_ctl); 3562 if (!stopped()) { 3563 // Normal case: The array type has been cached in the java.lang.Class. 3564 // The following call works fine even if the array type is polymorphic. 3565 // It could be a dynamic mix of int[], boolean[], Object[], etc. 3566 Node* obj = new_array(klass_node, count_val, 0); // no arguments to push 3567 result_reg->init_req(_normal_path, control()); 3568 result_val->init_req(_normal_path, obj); 3569 result_io ->init_req(_normal_path, i_o()); 3570 result_mem->init_req(_normal_path, reset_memory()); 3571 3572 if (uninitialized) { 3573 // Mark the allocation so that zeroing is skipped 3574 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(obj, &_gvn); 3575 alloc->maybe_set_complete(&_gvn); 3576 } 3577 } 3578 3579 // Return the combined state. 3580 set_i_o( _gvn.transform(result_io) ); 3581 set_all_memory( _gvn.transform(result_mem)); 3582 3583 C->set_has_split_ifs(true); // Has chance for split-if optimization 3584 set_result(result_reg, result_val); 3585 return true; 3586 } 3587 3588 //----------------------inline_native_getLength-------------------------- 3589 // public static native int java.lang.reflect.Array.getLength(Object array); 3590 bool LibraryCallKit::inline_native_getLength() { 3591 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; 3592 3593 Node* array = null_check(argument(0)); 3594 // If array is dead, only null-path is taken. 3595 if (stopped()) return true; 3596 3597 // Deoptimize if it is a non-array. 3598 Node* non_array = generate_non_array_guard(load_object_klass(array), NULL); 3599 3600 if (non_array != NULL) { 3601 PreserveJVMState pjvms(this); 3602 set_control(non_array); 3603 uncommon_trap(Deoptimization::Reason_intrinsic, 3604 Deoptimization::Action_maybe_recompile); 3605 } 3606 3607 // If control is dead, only non-array-path is taken. 3608 if (stopped()) return true; 3609 3610 // The works fine even if the array type is polymorphic. 3611 // It could be a dynamic mix of int[], boolean[], Object[], etc. 3612 Node* result = load_array_length(array); 3613 3614 C->set_has_split_ifs(true); // Has chance for split-if optimization 3615 set_result(result); 3616 return true; 3617 } 3618 3619 //------------------------inline_array_copyOf---------------------------- 3620 // public static <T,U> T[] java.util.Arrays.copyOf( U[] original, int newLength, Class<? extends T[]> newType); 3621 // public static <T,U> T[] java.util.Arrays.copyOfRange(U[] original, int from, int to, Class<? extends T[]> newType); 3622 bool LibraryCallKit::inline_array_copyOf(bool is_copyOfRange) { 3623 if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; 3624 3625 // Get the arguments. 3626 Node* original = argument(0); 3627 Node* start = is_copyOfRange? argument(1): intcon(0); 3628 Node* end = is_copyOfRange? argument(2): argument(1); 3629 Node* array_type_mirror = is_copyOfRange? argument(3): argument(2); 3630 3631 Node* newcopy = NULL; 3632 3633 // Set the original stack and the reexecute bit for the interpreter to reexecute 3634 // the bytecode that invokes Arrays.copyOf if deoptimization happens. 3635 { PreserveReexecuteState preexecs(this); 3636 jvms()->set_should_reexecute(true); 3637 3638 array_type_mirror = null_check(array_type_mirror); 3639 original = null_check(original); 3640 3641 // Check if a null path was taken unconditionally. 3642 if (stopped()) return true; 3643 3644 Node* orig_length = load_array_length(original); 3645 3646 Node* klass_node = load_klass_from_mirror(array_type_mirror, false, NULL, 0); 3647 klass_node = null_check(klass_node); 3648 3649 RegionNode* bailout = new RegionNode(1); 3650 record_for_igvn(bailout); 3651 3652 // Despite the generic type of Arrays.copyOf, the mirror might be int, int[], etc. 3653 // Bail out if that is so. 3654 Node* not_objArray = generate_non_objArray_guard(klass_node, bailout); 3655 if (not_objArray != NULL) { 3656 // Improve the klass node's type from the new optimistic assumption: 3657 ciKlass* ak = ciArrayKlass::make(env()->Object_klass()); 3658 const Type* akls = TypeKlassPtr::make(TypePtr::NotNull, ak, 0/*offset*/); 3659 Node* cast = new CastPPNode(klass_node, akls); 3660 cast->init_req(0, control()); 3661 klass_node = _gvn.transform(cast); 3662 } 3663 3664 // Bail out if either start or end is negative. 3665 generate_negative_guard(start, bailout, &start); 3666 generate_negative_guard(end, bailout, &end); 3667 3668 Node* length = end; 3669 if (_gvn.type(start) != TypeInt::ZERO) { 3670 length = _gvn.transform(new SubINode(end, start)); 3671 } 3672 3673 // Bail out if length is negative. 3674 // Without this the new_array would throw 3675 // NegativeArraySizeException but IllegalArgumentException is what 3676 // should be thrown 3677 generate_negative_guard(length, bailout, &length); 3678 3679 if (bailout->req() > 1) { 3680 PreserveJVMState pjvms(this); 3681 set_control(_gvn.transform(bailout)); 3682 uncommon_trap(Deoptimization::Reason_intrinsic, 3683 Deoptimization::Action_maybe_recompile); 3684 } 3685 3686 if (!stopped()) { 3687 // How many elements will we copy from the original? 3688 // The answer is MinI(orig_length - start, length). 3689 Node* orig_tail = _gvn.transform(new SubINode(orig_length, start)); 3690 Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length); 3691 3692 // Generate a direct call to the right arraycopy function(s). 3693 // We know the copy is disjoint but we might not know if the 3694 // oop stores need checking. 3695 // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class). 3696 // This will fail a store-check if x contains any non-nulls. 3697 3698 // ArrayCopyNode:Ideal may transform the ArrayCopyNode to 3699 // loads/stores but it is legal only if we're sure the 3700 // Arrays.copyOf would succeed. So we need all input arguments 3701 // to the copyOf to be validated, including that the copy to the 3702 // new array won't trigger an ArrayStoreException. That subtype 3703 // check can be optimized if we know something on the type of 3704 // the input array from type speculation. 3705 if (_gvn.type(klass_node)->singleton()) { 3706 ciKlass* subk = _gvn.type(load_object_klass(original))->is_klassptr()->klass(); 3707 ciKlass* superk = _gvn.type(klass_node)->is_klassptr()->klass(); 3708 3709 int test = C->static_subtype_check(superk, subk); 3710 if (test != Compile::SSC_always_true && test != Compile::SSC_always_false) { 3711 const TypeOopPtr* t_original = _gvn.type(original)->is_oopptr(); 3712 if (t_original->speculative_type() != NULL) { 3713 original = maybe_cast_profiled_obj(original, t_original->speculative_type(), true); 3714 } 3715 } 3716 } 3717 3718 bool validated = false; 3719 // Reason_class_check rather than Reason_intrinsic because we 3720 // want to intrinsify even if this traps. 3721 if (!too_many_traps(Deoptimization::Reason_class_check)) { 3722 Node* not_subtype_ctrl = gen_subtype_check(original, klass_node); 3723 3724 if (not_subtype_ctrl != top()) { 3725 PreserveJVMState pjvms(this); 3726 set_control(not_subtype_ctrl); 3727 uncommon_trap(Deoptimization::Reason_class_check, 3728 Deoptimization::Action_make_not_entrant); 3729 assert(stopped(), "Should be stopped"); 3730 } 3731 validated = true; 3732 } 3733 3734 if (!stopped()) { 3735 newcopy = new_array(klass_node, length, 0); // no arguments to push 3736 3737 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, original, start, newcopy, intcon(0), moved, true, false, 3738 load_object_klass(original), klass_node); 3739 if (!is_copyOfRange) { 3740 ac->set_copyof(validated); 3741 } else { 3742 ac->set_copyofrange(validated); 3743 } 3744 Node* n = _gvn.transform(ac); 3745 if (n == ac) { 3746 ac->connect_outputs(this); 3747 } else { 3748 assert(validated, "shouldn't transform if all arguments not validated"); 3749 set_all_memory(n); 3750 } 3751 } 3752 } 3753 } // original reexecute is set back here 3754 3755 C->set_has_split_ifs(true); // Has chance for split-if optimization 3756 if (!stopped()) { 3757 set_result(newcopy); 3758 } 3759 return true; 3760 } 3761 3762 3763 //----------------------generate_virtual_guard--------------------------- 3764 // Helper for hashCode and clone. Peeks inside the vtable to avoid a call. 3765 Node* LibraryCallKit::generate_virtual_guard(Node* obj_klass, 3766 RegionNode* slow_region) { 3767 ciMethod* method = callee(); 3768 int vtable_index = method->vtable_index(); 3769 assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index, 3770 "bad index %d", vtable_index); 3771 // Get the Method* out of the appropriate vtable entry. 3772 int entry_offset = in_bytes(Klass::vtable_start_offset()) + 3773 vtable_index*vtableEntry::size_in_bytes() + 3774 vtableEntry::method_offset_in_bytes(); 3775 Node* entry_addr = basic_plus_adr(obj_klass, entry_offset); 3776 Node* target_call = make_load(NULL, entry_addr, TypePtr::NOTNULL, T_ADDRESS, MemNode::unordered); 3777 3778 // Compare the target method with the expected method (e.g., Object.hashCode). 3779 const TypePtr* native_call_addr = TypeMetadataPtr::make(method); 3780 3781 Node* native_call = makecon(native_call_addr); 3782 Node* chk_native = _gvn.transform(new CmpPNode(target_call, native_call)); 3783 Node* test_native = _gvn.transform(new BoolNode(chk_native, BoolTest::ne)); 3784 3785 return generate_slow_guard(test_native, slow_region); 3786 } 3787 3788 //-----------------------generate_method_call---------------------------- 3789 // Use generate_method_call to make a slow-call to the real 3790 // method if the fast path fails. An alternative would be to 3791 // use a stub like OptoRuntime::slow_arraycopy_Java. 3792 // This only works for expanding the current library call, 3793 // not another intrinsic. (E.g., don't use this for making an 3794 // arraycopy call inside of the copyOf intrinsic.) 3795 CallJavaNode* 3796 LibraryCallKit::generate_method_call(vmIntrinsics::ID method_id, bool is_virtual, bool is_static) { 3797 // When compiling the intrinsic method itself, do not use this technique. 3798 guarantee(callee() != C->method(), "cannot make slow-call to self"); 3799 3800 ciMethod* method = callee(); 3801 // ensure the JVMS we have will be correct for this call 3802 guarantee(method_id == method->intrinsic_id(), "must match"); 3803 3804 const TypeFunc* tf = TypeFunc::make(method); 3805 CallJavaNode* slow_call; 3806 if (is_static) { 3807 assert(!is_virtual, ""); 3808 slow_call = new CallStaticJavaNode(C, tf, 3809 SharedRuntime::get_resolve_static_call_stub(), method); 3810 } else if (is_virtual) { 3811 null_check_receiver(); 3812 int vtable_index = Method::invalid_vtable_index; 3813 if (UseInlineCaches) { 3814 // Suppress the vtable call 3815 } else { 3816 // hashCode and clone are not a miranda methods, 3817 // so the vtable index is fixed. 3818 // No need to use the linkResolver to get it. 3819 vtable_index = method->vtable_index(); 3820 assert(vtable_index >= 0 || vtable_index == Method::nonvirtual_vtable_index, 3821 "bad index %d", vtable_index); 3822 } 3823 slow_call = new CallDynamicJavaNode(tf, 3824 SharedRuntime::get_resolve_virtual_call_stub(), 3825 method, vtable_index); 3826 } else { // neither virtual nor static: opt_virtual 3827 null_check_receiver(); 3828 slow_call = new CallStaticJavaNode(C, tf, 3829 SharedRuntime::get_resolve_opt_virtual_call_stub(), method); 3830 slow_call->set_optimized_virtual(true); 3831 } 3832 if (CallGenerator::is_inlined_method_handle_intrinsic(this->method(), bci(), callee())) { 3833 // To be able to issue a direct call (optimized virtual or virtual) 3834 // and skip a call to MH.linkTo*/invokeBasic adapter, additional information 3835 // about the method being invoked should be attached to the call site to 3836 // make resolution logic work (see SharedRuntime::resolve_{virtual,opt_virtual}_call_C). 3837 slow_call->set_override_symbolic_info(true); 3838 } 3839 set_arguments_for_java_call(slow_call); 3840 set_edges_for_java_call(slow_call); 3841 return slow_call; 3842 } 3843 3844 3845 /** 3846 * Build special case code for calls to hashCode on an object. This call may 3847 * be virtual (invokevirtual) or bound (invokespecial). For each case we generate 3848 * slightly different code. 3849 */ 3850 bool LibraryCallKit::inline_native_hashcode(bool is_virtual, bool is_static) { 3851 assert(is_static == callee()->is_static(), "correct intrinsic selection"); 3852 assert(!(is_virtual && is_static), "either virtual, special, or static"); 3853 3854 enum { _slow_path = 1, _fast_path, _null_path, PATH_LIMIT }; 3855 3856 RegionNode* result_reg = new RegionNode(PATH_LIMIT); 3857 PhiNode* result_val = new PhiNode(result_reg, TypeInt::INT); 3858 PhiNode* result_io = new PhiNode(result_reg, Type::ABIO); 3859 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM); 3860 Node* obj = NULL; 3861 if (!is_static) { 3862 // Check for hashing null object 3863 obj = null_check_receiver(); 3864 if (stopped()) return true; // unconditionally null 3865 result_reg->init_req(_null_path, top()); 3866 result_val->init_req(_null_path, top()); 3867 } else { 3868 // Do a null check, and return zero if null. 3869 // System.identityHashCode(null) == 0 3870 obj = argument(0); 3871 Node* null_ctl = top(); 3872 obj = null_check_oop(obj, &null_ctl); 3873 result_reg->init_req(_null_path, null_ctl); 3874 result_val->init_req(_null_path, _gvn.intcon(0)); 3875 } 3876 3877 // Unconditionally null? Then return right away. 3878 if (stopped()) { 3879 set_control( result_reg->in(_null_path)); 3880 if (!stopped()) 3881 set_result(result_val->in(_null_path)); 3882 return true; 3883 } 3884 3885 // We only go to the fast case code if we pass a number of guards. The 3886 // paths which do not pass are accumulated in the slow_region. 3887 RegionNode* slow_region = new RegionNode(1); 3888 record_for_igvn(slow_region); 3889 3890 // If this is a virtual call, we generate a funny guard. We pull out 3891 // the vtable entry corresponding to hashCode() from the target object. 3892 // If the target method which we are calling happens to be the native 3893 // Object hashCode() method, we pass the guard. We do not need this 3894 // guard for non-virtual calls -- the caller is known to be the native 3895 // Object hashCode(). 3896 if (is_virtual) { 3897 // After null check, get the object's klass. 3898 Node* obj_klass = load_object_klass(obj); 3899 generate_virtual_guard(obj_klass, slow_region); 3900 } 3901 3902 // Get the header out of the object, use LoadMarkNode when available 3903 Node* header_addr = basic_plus_adr(obj, oopDesc::mark_offset_in_bytes()); 3904 // The control of the load must be NULL. Otherwise, the load can move before 3905 // the null check after castPP removal. 3906 Node* no_ctrl = NULL; 3907 Node* header = make_load(no_ctrl, header_addr, TypeX_X, TypeX_X->basic_type(), MemNode::unordered); 3908 3909 // Test the header to see if it is unlocked. 3910 Node *lock_mask = _gvn.MakeConX(markWord::lock_mask_in_place); 3911 Node *lmasked_header = _gvn.transform(new AndXNode(header, lock_mask)); 3912 Node *unlocked_val = _gvn.MakeConX(markWord::unlocked_value); 3913 Node *chk_unlocked = _gvn.transform(new CmpXNode( lmasked_header, unlocked_val)); 3914 Node *test_unlocked = _gvn.transform(new BoolNode( chk_unlocked, BoolTest::ne)); 3915 3916 generate_slow_guard(test_unlocked, slow_region); 3917 3918 // Get the hash value and check to see that it has been properly assigned. 3919 // We depend on hash_mask being at most 32 bits and avoid the use of 3920 // hash_mask_in_place because it could be larger than 32 bits in a 64-bit 3921 // vm: see markWord.hpp. 3922 Node *hash_mask = _gvn.intcon(markWord::hash_mask); 3923 Node *hash_shift = _gvn.intcon(markWord::hash_shift); 3924 Node *hshifted_header= _gvn.transform(new URShiftXNode(header, hash_shift)); 3925 // This hack lets the hash bits live anywhere in the mark object now, as long 3926 // as the shift drops the relevant bits into the low 32 bits. Note that 3927 // Java spec says that HashCode is an int so there's no point in capturing 3928 // an 'X'-sized hashcode (32 in 32-bit build or 64 in 64-bit build). 3929 hshifted_header = ConvX2I(hshifted_header); 3930 Node *hash_val = _gvn.transform(new AndINode(hshifted_header, hash_mask)); 3931 3932 Node *no_hash_val = _gvn.intcon(markWord::no_hash); 3933 Node *chk_assigned = _gvn.transform(new CmpINode( hash_val, no_hash_val)); 3934 Node *test_assigned = _gvn.transform(new BoolNode( chk_assigned, BoolTest::eq)); 3935 3936 generate_slow_guard(test_assigned, slow_region); 3937 3938 Node* init_mem = reset_memory(); 3939 // fill in the rest of the null path: 3940 result_io ->init_req(_null_path, i_o()); 3941 result_mem->init_req(_null_path, init_mem); 3942 3943 result_val->init_req(_fast_path, hash_val); 3944 result_reg->init_req(_fast_path, control()); 3945 result_io ->init_req(_fast_path, i_o()); 3946 result_mem->init_req(_fast_path, init_mem); 3947 3948 // Generate code for the slow case. We make a call to hashCode(). 3949 set_control(_gvn.transform(slow_region)); 3950 if (!stopped()) { 3951 // No need for PreserveJVMState, because we're using up the present state. 3952 set_all_memory(init_mem); 3953 vmIntrinsics::ID hashCode_id = is_static ? vmIntrinsics::_identityHashCode : vmIntrinsics::_hashCode; 3954 CallJavaNode* slow_call = generate_method_call(hashCode_id, is_virtual, is_static); 3955 Node* slow_result = set_results_for_java_call(slow_call); 3956 // this->control() comes from set_results_for_java_call 3957 result_reg->init_req(_slow_path, control()); 3958 result_val->init_req(_slow_path, slow_result); 3959 result_io ->set_req(_slow_path, i_o()); 3960 result_mem ->set_req(_slow_path, reset_memory()); 3961 } 3962 3963 // Return the combined state. 3964 set_i_o( _gvn.transform(result_io) ); 3965 set_all_memory( _gvn.transform(result_mem)); 3966 3967 set_result(result_reg, result_val); 3968 return true; 3969 } 3970 3971 //---------------------------inline_native_getClass---------------------------- 3972 // public final native Class<?> java.lang.Object.getClass(); 3973 // 3974 // Build special case code for calls to getClass on an object. 3975 bool LibraryCallKit::inline_native_getClass() { 3976 Node* obj = null_check_receiver(); 3977 if (stopped()) return true; 3978 set_result(load_mirror_from_klass(load_object_klass(obj))); 3979 return true; 3980 } 3981 3982 //-----------------inline_native_Reflection_getCallerClass--------------------- 3983 // public static native Class<?> sun.reflect.Reflection.getCallerClass(); 3984 // 3985 // In the presence of deep enough inlining, getCallerClass() becomes a no-op. 3986 // 3987 // NOTE: This code must perform the same logic as JVM_GetCallerClass 3988 // in that it must skip particular security frames and checks for 3989 // caller sensitive methods. 3990 bool LibraryCallKit::inline_native_Reflection_getCallerClass() { 3991 #ifndef PRODUCT 3992 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 3993 tty->print_cr("Attempting to inline sun.reflect.Reflection.getCallerClass"); 3994 } 3995 #endif 3996 3997 if (!jvms()->has_method()) { 3998 #ifndef PRODUCT 3999 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 4000 tty->print_cr(" Bailing out because intrinsic was inlined at top level"); 4001 } 4002 #endif 4003 return false; 4004 } 4005 4006 // Walk back up the JVM state to find the caller at the required 4007 // depth. 4008 JVMState* caller_jvms = jvms(); 4009 4010 // Cf. JVM_GetCallerClass 4011 // NOTE: Start the loop at depth 1 because the current JVM state does 4012 // not include the Reflection.getCallerClass() frame. 4013 for (int n = 1; caller_jvms != NULL; caller_jvms = caller_jvms->caller(), n++) { 4014 ciMethod* m = caller_jvms->method(); 4015 switch (n) { 4016 case 0: 4017 fatal("current JVM state does not include the Reflection.getCallerClass frame"); 4018 break; 4019 case 1: 4020 // Frame 0 and 1 must be caller sensitive (see JVM_GetCallerClass). 4021 if (!m->caller_sensitive()) { 4022 #ifndef PRODUCT 4023 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 4024 tty->print_cr(" Bailing out: CallerSensitive annotation expected at frame %d", n); 4025 } 4026 #endif 4027 return false; // bail-out; let JVM_GetCallerClass do the work 4028 } 4029 break; 4030 default: 4031 if (!m->is_ignored_by_security_stack_walk()) { 4032 // We have reached the desired frame; return the holder class. 4033 // Acquire method holder as java.lang.Class and push as constant. 4034 ciInstanceKlass* caller_klass = caller_jvms->method()->holder(); 4035 ciInstance* caller_mirror = caller_klass->java_mirror(); 4036 set_result(makecon(TypeInstPtr::make(caller_mirror))); 4037 4038 #ifndef PRODUCT 4039 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 4040 tty->print_cr(" Succeeded: caller = %d) %s.%s, JVMS depth = %d", n, caller_klass->name()->as_utf8(), caller_jvms->method()->name()->as_utf8(), jvms()->depth()); 4041 tty->print_cr(" JVM state at this point:"); 4042 for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) { 4043 ciMethod* m = jvms()->of_depth(i)->method(); 4044 tty->print_cr(" %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8()); 4045 } 4046 } 4047 #endif 4048 return true; 4049 } 4050 break; 4051 } 4052 } 4053 4054 #ifndef PRODUCT 4055 if ((C->print_intrinsics() || C->print_inlining()) && Verbose) { 4056 tty->print_cr(" Bailing out because caller depth exceeded inlining depth = %d", jvms()->depth()); 4057 tty->print_cr(" JVM state at this point:"); 4058 for (int i = jvms()->depth(), n = 1; i >= 1; i--, n++) { 4059 ciMethod* m = jvms()->of_depth(i)->method(); 4060 tty->print_cr(" %d) %s.%s", n, m->holder()->name()->as_utf8(), m->name()->as_utf8()); 4061 } 4062 } 4063 #endif 4064 4065 return false; // bail-out; let JVM_GetCallerClass do the work 4066 } 4067 4068 bool LibraryCallKit::inline_fp_conversions(vmIntrinsics::ID id) { 4069 Node* arg = argument(0); 4070 Node* result = NULL; 4071 4072 switch (id) { 4073 case vmIntrinsics::_floatToRawIntBits: result = new MoveF2INode(arg); break; 4074 case vmIntrinsics::_intBitsToFloat: result = new MoveI2FNode(arg); break; 4075 case vmIntrinsics::_doubleToRawLongBits: result = new MoveD2LNode(arg); break; 4076 case vmIntrinsics::_longBitsToDouble: result = new MoveL2DNode(arg); break; 4077 4078 case vmIntrinsics::_doubleToLongBits: { 4079 // two paths (plus control) merge in a wood 4080 RegionNode *r = new RegionNode(3); 4081 Node *phi = new PhiNode(r, TypeLong::LONG); 4082 4083 Node *cmpisnan = _gvn.transform(new CmpDNode(arg, arg)); 4084 // Build the boolean node 4085 Node *bolisnan = _gvn.transform(new BoolNode(cmpisnan, BoolTest::ne)); 4086 4087 // Branch either way. 4088 // NaN case is less traveled, which makes all the difference. 4089 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); 4090 Node *opt_isnan = _gvn.transform(ifisnan); 4091 assert( opt_isnan->is_If(), "Expect an IfNode"); 4092 IfNode *opt_ifisnan = (IfNode*)opt_isnan; 4093 Node *iftrue = _gvn.transform(new IfTrueNode(opt_ifisnan)); 4094 4095 set_control(iftrue); 4096 4097 static const jlong nan_bits = CONST64(0x7ff8000000000000); 4098 Node *slow_result = longcon(nan_bits); // return NaN 4099 phi->init_req(1, _gvn.transform( slow_result )); 4100 r->init_req(1, iftrue); 4101 4102 // Else fall through 4103 Node *iffalse = _gvn.transform(new IfFalseNode(opt_ifisnan)); 4104 set_control(iffalse); 4105 4106 phi->init_req(2, _gvn.transform(new MoveD2LNode(arg))); 4107 r->init_req(2, iffalse); 4108 4109 // Post merge 4110 set_control(_gvn.transform(r)); 4111 record_for_igvn(r); 4112 4113 C->set_has_split_ifs(true); // Has chance for split-if optimization 4114 result = phi; 4115 assert(result->bottom_type()->isa_long(), "must be"); 4116 break; 4117 } 4118 4119 case vmIntrinsics::_floatToIntBits: { 4120 // two paths (plus control) merge in a wood 4121 RegionNode *r = new RegionNode(3); 4122 Node *phi = new PhiNode(r, TypeInt::INT); 4123 4124 Node *cmpisnan = _gvn.transform(new CmpFNode(arg, arg)); 4125 // Build the boolean node 4126 Node *bolisnan = _gvn.transform(new BoolNode(cmpisnan, BoolTest::ne)); 4127 4128 // Branch either way. 4129 // NaN case is less traveled, which makes all the difference. 4130 IfNode *ifisnan = create_and_xform_if(control(), bolisnan, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); 4131 Node *opt_isnan = _gvn.transform(ifisnan); 4132 assert( opt_isnan->is_If(), "Expect an IfNode"); 4133 IfNode *opt_ifisnan = (IfNode*)opt_isnan; 4134 Node *iftrue = _gvn.transform(new IfTrueNode(opt_ifisnan)); 4135 4136 set_control(iftrue); 4137 4138 static const jint nan_bits = 0x7fc00000; 4139 Node *slow_result = makecon(TypeInt::make(nan_bits)); // return NaN 4140 phi->init_req(1, _gvn.transform( slow_result )); 4141 r->init_req(1, iftrue); 4142 4143 // Else fall through 4144 Node *iffalse = _gvn.transform(new IfFalseNode(opt_ifisnan)); 4145 set_control(iffalse); 4146 4147 phi->init_req(2, _gvn.transform(new MoveF2INode(arg))); 4148 r->init_req(2, iffalse); 4149 4150 // Post merge 4151 set_control(_gvn.transform(r)); 4152 record_for_igvn(r); 4153 4154 C->set_has_split_ifs(true); // Has chance for split-if optimization 4155 result = phi; 4156 assert(result->bottom_type()->isa_int(), "must be"); 4157 break; 4158 } 4159 4160 default: 4161 fatal_unexpected_iid(id); 4162 break; 4163 } 4164 set_result(_gvn.transform(result)); 4165 return true; 4166 } 4167 4168 //----------------------inline_unsafe_copyMemory------------------------- 4169 // public native void Unsafe.copyMemory0(Object srcBase, long srcOffset, Object destBase, long destOffset, long bytes); 4170 4171 static bool has_wide_mem(PhaseGVN& gvn, Node* addr, Node* base) { 4172 const TypeAryPtr* addr_t = gvn.type(addr)->isa_aryptr(); 4173 const Type* base_t = gvn.type(base); 4174 4175 bool in_native = (base_t == TypePtr::NULL_PTR); 4176 bool in_heap = !TypePtr::NULL_PTR->higher_equal(base_t); 4177 bool is_mixed = !in_heap && !in_native; 4178 4179 if (is_mixed) { 4180 return true; // mixed accesses can touch both on-heap and off-heap memory 4181 } 4182 if (in_heap) { 4183 bool is_prim_array = (addr_t != NULL) && (addr_t->elem() != Type::BOTTOM); 4184 if (!is_prim_array) { 4185 // Though Unsafe.copyMemory() ensures at runtime for on-heap accesses that base is a primitive array, 4186 // there's not enough type information available to determine proper memory slice for it. 4187 return true; 4188 } 4189 } 4190 return false; 4191 } 4192 4193 bool LibraryCallKit::inline_unsafe_copyMemory() { 4194 if (callee()->is_static()) return false; // caller must have the capability! 4195 null_check_receiver(); // null-check receiver 4196 if (stopped()) return true; 4197 4198 C->set_has_unsafe_access(true); // Mark eventual nmethod as "unsafe". 4199 4200 Node* src_base = argument(1); // type: oop 4201 Node* src_off = ConvL2X(argument(2)); // type: long 4202 Node* dst_base = argument(4); // type: oop 4203 Node* dst_off = ConvL2X(argument(5)); // type: long 4204 Node* size = ConvL2X(argument(7)); // type: long 4205 4206 assert(Unsafe_field_offset_to_byte_offset(11) == 11, 4207 "fieldOffset must be byte-scaled"); 4208 4209 Node* src_addr = make_unsafe_address(src_base, src_off); 4210 Node* dst_addr = make_unsafe_address(dst_base, dst_off); 4211 4212 Node* thread = _gvn.transform(new ThreadLocalNode()); 4213 Node* doing_unsafe_access_addr = basic_plus_adr(top(), thread, in_bytes(JavaThread::doing_unsafe_access_offset())); 4214 BasicType doing_unsafe_access_bt = T_BYTE; 4215 assert((sizeof(bool) * CHAR_BIT) == 8, "not implemented"); 4216 4217 // update volatile field 4218 store_to_memory(control(), doing_unsafe_access_addr, intcon(1), doing_unsafe_access_bt, Compile::AliasIdxRaw, MemNode::unordered); 4219 4220 int flags = RC_LEAF | RC_NO_FP; 4221 4222 const TypePtr* dst_type = TypePtr::BOTTOM; 4223 4224 // Adjust memory effects of the runtime call based on input values. 4225 if (!has_wide_mem(_gvn, src_addr, src_base) && 4226 !has_wide_mem(_gvn, dst_addr, dst_base)) { 4227 dst_type = _gvn.type(dst_addr)->is_ptr(); // narrow out memory 4228 4229 const TypePtr* src_type = _gvn.type(src_addr)->is_ptr(); 4230 if (C->get_alias_index(src_type) == C->get_alias_index(dst_type)) { 4231 flags |= RC_NARROW_MEM; // narrow in memory 4232 } 4233 } 4234 4235 // Call it. Note that the length argument is not scaled. 4236 make_runtime_call(flags, 4237 OptoRuntime::fast_arraycopy_Type(), 4238 StubRoutines::unsafe_arraycopy(), 4239 "unsafe_arraycopy", 4240 dst_type, 4241 src_addr, dst_addr, size XTOP); 4242 4243 store_to_memory(control(), doing_unsafe_access_addr, intcon(0), doing_unsafe_access_bt, Compile::AliasIdxRaw, MemNode::unordered); 4244 4245 return true; 4246 } 4247 4248 #undef XTOP 4249 4250 //------------------------clone_coping----------------------------------- 4251 // Helper function for inline_native_clone. 4252 void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array) { 4253 assert(obj_size != NULL, ""); 4254 Node* raw_obj = alloc_obj->in(1); 4255 assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); 4256 4257 AllocateNode* alloc = NULL; 4258 if (ReduceBulkZeroing) { 4259 // We will be completely responsible for initializing this object - 4260 // mark Initialize node as complete. 4261 alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); 4262 // The object was just allocated - there should be no any stores! 4263 guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), ""); 4264 // Mark as complete_with_arraycopy so that on AllocateNode 4265 // expansion, we know this AllocateNode is initialized by an array 4266 // copy and a StoreStore barrier exists after the array copy. 4267 alloc->initialization()->set_complete_with_arraycopy(); 4268 } 4269 4270 Node* size = _gvn.transform(obj_size); 4271 access_clone(obj, alloc_obj, size, is_array); 4272 4273 // Do not let reads from the cloned object float above the arraycopy. 4274 if (alloc != NULL) { 4275 // Do not let stores that initialize this object be reordered with 4276 // a subsequent store that would make this object accessible by 4277 // other threads. 4278 // Record what AllocateNode this StoreStore protects so that 4279 // escape analysis can go from the MemBarStoreStoreNode to the 4280 // AllocateNode and eliminate the MemBarStoreStoreNode if possible 4281 // based on the escape status of the AllocateNode. 4282 insert_mem_bar(Op_MemBarStoreStore, alloc->proj_out_or_null(AllocateNode::RawAddress)); 4283 } else { 4284 insert_mem_bar(Op_MemBarCPUOrder); 4285 } 4286 } 4287 4288 //------------------------inline_native_clone---------------------------- 4289 // protected native Object java.lang.Object.clone(); 4290 // 4291 // Here are the simple edge cases: 4292 // null receiver => normal trap 4293 // virtual and clone was overridden => slow path to out-of-line clone 4294 // not cloneable or finalizer => slow path to out-of-line Object.clone 4295 // 4296 // The general case has two steps, allocation and copying. 4297 // Allocation has two cases, and uses GraphKit::new_instance or new_array. 4298 // 4299 // Copying also has two cases, oop arrays and everything else. 4300 // Oop arrays use arrayof_oop_arraycopy (same as System.arraycopy). 4301 // Everything else uses the tight inline loop supplied by CopyArrayNode. 4302 // 4303 // These steps fold up nicely if and when the cloned object's klass 4304 // can be sharply typed as an object array, a type array, or an instance. 4305 // 4306 bool LibraryCallKit::inline_native_clone(bool is_virtual) { 4307 PhiNode* result_val; 4308 4309 // Set the reexecute bit for the interpreter to reexecute 4310 // the bytecode that invokes Object.clone if deoptimization happens. 4311 { PreserveReexecuteState preexecs(this); 4312 jvms()->set_should_reexecute(true); 4313 4314 Node* obj = null_check_receiver(); 4315 if (stopped()) return true; 4316 4317 const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr(); 4318 4319 // If we are going to clone an instance, we need its exact type to 4320 // know the number and types of fields to convert the clone to 4321 // loads/stores. Maybe a speculative type can help us. 4322 if (!obj_type->klass_is_exact() && 4323 obj_type->speculative_type() != NULL && 4324 obj_type->speculative_type()->is_instance_klass()) { 4325 ciInstanceKlass* spec_ik = obj_type->speculative_type()->as_instance_klass(); 4326 if (spec_ik->nof_nonstatic_fields() <= ArrayCopyLoadStoreMaxElem && 4327 !spec_ik->has_injected_fields()) { 4328 ciKlass* k = obj_type->klass(); 4329 if (!k->is_instance_klass() || 4330 k->as_instance_klass()->is_interface() || 4331 k->as_instance_klass()->has_subklass()) { 4332 obj = maybe_cast_profiled_obj(obj, obj_type->speculative_type(), false); 4333 } 4334 } 4335 } 4336 4337 // Conservatively insert a memory barrier on all memory slices. 4338 // Do not let writes into the original float below the clone. 4339 insert_mem_bar(Op_MemBarCPUOrder); 4340 4341 // paths into result_reg: 4342 enum { 4343 _slow_path = 1, // out-of-line call to clone method (virtual or not) 4344 _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy 4345 _array_path, // plain array allocation, plus arrayof_long_arraycopy 4346 _instance_path, // plain instance allocation, plus arrayof_long_arraycopy 4347 PATH_LIMIT 4348 }; 4349 RegionNode* result_reg = new RegionNode(PATH_LIMIT); 4350 result_val = new PhiNode(result_reg, TypeInstPtr::NOTNULL); 4351 PhiNode* result_i_o = new PhiNode(result_reg, Type::ABIO); 4352 PhiNode* result_mem = new PhiNode(result_reg, Type::MEMORY, TypePtr::BOTTOM); 4353 record_for_igvn(result_reg); 4354 4355 Node* obj_klass = load_object_klass(obj); 4356 Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL); 4357 if (array_ctl != NULL) { 4358 // It's an array. 4359 PreserveJVMState pjvms(this); 4360 set_control(array_ctl); 4361 Node* obj_length = load_array_length(obj); 4362 Node* obj_size = NULL; 4363 Node* alloc_obj = new_array(obj_klass, obj_length, 0, &obj_size, /*deoptimize_on_exception=*/true); 4364 4365 BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); 4366 if (bs->array_copy_requires_gc_barriers(true, T_OBJECT, true, false, BarrierSetC2::Parsing)) { 4367 // If it is an oop array, it requires very special treatment, 4368 // because gc barriers are required when accessing the array. 4369 Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL); 4370 if (is_obja != NULL) { 4371 PreserveJVMState pjvms2(this); 4372 set_control(is_obja); 4373 // Generate a direct call to the right arraycopy function(s). 4374 // Clones are always tightly coupled. 4375 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, obj, intcon(0), alloc_obj, intcon(0), obj_length, true, false); 4376 ac->set_clone_oop_array(); 4377 Node* n = _gvn.transform(ac); 4378 assert(n == ac, "cannot disappear"); 4379 ac->connect_outputs(this, /*deoptimize_on_exception=*/true); 4380 4381 result_reg->init_req(_objArray_path, control()); 4382 result_val->init_req(_objArray_path, alloc_obj); 4383 result_i_o ->set_req(_objArray_path, i_o()); 4384 result_mem ->set_req(_objArray_path, reset_memory()); 4385 } 4386 } 4387 // Otherwise, there are no barriers to worry about. 4388 // (We can dispense with card marks if we know the allocation 4389 // comes out of eden (TLAB)... In fact, ReduceInitialCardMarks 4390 // causes the non-eden paths to take compensating steps to 4391 // simulate a fresh allocation, so that no further 4392 // card marks are required in compiled code to initialize 4393 // the object.) 4394 4395 if (!stopped()) { 4396 copy_to_clone(obj, alloc_obj, obj_size, true); 4397 4398 // Present the results of the copy. 4399 result_reg->init_req(_array_path, control()); 4400 result_val->init_req(_array_path, alloc_obj); 4401 result_i_o ->set_req(_array_path, i_o()); 4402 result_mem ->set_req(_array_path, reset_memory()); 4403 } 4404 } 4405 4406 // We only go to the instance fast case code if we pass a number of guards. 4407 // The paths which do not pass are accumulated in the slow_region. 4408 RegionNode* slow_region = new RegionNode(1); 4409 record_for_igvn(slow_region); 4410 if (!stopped()) { 4411 // It's an instance (we did array above). Make the slow-path tests. 4412 // If this is a virtual call, we generate a funny guard. We grab 4413 // the vtable entry corresponding to clone() from the target object. 4414 // If the target method which we are calling happens to be the 4415 // Object clone() method, we pass the guard. We do not need this 4416 // guard for non-virtual calls; the caller is known to be the native 4417 // Object clone(). 4418 if (is_virtual) { 4419 generate_virtual_guard(obj_klass, slow_region); 4420 } 4421 4422 // The object must be easily cloneable and must not have a finalizer. 4423 // Both of these conditions may be checked in a single test. 4424 // We could optimize the test further, but we don't care. 4425 generate_access_flags_guard(obj_klass, 4426 // Test both conditions: 4427 JVM_ACC_IS_CLONEABLE_FAST | JVM_ACC_HAS_FINALIZER, 4428 // Must be cloneable but not finalizer: 4429 JVM_ACC_IS_CLONEABLE_FAST, 4430 slow_region); 4431 } 4432 4433 if (!stopped()) { 4434 // It's an instance, and it passed the slow-path tests. 4435 PreserveJVMState pjvms(this); 4436 Node* obj_size = NULL; 4437 // Need to deoptimize on exception from allocation since Object.clone intrinsic 4438 // is reexecuted if deoptimization occurs and there could be problems when merging 4439 // exception state between multiple Object.clone versions (reexecute=true vs reexecute=false). 4440 Node* alloc_obj = new_instance(obj_klass, NULL, &obj_size, /*deoptimize_on_exception=*/true); 4441 4442 copy_to_clone(obj, alloc_obj, obj_size, false); 4443 4444 // Present the results of the slow call. 4445 result_reg->init_req(_instance_path, control()); 4446 result_val->init_req(_instance_path, alloc_obj); 4447 result_i_o ->set_req(_instance_path, i_o()); 4448 result_mem ->set_req(_instance_path, reset_memory()); 4449 } 4450 4451 // Generate code for the slow case. We make a call to clone(). 4452 set_control(_gvn.transform(slow_region)); 4453 if (!stopped()) { 4454 PreserveJVMState pjvms(this); 4455 CallJavaNode* slow_call = generate_method_call(vmIntrinsics::_clone, is_virtual); 4456 // We need to deoptimize on exception (see comment above) 4457 Node* slow_result = set_results_for_java_call(slow_call, false, /* deoptimize */ true); 4458 // this->control() comes from set_results_for_java_call 4459 result_reg->init_req(_slow_path, control()); 4460 result_val->init_req(_slow_path, slow_result); 4461 result_i_o ->set_req(_slow_path, i_o()); 4462 result_mem ->set_req(_slow_path, reset_memory()); 4463 } 4464 4465 // Return the combined state. 4466 set_control( _gvn.transform(result_reg)); 4467 set_i_o( _gvn.transform(result_i_o)); 4468 set_all_memory( _gvn.transform(result_mem)); 4469 } // original reexecute is set back here 4470 4471 set_result(_gvn.transform(result_val)); 4472 return true; 4473 } 4474 4475 // If we have a tightly coupled allocation, the arraycopy may take care 4476 // of the array initialization. If one of the guards we insert between 4477 // the allocation and the arraycopy causes a deoptimization, an 4478 // uninitialized array will escape the compiled method. To prevent that 4479 // we set the JVM state for uncommon traps between the allocation and 4480 // the arraycopy to the state before the allocation so, in case of 4481 // deoptimization, we'll reexecute the allocation and the 4482 // initialization. 4483 JVMState* LibraryCallKit::arraycopy_restore_alloc_state(AllocateArrayNode* alloc, int& saved_reexecute_sp) { 4484 if (alloc != NULL) { 4485 ciMethod* trap_method = alloc->jvms()->method(); 4486 int trap_bci = alloc->jvms()->bci(); 4487 4488 if (!C->too_many_traps(trap_method, trap_bci, Deoptimization::Reason_intrinsic) && 4489 !C->too_many_traps(trap_method, trap_bci, Deoptimization::Reason_null_check)) { 4490 // Make sure there's no store between the allocation and the 4491 // arraycopy otherwise visible side effects could be rexecuted 4492 // in case of deoptimization and cause incorrect execution. 4493 bool no_interfering_store = true; 4494 Node* mem = alloc->in(TypeFunc::Memory); 4495 if (mem->is_MergeMem()) { 4496 for (MergeMemStream mms(merged_memory(), mem->as_MergeMem()); mms.next_non_empty2(); ) { 4497 Node* n = mms.memory(); 4498 if (n != mms.memory2() && !(n->is_Proj() && n->in(0) == alloc->initialization())) { 4499 assert(n->is_Store(), "what else?"); 4500 no_interfering_store = false; 4501 break; 4502 } 4503 } 4504 } else { 4505 for (MergeMemStream mms(merged_memory()); mms.next_non_empty(); ) { 4506 Node* n = mms.memory(); 4507 if (n != mem && !(n->is_Proj() && n->in(0) == alloc->initialization())) { 4508 assert(n->is_Store(), "what else?"); 4509 no_interfering_store = false; 4510 break; 4511 } 4512 } 4513 } 4514 4515 if (no_interfering_store) { 4516 JVMState* old_jvms = alloc->jvms()->clone_shallow(C); 4517 uint size = alloc->req(); 4518 SafePointNode* sfpt = new SafePointNode(size, old_jvms); 4519 old_jvms->set_map(sfpt); 4520 for (uint i = 0; i < size; i++) { 4521 sfpt->init_req(i, alloc->in(i)); 4522 } 4523 // re-push array length for deoptimization 4524 sfpt->ins_req(old_jvms->stkoff() + old_jvms->sp(), alloc->in(AllocateNode::ALength)); 4525 old_jvms->set_sp(old_jvms->sp()+1); 4526 old_jvms->set_monoff(old_jvms->monoff()+1); 4527 old_jvms->set_scloff(old_jvms->scloff()+1); 4528 old_jvms->set_endoff(old_jvms->endoff()+1); 4529 old_jvms->set_should_reexecute(true); 4530 4531 sfpt->set_i_o(map()->i_o()); 4532 sfpt->set_memory(map()->memory()); 4533 sfpt->set_control(map()->control()); 4534 4535 JVMState* saved_jvms = jvms(); 4536 saved_reexecute_sp = _reexecute_sp; 4537 4538 set_jvms(sfpt->jvms()); 4539 _reexecute_sp = jvms()->sp(); 4540 4541 return saved_jvms; 4542 } 4543 } 4544 } 4545 return NULL; 4546 } 4547 4548 // In case of a deoptimization, we restart execution at the 4549 // allocation, allocating a new array. We would leave an uninitialized 4550 // array in the heap that GCs wouldn't expect. Move the allocation 4551 // after the traps so we don't allocate the array if we 4552 // deoptimize. This is possible because tightly_coupled_allocation() 4553 // guarantees there's no observer of the allocated array at this point 4554 // and the control flow is simple enough. 4555 void LibraryCallKit::arraycopy_move_allocation_here(AllocateArrayNode* alloc, Node* dest, JVMState* saved_jvms, 4556 int saved_reexecute_sp, uint new_idx) { 4557 if (saved_jvms != NULL && !stopped()) { 4558 assert(alloc != NULL, "only with a tightly coupled allocation"); 4559 // restore JVM state to the state at the arraycopy 4560 saved_jvms->map()->set_control(map()->control()); 4561 assert(saved_jvms->map()->memory() == map()->memory(), "memory state changed?"); 4562 assert(saved_jvms->map()->i_o() == map()->i_o(), "IO state changed?"); 4563 // If we've improved the types of some nodes (null check) while 4564 // emitting the guards, propagate them to the current state 4565 map()->replaced_nodes().apply(saved_jvms->map(), new_idx); 4566 set_jvms(saved_jvms); 4567 _reexecute_sp = saved_reexecute_sp; 4568 4569 // Remove the allocation from above the guards 4570 CallProjections callprojs; 4571 alloc->extract_projections(&callprojs, true); 4572 InitializeNode* init = alloc->initialization(); 4573 Node* alloc_mem = alloc->in(TypeFunc::Memory); 4574 C->gvn_replace_by(callprojs.fallthrough_ioproj, alloc->in(TypeFunc::I_O)); 4575 C->gvn_replace_by(init->proj_out(TypeFunc::Memory), alloc_mem); 4576 4577 // The CastIINode created in GraphKit::new_array (in AllocateArrayNode::make_ideal_length) must stay below 4578 // the allocation (i.e. is only valid if the allocation succeeds): 4579 // 1) replace CastIINode with AllocateArrayNode's length here 4580 // 2) Create CastIINode again once allocation has moved (see below) at the end of this method 4581 // 4582 // Multiple identical CastIINodes might exist here. Each GraphKit::load_array_length() call will generate 4583 // new separate CastIINode (arraycopy guard checks or any array length use between array allocation and ararycopy) 4584 Node* init_control = init->proj_out(TypeFunc::Control); 4585 Node* alloc_length = alloc->Ideal_length(); 4586 #ifdef ASSERT 4587 Node* prev_cast = NULL; 4588 #endif 4589 for (uint i = 0; i < init_control->outcnt(); i++) { 4590 Node* init_out = init_control->raw_out(i); 4591 if (init_out->is_CastII() && init_out->in(TypeFunc::Control) == init_control && init_out->in(1) == alloc_length) { 4592 #ifdef ASSERT 4593 if (prev_cast == NULL) { 4594 prev_cast = init_out; 4595 } else { 4596 if (prev_cast->cmp(*init_out) == false) { 4597 prev_cast->dump(); 4598 init_out->dump(); 4599 assert(false, "not equal CastIINode"); 4600 } 4601 } 4602 #endif 4603 C->gvn_replace_by(init_out, alloc_length); 4604 } 4605 } 4606 C->gvn_replace_by(init->proj_out(TypeFunc::Control), alloc->in(0)); 4607 4608 // move the allocation here (after the guards) 4609 _gvn.hash_delete(alloc); 4610 alloc->set_req(TypeFunc::Control, control()); 4611 alloc->set_req(TypeFunc::I_O, i_o()); 4612 Node *mem = reset_memory(); 4613 set_all_memory(mem); 4614 alloc->set_req(TypeFunc::Memory, mem); 4615 set_control(init->proj_out_or_null(TypeFunc::Control)); 4616 set_i_o(callprojs.fallthrough_ioproj); 4617 4618 // Update memory as done in GraphKit::set_output_for_allocation() 4619 const TypeInt* length_type = _gvn.find_int_type(alloc->in(AllocateNode::ALength)); 4620 const TypeOopPtr* ary_type = _gvn.type(alloc->in(AllocateNode::KlassNode))->is_klassptr()->as_instance_type(); 4621 if (ary_type->isa_aryptr() && length_type != NULL) { 4622 ary_type = ary_type->is_aryptr()->cast_to_size(length_type); 4623 } 4624 const TypePtr* telemref = ary_type->add_offset(Type::OffsetBot); 4625 int elemidx = C->get_alias_index(telemref); 4626 set_memory(init->proj_out_or_null(TypeFunc::Memory), Compile::AliasIdxRaw); 4627 set_memory(init->proj_out_or_null(TypeFunc::Memory), elemidx); 4628 4629 Node* allocx = _gvn.transform(alloc); 4630 assert(allocx == alloc, "where has the allocation gone?"); 4631 assert(dest->is_CheckCastPP(), "not an allocation result?"); 4632 4633 _gvn.hash_delete(dest); 4634 dest->set_req(0, control()); 4635 Node* destx = _gvn.transform(dest); 4636 assert(destx == dest, "where has the allocation result gone?"); 4637 4638 array_ideal_length(alloc, ary_type, true); 4639 } 4640 } 4641 4642 4643 //------------------------------inline_arraycopy----------------------- 4644 // public static native void java.lang.System.arraycopy(Object src, int srcPos, 4645 // Object dest, int destPos, 4646 // int length); 4647 bool LibraryCallKit::inline_arraycopy() { 4648 // Get the arguments. 4649 Node* src = argument(0); // type: oop 4650 Node* src_offset = argument(1); // type: int 4651 Node* dest = argument(2); // type: oop 4652 Node* dest_offset = argument(3); // type: int 4653 Node* length = argument(4); // type: int 4654 4655 uint new_idx = C->unique(); 4656 4657 // Check for allocation before we add nodes that would confuse 4658 // tightly_coupled_allocation() 4659 AllocateArrayNode* alloc = tightly_coupled_allocation(dest); 4660 4661 int saved_reexecute_sp = -1; 4662 JVMState* saved_jvms = arraycopy_restore_alloc_state(alloc, saved_reexecute_sp); 4663 // See arraycopy_restore_alloc_state() comment 4664 // if alloc == NULL we don't have to worry about a tightly coupled allocation so we can emit all needed guards 4665 // if saved_jvms != NULL (then alloc != NULL) then we can handle guards and a tightly coupled allocation 4666 // if saved_jvms == NULL and alloc != NULL, we can't emit any guards 4667 bool can_emit_guards = (alloc == NULL || saved_jvms != NULL); 4668 4669 // The following tests must be performed 4670 // (1) src and dest are arrays. 4671 // (2) src and dest arrays must have elements of the same BasicType 4672 // (3) src and dest must not be null. 4673 // (4) src_offset must not be negative. 4674 // (5) dest_offset must not be negative. 4675 // (6) length must not be negative. 4676 // (7) src_offset + length must not exceed length of src. 4677 // (8) dest_offset + length must not exceed length of dest. 4678 // (9) each element of an oop array must be assignable 4679 4680 // (3) src and dest must not be null. 4681 // always do this here because we need the JVM state for uncommon traps 4682 Node* null_ctl = top(); 4683 src = saved_jvms != NULL ? null_check_oop(src, &null_ctl, true, true) : null_check(src, T_ARRAY); 4684 assert(null_ctl->is_top(), "no null control here"); 4685 dest = null_check(dest, T_ARRAY); 4686 4687 if (!can_emit_guards) { 4688 // if saved_jvms == NULL and alloc != NULL, we don't emit any 4689 // guards but the arraycopy node could still take advantage of a 4690 // tightly allocated allocation. tightly_coupled_allocation() is 4691 // called again to make sure it takes the null check above into 4692 // account: the null check is mandatory and if it caused an 4693 // uncommon trap to be emitted then the allocation can't be 4694 // considered tightly coupled in this context. 4695 alloc = tightly_coupled_allocation(dest); 4696 } 4697 4698 bool validated = false; 4699 4700 const Type* src_type = _gvn.type(src); 4701 const Type* dest_type = _gvn.type(dest); 4702 const TypeAryPtr* top_src = src_type->isa_aryptr(); 4703 const TypeAryPtr* top_dest = dest_type->isa_aryptr(); 4704 4705 // Do we have the type of src? 4706 bool has_src = (top_src != NULL && top_src->klass() != NULL); 4707 // Do we have the type of dest? 4708 bool has_dest = (top_dest != NULL && top_dest->klass() != NULL); 4709 // Is the type for src from speculation? 4710 bool src_spec = false; 4711 // Is the type for dest from speculation? 4712 bool dest_spec = false; 4713 4714 if ((!has_src || !has_dest) && can_emit_guards) { 4715 // We don't have sufficient type information, let's see if 4716 // speculative types can help. We need to have types for both src 4717 // and dest so that it pays off. 4718 4719 // Do we already have or could we have type information for src 4720 bool could_have_src = has_src; 4721 // Do we already have or could we have type information for dest 4722 bool could_have_dest = has_dest; 4723 4724 ciKlass* src_k = NULL; 4725 if (!has_src) { 4726 src_k = src_type->speculative_type_not_null(); 4727 if (src_k != NULL && src_k->is_array_klass()) { 4728 could_have_src = true; 4729 } 4730 } 4731 4732 ciKlass* dest_k = NULL; 4733 if (!has_dest) { 4734 dest_k = dest_type->speculative_type_not_null(); 4735 if (dest_k != NULL && dest_k->is_array_klass()) { 4736 could_have_dest = true; 4737 } 4738 } 4739 4740 if (could_have_src && could_have_dest) { 4741 // This is going to pay off so emit the required guards 4742 if (!has_src) { 4743 src = maybe_cast_profiled_obj(src, src_k, true); 4744 src_type = _gvn.type(src); 4745 top_src = src_type->isa_aryptr(); 4746 has_src = (top_src != NULL && top_src->klass() != NULL); 4747 src_spec = true; 4748 } 4749 if (!has_dest) { 4750 dest = maybe_cast_profiled_obj(dest, dest_k, true); 4751 dest_type = _gvn.type(dest); 4752 top_dest = dest_type->isa_aryptr(); 4753 has_dest = (top_dest != NULL && top_dest->klass() != NULL); 4754 dest_spec = true; 4755 } 4756 } 4757 } 4758 4759 if (has_src && has_dest && can_emit_guards) { 4760 BasicType src_elem = top_src->klass()->as_array_klass()->element_type()->basic_type(); 4761 BasicType dest_elem = top_dest->klass()->as_array_klass()->element_type()->basic_type(); 4762 if (is_reference_type(src_elem)) src_elem = T_OBJECT; 4763 if (is_reference_type(dest_elem)) dest_elem = T_OBJECT; 4764 4765 if (src_elem == dest_elem && src_elem == T_OBJECT) { 4766 // If both arrays are object arrays then having the exact types 4767 // for both will remove the need for a subtype check at runtime 4768 // before the call and may make it possible to pick a faster copy 4769 // routine (without a subtype check on every element) 4770 // Do we have the exact type of src? 4771 bool could_have_src = src_spec; 4772 // Do we have the exact type of dest? 4773 bool could_have_dest = dest_spec; 4774 ciKlass* src_k = top_src->klass(); 4775 ciKlass* dest_k = top_dest->klass(); 4776 if (!src_spec) { 4777 src_k = src_type->speculative_type_not_null(); 4778 if (src_k != NULL && src_k->is_array_klass()) { 4779 could_have_src = true; 4780 } 4781 } 4782 if (!dest_spec) { 4783 dest_k = dest_type->speculative_type_not_null(); 4784 if (dest_k != NULL && dest_k->is_array_klass()) { 4785 could_have_dest = true; 4786 } 4787 } 4788 if (could_have_src && could_have_dest) { 4789 // If we can have both exact types, emit the missing guards 4790 if (could_have_src && !src_spec) { 4791 src = maybe_cast_profiled_obj(src, src_k, true); 4792 } 4793 if (could_have_dest && !dest_spec) { 4794 dest = maybe_cast_profiled_obj(dest, dest_k, true); 4795 } 4796 } 4797 } 4798 } 4799 4800 ciMethod* trap_method = method(); 4801 int trap_bci = bci(); 4802 if (saved_jvms != NULL) { 4803 trap_method = alloc->jvms()->method(); 4804 trap_bci = alloc->jvms()->bci(); 4805 } 4806 4807 bool negative_length_guard_generated = false; 4808 4809 if (!C->too_many_traps(trap_method, trap_bci, Deoptimization::Reason_intrinsic) && 4810 can_emit_guards && 4811 !src->is_top() && !dest->is_top()) { 4812 // validate arguments: enables transformation the ArrayCopyNode 4813 validated = true; 4814 4815 RegionNode* slow_region = new RegionNode(1); 4816 record_for_igvn(slow_region); 4817 4818 // (1) src and dest are arrays. 4819 generate_non_array_guard(load_object_klass(src), slow_region); 4820 generate_non_array_guard(load_object_klass(dest), slow_region); 4821 4822 // (2) src and dest arrays must have elements of the same BasicType 4823 // done at macro expansion or at Ideal transformation time 4824 4825 // (4) src_offset must not be negative. 4826 generate_negative_guard(src_offset, slow_region); 4827 4828 // (5) dest_offset must not be negative. 4829 generate_negative_guard(dest_offset, slow_region); 4830 4831 // (7) src_offset + length must not exceed length of src. 4832 generate_limit_guard(src_offset, length, 4833 load_array_length(src), 4834 slow_region); 4835 4836 // (8) dest_offset + length must not exceed length of dest. 4837 generate_limit_guard(dest_offset, length, 4838 load_array_length(dest), 4839 slow_region); 4840 4841 // (6) length must not be negative. 4842 // This is also checked in generate_arraycopy() during macro expansion, but 4843 // we also have to check it here for the case where the ArrayCopyNode will 4844 // be eliminated by Escape Analysis. 4845 if (EliminateAllocations) { 4846 generate_negative_guard(length, slow_region); 4847 negative_length_guard_generated = true; 4848 } 4849 4850 // (9) each element of an oop array must be assignable 4851 Node* dest_klass = load_object_klass(dest); 4852 if (src != dest) { 4853 Node* not_subtype_ctrl = gen_subtype_check(src, dest_klass); 4854 4855 if (not_subtype_ctrl != top()) { 4856 PreserveJVMState pjvms(this); 4857 set_control(not_subtype_ctrl); 4858 uncommon_trap(Deoptimization::Reason_intrinsic, 4859 Deoptimization::Action_make_not_entrant); 4860 assert(stopped(), "Should be stopped"); 4861 } 4862 } 4863 { 4864 PreserveJVMState pjvms(this); 4865 set_control(_gvn.transform(slow_region)); 4866 uncommon_trap(Deoptimization::Reason_intrinsic, 4867 Deoptimization::Action_make_not_entrant); 4868 assert(stopped(), "Should be stopped"); 4869 } 4870 4871 const TypeKlassPtr* dest_klass_t = _gvn.type(dest_klass)->is_klassptr(); 4872 const Type *toop = TypeOopPtr::make_from_klass(dest_klass_t->klass()); 4873 src = _gvn.transform(new CheckCastPPNode(control(), src, toop)); 4874 } 4875 4876 arraycopy_move_allocation_here(alloc, dest, saved_jvms, saved_reexecute_sp, new_idx); 4877 4878 if (stopped()) { 4879 return true; 4880 } 4881 4882 ArrayCopyNode* ac = ArrayCopyNode::make(this, true, src, src_offset, dest, dest_offset, length, alloc != NULL, negative_length_guard_generated, 4883 // Create LoadRange and LoadKlass nodes for use during macro expansion here 4884 // so the compiler has a chance to eliminate them: during macro expansion, 4885 // we have to set their control (CastPP nodes are eliminated). 4886 load_object_klass(src), load_object_klass(dest), 4887 load_array_length(src), load_array_length(dest)); 4888 4889 ac->set_arraycopy(validated); 4890 4891 Node* n = _gvn.transform(ac); 4892 if (n == ac) { 4893 ac->connect_outputs(this); 4894 } else { 4895 assert(validated, "shouldn't transform if all arguments not validated"); 4896 set_all_memory(n); 4897 } 4898 clear_upper_avx(); 4899 4900 4901 return true; 4902 } 4903 4904 4905 // Helper function which determines if an arraycopy immediately follows 4906 // an allocation, with no intervening tests or other escapes for the object. 4907 AllocateArrayNode* 4908 LibraryCallKit::tightly_coupled_allocation(Node* ptr) { 4909 if (stopped()) return NULL; // no fast path 4910 if (C->AliasLevel() == 0) return NULL; // no MergeMems around 4911 4912 AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn); 4913 if (alloc == NULL) return NULL; 4914 4915 Node* rawmem = memory(Compile::AliasIdxRaw); 4916 // Is the allocation's memory state untouched? 4917 if (!(rawmem->is_Proj() && rawmem->in(0)->is_Initialize())) { 4918 // Bail out if there have been raw-memory effects since the allocation. 4919 // (Example: There might have been a call or safepoint.) 4920 return NULL; 4921 } 4922 rawmem = rawmem->in(0)->as_Initialize()->memory(Compile::AliasIdxRaw); 4923 if (!(rawmem->is_Proj() && rawmem->in(0) == alloc)) { 4924 return NULL; 4925 } 4926 4927 // There must be no unexpected observers of this allocation. 4928 for (DUIterator_Fast imax, i = ptr->fast_outs(imax); i < imax; i++) { 4929 Node* obs = ptr->fast_out(i); 4930 if (obs != this->map()) { 4931 return NULL; 4932 } 4933 } 4934 4935 // This arraycopy must unconditionally follow the allocation of the ptr. 4936 Node* alloc_ctl = ptr->in(0); 4937 Node* ctl = control(); 4938 while (ctl != alloc_ctl) { 4939 // There may be guards which feed into the slow_region. 4940 // Any other control flow means that we might not get a chance 4941 // to finish initializing the allocated object. 4942 if ((ctl->is_IfFalse() || ctl->is_IfTrue()) && ctl->in(0)->is_If()) { 4943 IfNode* iff = ctl->in(0)->as_If(); 4944 Node* not_ctl = iff->proj_out_or_null(1 - ctl->as_Proj()->_con); 4945 assert(not_ctl != NULL && not_ctl != ctl, "found alternate"); 4946 // One more try: Various low-level checks bottom out in 4947 // uncommon traps. If the debug-info of the trap omits 4948 // any reference to the allocation, as we've already 4949 // observed, then there can be no objection to the trap. 4950 bool found_trap = false; 4951 for (DUIterator_Fast jmax, j = not_ctl->fast_outs(jmax); j < jmax; j++) { 4952 Node* obs = not_ctl->fast_out(j); 4953 if (obs->in(0) == not_ctl && obs->is_Call() && 4954 (obs->as_Call()->entry_point() == SharedRuntime::uncommon_trap_blob()->entry_point())) { 4955 found_trap = true; break; 4956 } 4957 } 4958 if (found_trap) { 4959 ctl = iff->in(0); // This test feeds a harmless uncommon trap. 4960 continue; 4961 } 4962 } 4963 return NULL; 4964 } 4965 4966 // If we get this far, we have an allocation which immediately 4967 // precedes the arraycopy, and we can take over zeroing the new object. 4968 // The arraycopy will finish the initialization, and provide 4969 // a new control state to which we will anchor the destination pointer. 4970 4971 return alloc; 4972 } 4973 4974 //-------------inline_encodeISOArray----------------------------------- 4975 // encode char[] to byte[] in ISO_8859_1 or ASCII 4976 bool LibraryCallKit::inline_encodeISOArray(bool ascii) { 4977 assert(callee()->signature()->size() == 5, "encodeISOArray has 5 parameters"); 4978 // no receiver since it is static method 4979 Node *src = argument(0); 4980 Node *src_offset = argument(1); 4981 Node *dst = argument(2); 4982 Node *dst_offset = argument(3); 4983 Node *length = argument(4); 4984 4985 src = must_be_not_null(src, true); 4986 dst = must_be_not_null(dst, true); 4987 4988 const Type* src_type = src->Value(&_gvn); 4989 const Type* dst_type = dst->Value(&_gvn); 4990 const TypeAryPtr* top_src = src_type->isa_aryptr(); 4991 const TypeAryPtr* top_dest = dst_type->isa_aryptr(); 4992 if (top_src == NULL || top_src->klass() == NULL || 4993 top_dest == NULL || top_dest->klass() == NULL) { 4994 // failed array check 4995 return false; 4996 } 4997 4998 // Figure out the size and type of the elements we will be copying. 4999 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5000 BasicType dst_elem = dst_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5001 if (!((src_elem == T_CHAR) || (src_elem== T_BYTE)) || dst_elem != T_BYTE) { 5002 return false; 5003 } 5004 5005 Node* src_start = array_element_address(src, src_offset, T_CHAR); 5006 Node* dst_start = array_element_address(dst, dst_offset, dst_elem); 5007 // 'src_start' points to src array + scaled offset 5008 // 'dst_start' points to dst array + scaled offset 5009 5010 const TypeAryPtr* mtype = TypeAryPtr::BYTES; 5011 Node* enc = new EncodeISOArrayNode(control(), memory(mtype), src_start, dst_start, length, ascii); 5012 enc = _gvn.transform(enc); 5013 Node* res_mem = _gvn.transform(new SCMemProjNode(enc)); 5014 set_memory(res_mem, mtype); 5015 set_result(enc); 5016 clear_upper_avx(); 5017 5018 return true; 5019 } 5020 5021 //-------------inline_multiplyToLen----------------------------------- 5022 bool LibraryCallKit::inline_multiplyToLen() { 5023 assert(UseMultiplyToLenIntrinsic, "not implemented on this platform"); 5024 5025 address stubAddr = StubRoutines::multiplyToLen(); 5026 if (stubAddr == NULL) { 5027 return false; // Intrinsic's stub is not implemented on this platform 5028 } 5029 const char* stubName = "multiplyToLen"; 5030 5031 assert(callee()->signature()->size() == 5, "multiplyToLen has 5 parameters"); 5032 5033 // no receiver because it is a static method 5034 Node* x = argument(0); 5035 Node* xlen = argument(1); 5036 Node* y = argument(2); 5037 Node* ylen = argument(3); 5038 Node* z = argument(4); 5039 5040 x = must_be_not_null(x, true); 5041 y = must_be_not_null(y, true); 5042 5043 const Type* x_type = x->Value(&_gvn); 5044 const Type* y_type = y->Value(&_gvn); 5045 const TypeAryPtr* top_x = x_type->isa_aryptr(); 5046 const TypeAryPtr* top_y = y_type->isa_aryptr(); 5047 if (top_x == NULL || top_x->klass() == NULL || 5048 top_y == NULL || top_y->klass() == NULL) { 5049 // failed array check 5050 return false; 5051 } 5052 5053 BasicType x_elem = x_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5054 BasicType y_elem = y_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5055 if (x_elem != T_INT || y_elem != T_INT) { 5056 return false; 5057 } 5058 5059 // Set the original stack and the reexecute bit for the interpreter to reexecute 5060 // the bytecode that invokes BigInteger.multiplyToLen() if deoptimization happens 5061 // on the return from z array allocation in runtime. 5062 { PreserveReexecuteState preexecs(this); 5063 jvms()->set_should_reexecute(true); 5064 5065 Node* x_start = array_element_address(x, intcon(0), x_elem); 5066 Node* y_start = array_element_address(y, intcon(0), y_elem); 5067 // 'x_start' points to x array + scaled xlen 5068 // 'y_start' points to y array + scaled ylen 5069 5070 // Allocate the result array 5071 Node* zlen = _gvn.transform(new AddINode(xlen, ylen)); 5072 ciKlass* klass = ciTypeArrayKlass::make(T_INT); 5073 Node* klass_node = makecon(TypeKlassPtr::make(klass)); 5074 5075 IdealKit ideal(this); 5076 5077 #define __ ideal. 5078 Node* one = __ ConI(1); 5079 Node* zero = __ ConI(0); 5080 IdealVariable need_alloc(ideal), z_alloc(ideal); __ declarations_done(); 5081 __ set(need_alloc, zero); 5082 __ set(z_alloc, z); 5083 __ if_then(z, BoolTest::eq, null()); { 5084 __ increment (need_alloc, one); 5085 } __ else_(); { 5086 // Update graphKit memory and control from IdealKit. 5087 sync_kit(ideal); 5088 Node *cast = new CastPPNode(z, TypePtr::NOTNULL); 5089 cast->init_req(0, control()); 5090 _gvn.set_type(cast, cast->bottom_type()); 5091 C->record_for_igvn(cast); 5092 5093 Node* zlen_arg = load_array_length(cast); 5094 // Update IdealKit memory and control from graphKit. 5095 __ sync_kit(this); 5096 __ if_then(zlen_arg, BoolTest::lt, zlen); { 5097 __ increment (need_alloc, one); 5098 } __ end_if(); 5099 } __ end_if(); 5100 5101 __ if_then(__ value(need_alloc), BoolTest::ne, zero); { 5102 // Update graphKit memory and control from IdealKit. 5103 sync_kit(ideal); 5104 Node * narr = new_array(klass_node, zlen, 1); 5105 // Update IdealKit memory and control from graphKit. 5106 __ sync_kit(this); 5107 __ set(z_alloc, narr); 5108 } __ end_if(); 5109 5110 sync_kit(ideal); 5111 z = __ value(z_alloc); 5112 // Can't use TypeAryPtr::INTS which uses Bottom offset. 5113 _gvn.set_type(z, TypeOopPtr::make_from_klass(klass)); 5114 // Final sync IdealKit and GraphKit. 5115 final_sync(ideal); 5116 #undef __ 5117 5118 Node* z_start = array_element_address(z, intcon(0), T_INT); 5119 5120 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, 5121 OptoRuntime::multiplyToLen_Type(), 5122 stubAddr, stubName, TypePtr::BOTTOM, 5123 x_start, xlen, y_start, ylen, z_start, zlen); 5124 } // original reexecute is set back here 5125 5126 C->set_has_split_ifs(true); // Has chance for split-if optimization 5127 set_result(z); 5128 return true; 5129 } 5130 5131 //-------------inline_squareToLen------------------------------------ 5132 bool LibraryCallKit::inline_squareToLen() { 5133 assert(UseSquareToLenIntrinsic, "not implemented on this platform"); 5134 5135 address stubAddr = StubRoutines::squareToLen(); 5136 if (stubAddr == NULL) { 5137 return false; // Intrinsic's stub is not implemented on this platform 5138 } 5139 const char* stubName = "squareToLen"; 5140 5141 assert(callee()->signature()->size() == 4, "implSquareToLen has 4 parameters"); 5142 5143 Node* x = argument(0); 5144 Node* len = argument(1); 5145 Node* z = argument(2); 5146 Node* zlen = argument(3); 5147 5148 x = must_be_not_null(x, true); 5149 z = must_be_not_null(z, true); 5150 5151 const Type* x_type = x->Value(&_gvn); 5152 const Type* z_type = z->Value(&_gvn); 5153 const TypeAryPtr* top_x = x_type->isa_aryptr(); 5154 const TypeAryPtr* top_z = z_type->isa_aryptr(); 5155 if (top_x == NULL || top_x->klass() == NULL || 5156 top_z == NULL || top_z->klass() == NULL) { 5157 // failed array check 5158 return false; 5159 } 5160 5161 BasicType x_elem = x_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5162 BasicType z_elem = z_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5163 if (x_elem != T_INT || z_elem != T_INT) { 5164 return false; 5165 } 5166 5167 5168 Node* x_start = array_element_address(x, intcon(0), x_elem); 5169 Node* z_start = array_element_address(z, intcon(0), z_elem); 5170 5171 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, 5172 OptoRuntime::squareToLen_Type(), 5173 stubAddr, stubName, TypePtr::BOTTOM, 5174 x_start, len, z_start, zlen); 5175 5176 set_result(z); 5177 return true; 5178 } 5179 5180 //-------------inline_mulAdd------------------------------------------ 5181 bool LibraryCallKit::inline_mulAdd() { 5182 assert(UseMulAddIntrinsic, "not implemented on this platform"); 5183 5184 address stubAddr = StubRoutines::mulAdd(); 5185 if (stubAddr == NULL) { 5186 return false; // Intrinsic's stub is not implemented on this platform 5187 } 5188 const char* stubName = "mulAdd"; 5189 5190 assert(callee()->signature()->size() == 5, "mulAdd has 5 parameters"); 5191 5192 Node* out = argument(0); 5193 Node* in = argument(1); 5194 Node* offset = argument(2); 5195 Node* len = argument(3); 5196 Node* k = argument(4); 5197 5198 out = must_be_not_null(out, true); 5199 5200 const Type* out_type = out->Value(&_gvn); 5201 const Type* in_type = in->Value(&_gvn); 5202 const TypeAryPtr* top_out = out_type->isa_aryptr(); 5203 const TypeAryPtr* top_in = in_type->isa_aryptr(); 5204 if (top_out == NULL || top_out->klass() == NULL || 5205 top_in == NULL || top_in->klass() == NULL) { 5206 // failed array check 5207 return false; 5208 } 5209 5210 BasicType out_elem = out_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5211 BasicType in_elem = in_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5212 if (out_elem != T_INT || in_elem != T_INT) { 5213 return false; 5214 } 5215 5216 Node* outlen = load_array_length(out); 5217 Node* new_offset = _gvn.transform(new SubINode(outlen, offset)); 5218 Node* out_start = array_element_address(out, intcon(0), out_elem); 5219 Node* in_start = array_element_address(in, intcon(0), in_elem); 5220 5221 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, 5222 OptoRuntime::mulAdd_Type(), 5223 stubAddr, stubName, TypePtr::BOTTOM, 5224 out_start,in_start, new_offset, len, k); 5225 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 5226 set_result(result); 5227 return true; 5228 } 5229 5230 //-------------inline_montgomeryMultiply----------------------------------- 5231 bool LibraryCallKit::inline_montgomeryMultiply() { 5232 address stubAddr = StubRoutines::montgomeryMultiply(); 5233 if (stubAddr == NULL) { 5234 return false; // Intrinsic's stub is not implemented on this platform 5235 } 5236 5237 assert(UseMontgomeryMultiplyIntrinsic, "not implemented on this platform"); 5238 const char* stubName = "montgomery_multiply"; 5239 5240 assert(callee()->signature()->size() == 7, "montgomeryMultiply has 7 parameters"); 5241 5242 Node* a = argument(0); 5243 Node* b = argument(1); 5244 Node* n = argument(2); 5245 Node* len = argument(3); 5246 Node* inv = argument(4); 5247 Node* m = argument(6); 5248 5249 const Type* a_type = a->Value(&_gvn); 5250 const TypeAryPtr* top_a = a_type->isa_aryptr(); 5251 const Type* b_type = b->Value(&_gvn); 5252 const TypeAryPtr* top_b = b_type->isa_aryptr(); 5253 const Type* n_type = a->Value(&_gvn); 5254 const TypeAryPtr* top_n = n_type->isa_aryptr(); 5255 const Type* m_type = a->Value(&_gvn); 5256 const TypeAryPtr* top_m = m_type->isa_aryptr(); 5257 if (top_a == NULL || top_a->klass() == NULL || 5258 top_b == NULL || top_b->klass() == NULL || 5259 top_n == NULL || top_n->klass() == NULL || 5260 top_m == NULL || top_m->klass() == NULL) { 5261 // failed array check 5262 return false; 5263 } 5264 5265 BasicType a_elem = a_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5266 BasicType b_elem = b_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5267 BasicType n_elem = n_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5268 BasicType m_elem = m_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5269 if (a_elem != T_INT || b_elem != T_INT || n_elem != T_INT || m_elem != T_INT) { 5270 return false; 5271 } 5272 5273 // Make the call 5274 { 5275 Node* a_start = array_element_address(a, intcon(0), a_elem); 5276 Node* b_start = array_element_address(b, intcon(0), b_elem); 5277 Node* n_start = array_element_address(n, intcon(0), n_elem); 5278 Node* m_start = array_element_address(m, intcon(0), m_elem); 5279 5280 Node* call = make_runtime_call(RC_LEAF, 5281 OptoRuntime::montgomeryMultiply_Type(), 5282 stubAddr, stubName, TypePtr::BOTTOM, 5283 a_start, b_start, n_start, len, inv, top(), 5284 m_start); 5285 set_result(m); 5286 } 5287 5288 return true; 5289 } 5290 5291 bool LibraryCallKit::inline_montgomerySquare() { 5292 address stubAddr = StubRoutines::montgomerySquare(); 5293 if (stubAddr == NULL) { 5294 return false; // Intrinsic's stub is not implemented on this platform 5295 } 5296 5297 assert(UseMontgomerySquareIntrinsic, "not implemented on this platform"); 5298 const char* stubName = "montgomery_square"; 5299 5300 assert(callee()->signature()->size() == 6, "montgomerySquare has 6 parameters"); 5301 5302 Node* a = argument(0); 5303 Node* n = argument(1); 5304 Node* len = argument(2); 5305 Node* inv = argument(3); 5306 Node* m = argument(5); 5307 5308 const Type* a_type = a->Value(&_gvn); 5309 const TypeAryPtr* top_a = a_type->isa_aryptr(); 5310 const Type* n_type = a->Value(&_gvn); 5311 const TypeAryPtr* top_n = n_type->isa_aryptr(); 5312 const Type* m_type = a->Value(&_gvn); 5313 const TypeAryPtr* top_m = m_type->isa_aryptr(); 5314 if (top_a == NULL || top_a->klass() == NULL || 5315 top_n == NULL || top_n->klass() == NULL || 5316 top_m == NULL || top_m->klass() == NULL) { 5317 // failed array check 5318 return false; 5319 } 5320 5321 BasicType a_elem = a_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5322 BasicType n_elem = n_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5323 BasicType m_elem = m_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5324 if (a_elem != T_INT || n_elem != T_INT || m_elem != T_INT) { 5325 return false; 5326 } 5327 5328 // Make the call 5329 { 5330 Node* a_start = array_element_address(a, intcon(0), a_elem); 5331 Node* n_start = array_element_address(n, intcon(0), n_elem); 5332 Node* m_start = array_element_address(m, intcon(0), m_elem); 5333 5334 Node* call = make_runtime_call(RC_LEAF, 5335 OptoRuntime::montgomerySquare_Type(), 5336 stubAddr, stubName, TypePtr::BOTTOM, 5337 a_start, n_start, len, inv, top(), 5338 m_start); 5339 set_result(m); 5340 } 5341 5342 return true; 5343 } 5344 5345 bool LibraryCallKit::inline_bigIntegerShift(bool isRightShift) { 5346 address stubAddr = NULL; 5347 const char* stubName = NULL; 5348 5349 stubAddr = isRightShift? StubRoutines::bigIntegerRightShift(): StubRoutines::bigIntegerLeftShift(); 5350 if (stubAddr == NULL) { 5351 return false; // Intrinsic's stub is not implemented on this platform 5352 } 5353 5354 stubName = isRightShift? "bigIntegerRightShiftWorker" : "bigIntegerLeftShiftWorker"; 5355 5356 assert(callee()->signature()->size() == 5, "expected 5 arguments"); 5357 5358 Node* newArr = argument(0); 5359 Node* oldArr = argument(1); 5360 Node* newIdx = argument(2); 5361 Node* shiftCount = argument(3); 5362 Node* numIter = argument(4); 5363 5364 const Type* newArr_type = newArr->Value(&_gvn); 5365 const TypeAryPtr* top_newArr = newArr_type->isa_aryptr(); 5366 const Type* oldArr_type = oldArr->Value(&_gvn); 5367 const TypeAryPtr* top_oldArr = oldArr_type->isa_aryptr(); 5368 if (top_newArr == NULL || top_newArr->klass() == NULL || top_oldArr == NULL 5369 || top_oldArr->klass() == NULL) { 5370 return false; 5371 } 5372 5373 BasicType newArr_elem = newArr_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5374 BasicType oldArr_elem = oldArr_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5375 if (newArr_elem != T_INT || oldArr_elem != T_INT) { 5376 return false; 5377 } 5378 5379 // Make the call 5380 { 5381 Node* newArr_start = array_element_address(newArr, intcon(0), newArr_elem); 5382 Node* oldArr_start = array_element_address(oldArr, intcon(0), oldArr_elem); 5383 5384 Node* call = make_runtime_call(RC_LEAF, 5385 OptoRuntime::bigIntegerShift_Type(), 5386 stubAddr, 5387 stubName, 5388 TypePtr::BOTTOM, 5389 newArr_start, 5390 oldArr_start, 5391 newIdx, 5392 shiftCount, 5393 numIter); 5394 } 5395 5396 return true; 5397 } 5398 5399 //-------------inline_vectorizedMismatch------------------------------ 5400 bool LibraryCallKit::inline_vectorizedMismatch() { 5401 assert(UseVectorizedMismatchIntrinsic, "not implemented on this platform"); 5402 5403 assert(callee()->signature()->size() == 8, "vectorizedMismatch has 6 parameters"); 5404 Node* obja = argument(0); // Object 5405 Node* aoffset = argument(1); // long 5406 Node* objb = argument(3); // Object 5407 Node* boffset = argument(4); // long 5408 Node* length = argument(6); // int 5409 Node* scale = argument(7); // int 5410 5411 const TypeAryPtr* obja_t = _gvn.type(obja)->isa_aryptr(); 5412 const TypeAryPtr* objb_t = _gvn.type(objb)->isa_aryptr(); 5413 if (obja_t == NULL || obja_t->klass() == NULL || 5414 objb_t == NULL || objb_t->klass() == NULL || 5415 scale == top()) { 5416 return false; // failed input validation 5417 } 5418 5419 Node* obja_adr = make_unsafe_address(obja, aoffset); 5420 Node* objb_adr = make_unsafe_address(objb, boffset); 5421 5422 // Partial inlining handling for inputs smaller than ArrayOperationPartialInlineSize bytes in size. 5423 // 5424 // inline_limit = ArrayOperationPartialInlineSize / element_size; 5425 // if (length <= inline_limit) { 5426 // inline_path: 5427 // vmask = VectorMaskGen length 5428 // vload1 = LoadVectorMasked obja, vmask 5429 // vload2 = LoadVectorMasked objb, vmask 5430 // result1 = VectorCmpMasked vload1, vload2, vmask 5431 // } else { 5432 // call_stub_path: 5433 // result2 = call vectorizedMismatch_stub(obja, objb, length, scale) 5434 // } 5435 // exit_block: 5436 // return Phi(result1, result2); 5437 // 5438 enum { inline_path = 1, // input is small enough to process it all at once 5439 stub_path = 2, // input is too large; call into the VM 5440 PATH_LIMIT = 3 5441 }; 5442 5443 Node* exit_block = new RegionNode(PATH_LIMIT); 5444 Node* result_phi = new PhiNode(exit_block, TypeInt::INT); 5445 Node* memory_phi = new PhiNode(exit_block, Type::MEMORY, TypePtr::BOTTOM); 5446 5447 Node* call_stub_path = control(); 5448 5449 BasicType elem_bt = T_ILLEGAL; 5450 5451 const TypeInt* scale_t = _gvn.type(scale)->is_int(); 5452 if (scale_t->is_con()) { 5453 switch (scale_t->get_con()) { 5454 case 0: elem_bt = T_BYTE; break; 5455 case 1: elem_bt = T_SHORT; break; 5456 case 2: elem_bt = T_INT; break; 5457 case 3: elem_bt = T_LONG; break; 5458 5459 default: elem_bt = T_ILLEGAL; break; // not supported 5460 } 5461 } 5462 5463 int inline_limit = 0; 5464 bool do_partial_inline = false; 5465 5466 if (elem_bt != T_ILLEGAL && ArrayOperationPartialInlineSize > 0) { 5467 inline_limit = ArrayOperationPartialInlineSize / type2aelembytes(elem_bt); 5468 do_partial_inline = inline_limit >= 16; 5469 } 5470 5471 if (do_partial_inline) { 5472 assert(elem_bt != T_ILLEGAL, "sanity"); 5473 5474 if (Matcher::match_rule_supported_vector(Op_VectorMaskGen, inline_limit, elem_bt) && 5475 Matcher::match_rule_supported_vector(Op_LoadVectorMasked, inline_limit, elem_bt) && 5476 Matcher::match_rule_supported_vector(Op_VectorCmpMasked, inline_limit, elem_bt)) { 5477 5478 const TypeVect* vt = TypeVect::make(elem_bt, inline_limit); 5479 Node* cmp_length = _gvn.transform(new CmpINode(length, intcon(inline_limit))); 5480 Node* bol_gt = _gvn.transform(new BoolNode(cmp_length, BoolTest::gt)); 5481 5482 call_stub_path = generate_guard(bol_gt, NULL, PROB_MIN); 5483 5484 if (!stopped()) { 5485 Node* casted_length = _gvn.transform(new CastIINode(control(), length, TypeInt::make(0, inline_limit, Type::WidenMin))); 5486 5487 const TypePtr* obja_adr_t = _gvn.type(obja_adr)->isa_ptr(); 5488 const TypePtr* objb_adr_t = _gvn.type(objb_adr)->isa_ptr(); 5489 Node* obja_adr_mem = memory(C->get_alias_index(obja_adr_t)); 5490 Node* objb_adr_mem = memory(C->get_alias_index(objb_adr_t)); 5491 5492 Node* vmask = _gvn.transform(VectorMaskGenNode::make(ConvI2X(casted_length), elem_bt)); 5493 Node* vload_obja = _gvn.transform(new LoadVectorMaskedNode(control(), obja_adr_mem, obja_adr, obja_adr_t, vt, vmask)); 5494 Node* vload_objb = _gvn.transform(new LoadVectorMaskedNode(control(), objb_adr_mem, objb_adr, objb_adr_t, vt, vmask)); 5495 Node* result = _gvn.transform(new VectorCmpMaskedNode(vload_obja, vload_objb, vmask, TypeInt::INT)); 5496 5497 exit_block->init_req(inline_path, control()); 5498 memory_phi->init_req(inline_path, map()->memory()); 5499 result_phi->init_req(inline_path, result); 5500 5501 C->set_max_vector_size(MAX2((uint)ArrayOperationPartialInlineSize, C->max_vector_size())); 5502 clear_upper_avx(); 5503 } 5504 } 5505 } 5506 5507 if (call_stub_path != NULL) { 5508 set_control(call_stub_path); 5509 5510 Node* call = make_runtime_call(RC_LEAF, 5511 OptoRuntime::vectorizedMismatch_Type(), 5512 StubRoutines::vectorizedMismatch(), "vectorizedMismatch", TypePtr::BOTTOM, 5513 obja_adr, objb_adr, length, scale); 5514 5515 exit_block->init_req(stub_path, control()); 5516 memory_phi->init_req(stub_path, map()->memory()); 5517 result_phi->init_req(stub_path, _gvn.transform(new ProjNode(call, TypeFunc::Parms))); 5518 } 5519 5520 exit_block = _gvn.transform(exit_block); 5521 memory_phi = _gvn.transform(memory_phi); 5522 result_phi = _gvn.transform(result_phi); 5523 5524 set_control(exit_block); 5525 set_all_memory(memory_phi); 5526 set_result(result_phi); 5527 5528 return true; 5529 } 5530 5531 /** 5532 * Calculate CRC32 for byte. 5533 * int java.util.zip.CRC32.update(int crc, int b) 5534 */ 5535 bool LibraryCallKit::inline_updateCRC32() { 5536 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support"); 5537 assert(callee()->signature()->size() == 2, "update has 2 parameters"); 5538 // no receiver since it is static method 5539 Node* crc = argument(0); // type: int 5540 Node* b = argument(1); // type: int 5541 5542 /* 5543 * int c = ~ crc; 5544 * b = timesXtoThe32[(b ^ c) & 0xFF]; 5545 * b = b ^ (c >>> 8); 5546 * crc = ~b; 5547 */ 5548 5549 Node* M1 = intcon(-1); 5550 crc = _gvn.transform(new XorINode(crc, M1)); 5551 Node* result = _gvn.transform(new XorINode(crc, b)); 5552 result = _gvn.transform(new AndINode(result, intcon(0xFF))); 5553 5554 Node* base = makecon(TypeRawPtr::make(StubRoutines::crc_table_addr())); 5555 Node* offset = _gvn.transform(new LShiftINode(result, intcon(0x2))); 5556 Node* adr = basic_plus_adr(top(), base, ConvI2X(offset)); 5557 result = make_load(control(), adr, TypeInt::INT, T_INT, MemNode::unordered); 5558 5559 crc = _gvn.transform(new URShiftINode(crc, intcon(8))); 5560 result = _gvn.transform(new XorINode(crc, result)); 5561 result = _gvn.transform(new XorINode(result, M1)); 5562 set_result(result); 5563 return true; 5564 } 5565 5566 /** 5567 * Calculate CRC32 for byte[] array. 5568 * int java.util.zip.CRC32.updateBytes(int crc, byte[] buf, int off, int len) 5569 */ 5570 bool LibraryCallKit::inline_updateBytesCRC32() { 5571 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support"); 5572 assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters"); 5573 // no receiver since it is static method 5574 Node* crc = argument(0); // type: int 5575 Node* src = argument(1); // type: oop 5576 Node* offset = argument(2); // type: int 5577 Node* length = argument(3); // type: int 5578 5579 const Type* src_type = src->Value(&_gvn); 5580 const TypeAryPtr* top_src = src_type->isa_aryptr(); 5581 if (top_src == NULL || top_src->klass() == NULL) { 5582 // failed array check 5583 return false; 5584 } 5585 5586 // Figure out the size and type of the elements we will be copying. 5587 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5588 if (src_elem != T_BYTE) { 5589 return false; 5590 } 5591 5592 // 'src_start' points to src array + scaled offset 5593 src = must_be_not_null(src, true); 5594 Node* src_start = array_element_address(src, offset, src_elem); 5595 5596 // We assume that range check is done by caller. 5597 // TODO: generate range check (offset+length < src.length) in debug VM. 5598 5599 // Call the stub. 5600 address stubAddr = StubRoutines::updateBytesCRC32(); 5601 const char *stubName = "updateBytesCRC32"; 5602 5603 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(), 5604 stubAddr, stubName, TypePtr::BOTTOM, 5605 crc, src_start, length); 5606 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 5607 set_result(result); 5608 return true; 5609 } 5610 5611 /** 5612 * Calculate CRC32 for ByteBuffer. 5613 * int java.util.zip.CRC32.updateByteBuffer(int crc, long buf, int off, int len) 5614 */ 5615 bool LibraryCallKit::inline_updateByteBufferCRC32() { 5616 assert(UseCRC32Intrinsics, "need AVX and LCMUL instructions support"); 5617 assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long"); 5618 // no receiver since it is static method 5619 Node* crc = argument(0); // type: int 5620 Node* src = argument(1); // type: long 5621 Node* offset = argument(3); // type: int 5622 Node* length = argument(4); // type: int 5623 5624 src = ConvL2X(src); // adjust Java long to machine word 5625 Node* base = _gvn.transform(new CastX2PNode(src)); 5626 offset = ConvI2X(offset); 5627 5628 // 'src_start' points to src array + scaled offset 5629 Node* src_start = basic_plus_adr(top(), base, offset); 5630 5631 // Call the stub. 5632 address stubAddr = StubRoutines::updateBytesCRC32(); 5633 const char *stubName = "updateBytesCRC32"; 5634 5635 Node* call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::updateBytesCRC32_Type(), 5636 stubAddr, stubName, TypePtr::BOTTOM, 5637 crc, src_start, length); 5638 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 5639 set_result(result); 5640 return true; 5641 } 5642 5643 //------------------------------get_table_from_crc32c_class----------------------- 5644 Node * LibraryCallKit::get_table_from_crc32c_class(ciInstanceKlass *crc32c_class) { 5645 Node* table = load_field_from_object(NULL, "byteTable", "[I", /*decorators*/ IN_HEAP, /*is_static*/ true, crc32c_class); 5646 assert (table != NULL, "wrong version of java.util.zip.CRC32C"); 5647 5648 return table; 5649 } 5650 5651 //------------------------------inline_updateBytesCRC32C----------------------- 5652 // 5653 // Calculate CRC32C for byte[] array. 5654 // int java.util.zip.CRC32C.updateBytes(int crc, byte[] buf, int off, int end) 5655 // 5656 bool LibraryCallKit::inline_updateBytesCRC32C() { 5657 assert(UseCRC32CIntrinsics, "need CRC32C instruction support"); 5658 assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters"); 5659 assert(callee()->holder()->is_loaded(), "CRC32C class must be loaded"); 5660 // no receiver since it is a static method 5661 Node* crc = argument(0); // type: int 5662 Node* src = argument(1); // type: oop 5663 Node* offset = argument(2); // type: int 5664 Node* end = argument(3); // type: int 5665 5666 Node* length = _gvn.transform(new SubINode(end, offset)); 5667 5668 const Type* src_type = src->Value(&_gvn); 5669 const TypeAryPtr* top_src = src_type->isa_aryptr(); 5670 if (top_src == NULL || top_src->klass() == NULL) { 5671 // failed array check 5672 return false; 5673 } 5674 5675 // Figure out the size and type of the elements we will be copying. 5676 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5677 if (src_elem != T_BYTE) { 5678 return false; 5679 } 5680 5681 // 'src_start' points to src array + scaled offset 5682 src = must_be_not_null(src, true); 5683 Node* src_start = array_element_address(src, offset, src_elem); 5684 5685 // static final int[] byteTable in class CRC32C 5686 Node* table = get_table_from_crc32c_class(callee()->holder()); 5687 table = must_be_not_null(table, true); 5688 Node* table_start = array_element_address(table, intcon(0), T_INT); 5689 5690 // We assume that range check is done by caller. 5691 // TODO: generate range check (offset+length < src.length) in debug VM. 5692 5693 // Call the stub. 5694 address stubAddr = StubRoutines::updateBytesCRC32C(); 5695 const char *stubName = "updateBytesCRC32C"; 5696 5697 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesCRC32C_Type(), 5698 stubAddr, stubName, TypePtr::BOTTOM, 5699 crc, src_start, length, table_start); 5700 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 5701 set_result(result); 5702 return true; 5703 } 5704 5705 //------------------------------inline_updateDirectByteBufferCRC32C----------------------- 5706 // 5707 // Calculate CRC32C for DirectByteBuffer. 5708 // int java.util.zip.CRC32C.updateDirectByteBuffer(int crc, long buf, int off, int end) 5709 // 5710 bool LibraryCallKit::inline_updateDirectByteBufferCRC32C() { 5711 assert(UseCRC32CIntrinsics, "need CRC32C instruction support"); 5712 assert(callee()->signature()->size() == 5, "updateDirectByteBuffer has 4 parameters and one is long"); 5713 assert(callee()->holder()->is_loaded(), "CRC32C class must be loaded"); 5714 // no receiver since it is a static method 5715 Node* crc = argument(0); // type: int 5716 Node* src = argument(1); // type: long 5717 Node* offset = argument(3); // type: int 5718 Node* end = argument(4); // type: int 5719 5720 Node* length = _gvn.transform(new SubINode(end, offset)); 5721 5722 src = ConvL2X(src); // adjust Java long to machine word 5723 Node* base = _gvn.transform(new CastX2PNode(src)); 5724 offset = ConvI2X(offset); 5725 5726 // 'src_start' points to src array + scaled offset 5727 Node* src_start = basic_plus_adr(top(), base, offset); 5728 5729 // static final int[] byteTable in class CRC32C 5730 Node* table = get_table_from_crc32c_class(callee()->holder()); 5731 table = must_be_not_null(table, true); 5732 Node* table_start = array_element_address(table, intcon(0), T_INT); 5733 5734 // Call the stub. 5735 address stubAddr = StubRoutines::updateBytesCRC32C(); 5736 const char *stubName = "updateBytesCRC32C"; 5737 5738 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesCRC32C_Type(), 5739 stubAddr, stubName, TypePtr::BOTTOM, 5740 crc, src_start, length, table_start); 5741 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 5742 set_result(result); 5743 return true; 5744 } 5745 5746 //------------------------------inline_updateBytesAdler32---------------------- 5747 // 5748 // Calculate Adler32 checksum for byte[] array. 5749 // int java.util.zip.Adler32.updateBytes(int crc, byte[] buf, int off, int len) 5750 // 5751 bool LibraryCallKit::inline_updateBytesAdler32() { 5752 assert(UseAdler32Intrinsics, "Adler32 Intrinsic support need"); // check if we actually need to check this flag or check a different one 5753 assert(callee()->signature()->size() == 4, "updateBytes has 4 parameters"); 5754 assert(callee()->holder()->is_loaded(), "Adler32 class must be loaded"); 5755 // no receiver since it is static method 5756 Node* crc = argument(0); // type: int 5757 Node* src = argument(1); // type: oop 5758 Node* offset = argument(2); // type: int 5759 Node* length = argument(3); // type: int 5760 5761 const Type* src_type = src->Value(&_gvn); 5762 const TypeAryPtr* top_src = src_type->isa_aryptr(); 5763 if (top_src == NULL || top_src->klass() == NULL) { 5764 // failed array check 5765 return false; 5766 } 5767 5768 // Figure out the size and type of the elements we will be copying. 5769 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 5770 if (src_elem != T_BYTE) { 5771 return false; 5772 } 5773 5774 // 'src_start' points to src array + scaled offset 5775 Node* src_start = array_element_address(src, offset, src_elem); 5776 5777 // We assume that range check is done by caller. 5778 // TODO: generate range check (offset+length < src.length) in debug VM. 5779 5780 // Call the stub. 5781 address stubAddr = StubRoutines::updateBytesAdler32(); 5782 const char *stubName = "updateBytesAdler32"; 5783 5784 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesAdler32_Type(), 5785 stubAddr, stubName, TypePtr::BOTTOM, 5786 crc, src_start, length); 5787 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 5788 set_result(result); 5789 return true; 5790 } 5791 5792 //------------------------------inline_updateByteBufferAdler32--------------- 5793 // 5794 // Calculate Adler32 checksum for DirectByteBuffer. 5795 // int java.util.zip.Adler32.updateByteBuffer(int crc, long buf, int off, int len) 5796 // 5797 bool LibraryCallKit::inline_updateByteBufferAdler32() { 5798 assert(UseAdler32Intrinsics, "Adler32 Intrinsic support need"); // check if we actually need to check this flag or check a different one 5799 assert(callee()->signature()->size() == 5, "updateByteBuffer has 4 parameters and one is long"); 5800 assert(callee()->holder()->is_loaded(), "Adler32 class must be loaded"); 5801 // no receiver since it is static method 5802 Node* crc = argument(0); // type: int 5803 Node* src = argument(1); // type: long 5804 Node* offset = argument(3); // type: int 5805 Node* length = argument(4); // type: int 5806 5807 src = ConvL2X(src); // adjust Java long to machine word 5808 Node* base = _gvn.transform(new CastX2PNode(src)); 5809 offset = ConvI2X(offset); 5810 5811 // 'src_start' points to src array + scaled offset 5812 Node* src_start = basic_plus_adr(top(), base, offset); 5813 5814 // Call the stub. 5815 address stubAddr = StubRoutines::updateBytesAdler32(); 5816 const char *stubName = "updateBytesAdler32"; 5817 5818 Node* call = make_runtime_call(RC_LEAF, OptoRuntime::updateBytesAdler32_Type(), 5819 stubAddr, stubName, TypePtr::BOTTOM, 5820 crc, src_start, length); 5821 5822 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 5823 set_result(result); 5824 return true; 5825 } 5826 5827 //----------------------------inline_reference_get---------------------------- 5828 // public T java.lang.ref.Reference.get(); 5829 bool LibraryCallKit::inline_reference_get() { 5830 const int referent_offset = java_lang_ref_Reference::referent_offset(); 5831 5832 // Get the argument: 5833 Node* reference_obj = null_check_receiver(); 5834 if (stopped()) return true; 5835 5836 DecoratorSet decorators = IN_HEAP | ON_WEAK_OOP_REF; 5837 Node* result = load_field_from_object(reference_obj, "referent", "Ljava/lang/Object;", 5838 decorators, /*is_static*/ false, NULL); 5839 if (result == NULL) return false; 5840 5841 // Add memory barrier to prevent commoning reads from this field 5842 // across safepoint since GC can change its value. 5843 insert_mem_bar(Op_MemBarCPUOrder); 5844 5845 set_result(result); 5846 return true; 5847 } 5848 5849 //----------------------------inline_reference_refersTo0---------------------------- 5850 // bool java.lang.ref.Reference.refersTo0(); 5851 // bool java.lang.ref.PhantomReference.refersTo0(); 5852 bool LibraryCallKit::inline_reference_refersTo0(bool is_phantom) { 5853 // Get arguments: 5854 Node* reference_obj = null_check_receiver(); 5855 Node* other_obj = argument(1); 5856 if (stopped()) return true; 5857 5858 DecoratorSet decorators = IN_HEAP | AS_NO_KEEPALIVE; 5859 decorators |= (is_phantom ? ON_PHANTOM_OOP_REF : ON_WEAK_OOP_REF); 5860 Node* referent = load_field_from_object(reference_obj, "referent", "Ljava/lang/Object;", 5861 decorators, /*is_static*/ false, NULL); 5862 if (referent == NULL) return false; 5863 5864 // Add memory barrier to prevent commoning reads from this field 5865 // across safepoint since GC can change its value. 5866 insert_mem_bar(Op_MemBarCPUOrder); 5867 5868 Node* cmp = _gvn.transform(new CmpPNode(referent, other_obj)); 5869 Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::eq)); 5870 IfNode* if_node = create_and_map_if(control(), bol, PROB_FAIR, COUNT_UNKNOWN); 5871 5872 RegionNode* region = new RegionNode(3); 5873 PhiNode* phi = new PhiNode(region, TypeInt::BOOL); 5874 5875 Node* if_true = _gvn.transform(new IfTrueNode(if_node)); 5876 region->init_req(1, if_true); 5877 phi->init_req(1, intcon(1)); 5878 5879 Node* if_false = _gvn.transform(new IfFalseNode(if_node)); 5880 region->init_req(2, if_false); 5881 phi->init_req(2, intcon(0)); 5882 5883 set_control(_gvn.transform(region)); 5884 record_for_igvn(region); 5885 set_result(_gvn.transform(phi)); 5886 return true; 5887 } 5888 5889 5890 Node* LibraryCallKit::load_field_from_object(Node* fromObj, const char* fieldName, const char* fieldTypeString, 5891 DecoratorSet decorators = IN_HEAP, bool is_static = false, 5892 ciInstanceKlass* fromKls = NULL) { 5893 if (fromKls == NULL) { 5894 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr(); 5895 assert(tinst != NULL, "obj is null"); 5896 assert(tinst->klass()->is_loaded(), "obj is not loaded"); 5897 fromKls = tinst->klass()->as_instance_klass(); 5898 } else { 5899 assert(is_static, "only for static field access"); 5900 } 5901 ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName), 5902 ciSymbol::make(fieldTypeString), 5903 is_static); 5904 5905 assert (field != NULL, "undefined field"); 5906 if (field == NULL) return (Node *) NULL; 5907 5908 if (is_static) { 5909 const TypeInstPtr* tip = TypeInstPtr::make(fromKls->java_mirror()); 5910 fromObj = makecon(tip); 5911 } 5912 5913 // Next code copied from Parse::do_get_xxx(): 5914 5915 // Compute address and memory type. 5916 int offset = field->offset_in_bytes(); 5917 bool is_vol = field->is_volatile(); 5918 ciType* field_klass = field->type(); 5919 assert(field_klass->is_loaded(), "should be loaded"); 5920 const TypePtr* adr_type = C->alias_type(field)->adr_type(); 5921 Node *adr = basic_plus_adr(fromObj, fromObj, offset); 5922 BasicType bt = field->layout_type(); 5923 5924 // Build the resultant type of the load 5925 const Type *type; 5926 if (bt == T_OBJECT) { 5927 type = TypeOopPtr::make_from_klass(field_klass->as_klass()); 5928 } else { 5929 type = Type::get_const_basic_type(bt); 5930 } 5931 5932 if (is_vol) { 5933 decorators |= MO_SEQ_CST; 5934 } 5935 5936 return access_load_at(fromObj, adr, adr_type, type, bt, decorators); 5937 } 5938 5939 Node * LibraryCallKit::field_address_from_object(Node * fromObj, const char * fieldName, const char * fieldTypeString, 5940 bool is_exact = true, bool is_static = false, 5941 ciInstanceKlass * fromKls = NULL) { 5942 if (fromKls == NULL) { 5943 const TypeInstPtr* tinst = _gvn.type(fromObj)->isa_instptr(); 5944 assert(tinst != NULL, "obj is null"); 5945 assert(tinst->klass()->is_loaded(), "obj is not loaded"); 5946 assert(!is_exact || tinst->klass_is_exact(), "klass not exact"); 5947 fromKls = tinst->klass()->as_instance_klass(); 5948 } 5949 else { 5950 assert(is_static, "only for static field access"); 5951 } 5952 ciField* field = fromKls->get_field_by_name(ciSymbol::make(fieldName), 5953 ciSymbol::make(fieldTypeString), 5954 is_static); 5955 5956 assert(field != NULL, "undefined field"); 5957 assert(!field->is_volatile(), "not defined for volatile fields"); 5958 5959 if (is_static) { 5960 const TypeInstPtr* tip = TypeInstPtr::make(fromKls->java_mirror()); 5961 fromObj = makecon(tip); 5962 } 5963 5964 // Next code copied from Parse::do_get_xxx(): 5965 5966 // Compute address and memory type. 5967 int offset = field->offset_in_bytes(); 5968 Node *adr = basic_plus_adr(fromObj, fromObj, offset); 5969 5970 return adr; 5971 } 5972 5973 //------------------------------inline_aescrypt_Block----------------------- 5974 bool LibraryCallKit::inline_aescrypt_Block(vmIntrinsics::ID id) { 5975 address stubAddr = NULL; 5976 const char *stubName; 5977 assert(UseAES, "need AES instruction support"); 5978 5979 switch(id) { 5980 case vmIntrinsics::_aescrypt_encryptBlock: 5981 stubAddr = StubRoutines::aescrypt_encryptBlock(); 5982 stubName = "aescrypt_encryptBlock"; 5983 break; 5984 case vmIntrinsics::_aescrypt_decryptBlock: 5985 stubAddr = StubRoutines::aescrypt_decryptBlock(); 5986 stubName = "aescrypt_decryptBlock"; 5987 break; 5988 default: 5989 break; 5990 } 5991 if (stubAddr == NULL) return false; 5992 5993 Node* aescrypt_object = argument(0); 5994 Node* src = argument(1); 5995 Node* src_offset = argument(2); 5996 Node* dest = argument(3); 5997 Node* dest_offset = argument(4); 5998 5999 src = must_be_not_null(src, true); 6000 dest = must_be_not_null(dest, true); 6001 6002 // (1) src and dest are arrays. 6003 const Type* src_type = src->Value(&_gvn); 6004 const Type* dest_type = dest->Value(&_gvn); 6005 const TypeAryPtr* top_src = src_type->isa_aryptr(); 6006 const TypeAryPtr* top_dest = dest_type->isa_aryptr(); 6007 assert (top_src != NULL && top_src->klass() != NULL && top_dest != NULL && top_dest->klass() != NULL, "args are strange"); 6008 6009 // for the quick and dirty code we will skip all the checks. 6010 // we are just trying to get the call to be generated. 6011 Node* src_start = src; 6012 Node* dest_start = dest; 6013 if (src_offset != NULL || dest_offset != NULL) { 6014 assert(src_offset != NULL && dest_offset != NULL, ""); 6015 src_start = array_element_address(src, src_offset, T_BYTE); 6016 dest_start = array_element_address(dest, dest_offset, T_BYTE); 6017 } 6018 6019 // now need to get the start of its expanded key array 6020 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java 6021 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); 6022 if (k_start == NULL) return false; 6023 6024 // Call the stub. 6025 make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::aescrypt_block_Type(), 6026 stubAddr, stubName, TypePtr::BOTTOM, 6027 src_start, dest_start, k_start); 6028 6029 return true; 6030 } 6031 6032 //------------------------------inline_cipherBlockChaining_AESCrypt----------------------- 6033 bool LibraryCallKit::inline_cipherBlockChaining_AESCrypt(vmIntrinsics::ID id) { 6034 address stubAddr = NULL; 6035 const char *stubName = NULL; 6036 6037 assert(UseAES, "need AES instruction support"); 6038 6039 switch(id) { 6040 case vmIntrinsics::_cipherBlockChaining_encryptAESCrypt: 6041 stubAddr = StubRoutines::cipherBlockChaining_encryptAESCrypt(); 6042 stubName = "cipherBlockChaining_encryptAESCrypt"; 6043 break; 6044 case vmIntrinsics::_cipherBlockChaining_decryptAESCrypt: 6045 stubAddr = StubRoutines::cipherBlockChaining_decryptAESCrypt(); 6046 stubName = "cipherBlockChaining_decryptAESCrypt"; 6047 break; 6048 default: 6049 break; 6050 } 6051 if (stubAddr == NULL) return false; 6052 6053 Node* cipherBlockChaining_object = argument(0); 6054 Node* src = argument(1); 6055 Node* src_offset = argument(2); 6056 Node* len = argument(3); 6057 Node* dest = argument(4); 6058 Node* dest_offset = argument(5); 6059 6060 src = must_be_not_null(src, false); 6061 dest = must_be_not_null(dest, false); 6062 6063 // (1) src and dest are arrays. 6064 const Type* src_type = src->Value(&_gvn); 6065 const Type* dest_type = dest->Value(&_gvn); 6066 const TypeAryPtr* top_src = src_type->isa_aryptr(); 6067 const TypeAryPtr* top_dest = dest_type->isa_aryptr(); 6068 assert (top_src != NULL && top_src->klass() != NULL 6069 && top_dest != NULL && top_dest->klass() != NULL, "args are strange"); 6070 6071 // checks are the responsibility of the caller 6072 Node* src_start = src; 6073 Node* dest_start = dest; 6074 if (src_offset != NULL || dest_offset != NULL) { 6075 assert(src_offset != NULL && dest_offset != NULL, ""); 6076 src_start = array_element_address(src, src_offset, T_BYTE); 6077 dest_start = array_element_address(dest, dest_offset, T_BYTE); 6078 } 6079 6080 // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object 6081 // (because of the predicated logic executed earlier). 6082 // so we cast it here safely. 6083 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java 6084 6085 Node* embeddedCipherObj = load_field_from_object(cipherBlockChaining_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6086 if (embeddedCipherObj == NULL) return false; 6087 6088 // cast it to what we know it will be at runtime 6089 const TypeInstPtr* tinst = _gvn.type(cipherBlockChaining_object)->isa_instptr(); 6090 assert(tinst != NULL, "CBC obj is null"); 6091 assert(tinst->klass()->is_loaded(), "CBC obj is not loaded"); 6092 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6093 assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded"); 6094 6095 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6096 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt); 6097 const TypeOopPtr* xtype = aklass->as_instance_type()->cast_to_ptr_type(TypePtr::NotNull); 6098 Node* aescrypt_object = new CheckCastPPNode(control(), embeddedCipherObj, xtype); 6099 aescrypt_object = _gvn.transform(aescrypt_object); 6100 6101 // we need to get the start of the aescrypt_object's expanded key array 6102 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); 6103 if (k_start == NULL) return false; 6104 6105 // similarly, get the start address of the r vector 6106 Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B"); 6107 if (objRvec == NULL) return false; 6108 Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE); 6109 6110 // Call the stub, passing src_start, dest_start, k_start, r_start and src_len 6111 Node* cbcCrypt = make_runtime_call(RC_LEAF|RC_NO_FP, 6112 OptoRuntime::cipherBlockChaining_aescrypt_Type(), 6113 stubAddr, stubName, TypePtr::BOTTOM, 6114 src_start, dest_start, k_start, r_start, len); 6115 6116 // return cipher length (int) 6117 Node* retvalue = _gvn.transform(new ProjNode(cbcCrypt, TypeFunc::Parms)); 6118 set_result(retvalue); 6119 return true; 6120 } 6121 6122 //------------------------------inline_electronicCodeBook_AESCrypt----------------------- 6123 bool LibraryCallKit::inline_electronicCodeBook_AESCrypt(vmIntrinsics::ID id) { 6124 address stubAddr = NULL; 6125 const char *stubName = NULL; 6126 6127 assert(UseAES, "need AES instruction support"); 6128 6129 switch (id) { 6130 case vmIntrinsics::_electronicCodeBook_encryptAESCrypt: 6131 stubAddr = StubRoutines::electronicCodeBook_encryptAESCrypt(); 6132 stubName = "electronicCodeBook_encryptAESCrypt"; 6133 break; 6134 case vmIntrinsics::_electronicCodeBook_decryptAESCrypt: 6135 stubAddr = StubRoutines::electronicCodeBook_decryptAESCrypt(); 6136 stubName = "electronicCodeBook_decryptAESCrypt"; 6137 break; 6138 default: 6139 break; 6140 } 6141 6142 if (stubAddr == NULL) return false; 6143 6144 Node* electronicCodeBook_object = argument(0); 6145 Node* src = argument(1); 6146 Node* src_offset = argument(2); 6147 Node* len = argument(3); 6148 Node* dest = argument(4); 6149 Node* dest_offset = argument(5); 6150 6151 // (1) src and dest are arrays. 6152 const Type* src_type = src->Value(&_gvn); 6153 const Type* dest_type = dest->Value(&_gvn); 6154 const TypeAryPtr* top_src = src_type->isa_aryptr(); 6155 const TypeAryPtr* top_dest = dest_type->isa_aryptr(); 6156 assert(top_src != NULL && top_src->klass() != NULL 6157 && top_dest != NULL && top_dest->klass() != NULL, "args are strange"); 6158 6159 // checks are the responsibility of the caller 6160 Node* src_start = src; 6161 Node* dest_start = dest; 6162 if (src_offset != NULL || dest_offset != NULL) { 6163 assert(src_offset != NULL && dest_offset != NULL, ""); 6164 src_start = array_element_address(src, src_offset, T_BYTE); 6165 dest_start = array_element_address(dest, dest_offset, T_BYTE); 6166 } 6167 6168 // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object 6169 // (because of the predicated logic executed earlier). 6170 // so we cast it here safely. 6171 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java 6172 6173 Node* embeddedCipherObj = load_field_from_object(electronicCodeBook_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6174 if (embeddedCipherObj == NULL) return false; 6175 6176 // cast it to what we know it will be at runtime 6177 const TypeInstPtr* tinst = _gvn.type(electronicCodeBook_object)->isa_instptr(); 6178 assert(tinst != NULL, "ECB obj is null"); 6179 assert(tinst->klass()->is_loaded(), "ECB obj is not loaded"); 6180 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6181 assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded"); 6182 6183 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6184 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt); 6185 const TypeOopPtr* xtype = aklass->as_instance_type()->cast_to_ptr_type(TypePtr::NotNull); 6186 Node* aescrypt_object = new CheckCastPPNode(control(), embeddedCipherObj, xtype); 6187 aescrypt_object = _gvn.transform(aescrypt_object); 6188 6189 // we need to get the start of the aescrypt_object's expanded key array 6190 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); 6191 if (k_start == NULL) return false; 6192 6193 // Call the stub, passing src_start, dest_start, k_start, r_start and src_len 6194 Node* ecbCrypt = make_runtime_call(RC_LEAF | RC_NO_FP, 6195 OptoRuntime::electronicCodeBook_aescrypt_Type(), 6196 stubAddr, stubName, TypePtr::BOTTOM, 6197 src_start, dest_start, k_start, len); 6198 6199 // return cipher length (int) 6200 Node* retvalue = _gvn.transform(new ProjNode(ecbCrypt, TypeFunc::Parms)); 6201 set_result(retvalue); 6202 return true; 6203 } 6204 6205 //------------------------------inline_counterMode_AESCrypt----------------------- 6206 bool LibraryCallKit::inline_counterMode_AESCrypt(vmIntrinsics::ID id) { 6207 assert(UseAES, "need AES instruction support"); 6208 if (!UseAESCTRIntrinsics) return false; 6209 6210 address stubAddr = NULL; 6211 const char *stubName = NULL; 6212 if (id == vmIntrinsics::_counterMode_AESCrypt) { 6213 stubAddr = StubRoutines::counterMode_AESCrypt(); 6214 stubName = "counterMode_AESCrypt"; 6215 } 6216 if (stubAddr == NULL) return false; 6217 6218 Node* counterMode_object = argument(0); 6219 Node* src = argument(1); 6220 Node* src_offset = argument(2); 6221 Node* len = argument(3); 6222 Node* dest = argument(4); 6223 Node* dest_offset = argument(5); 6224 6225 // (1) src and dest are arrays. 6226 const Type* src_type = src->Value(&_gvn); 6227 const Type* dest_type = dest->Value(&_gvn); 6228 const TypeAryPtr* top_src = src_type->isa_aryptr(); 6229 const TypeAryPtr* top_dest = dest_type->isa_aryptr(); 6230 assert(top_src != NULL && top_src->klass() != NULL && 6231 top_dest != NULL && top_dest->klass() != NULL, "args are strange"); 6232 6233 // checks are the responsibility of the caller 6234 Node* src_start = src; 6235 Node* dest_start = dest; 6236 if (src_offset != NULL || dest_offset != NULL) { 6237 assert(src_offset != NULL && dest_offset != NULL, ""); 6238 src_start = array_element_address(src, src_offset, T_BYTE); 6239 dest_start = array_element_address(dest, dest_offset, T_BYTE); 6240 } 6241 6242 // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object 6243 // (because of the predicated logic executed earlier). 6244 // so we cast it here safely. 6245 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java 6246 Node* embeddedCipherObj = load_field_from_object(counterMode_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6247 if (embeddedCipherObj == NULL) return false; 6248 // cast it to what we know it will be at runtime 6249 const TypeInstPtr* tinst = _gvn.type(counterMode_object)->isa_instptr(); 6250 assert(tinst != NULL, "CTR obj is null"); 6251 assert(tinst->klass()->is_loaded(), "CTR obj is not loaded"); 6252 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6253 assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded"); 6254 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6255 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt); 6256 const TypeOopPtr* xtype = aklass->as_instance_type()->cast_to_ptr_type(TypePtr::NotNull); 6257 Node* aescrypt_object = new CheckCastPPNode(control(), embeddedCipherObj, xtype); 6258 aescrypt_object = _gvn.transform(aescrypt_object); 6259 // we need to get the start of the aescrypt_object's expanded key array 6260 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); 6261 if (k_start == NULL) return false; 6262 // similarly, get the start address of the r vector 6263 Node* obj_counter = load_field_from_object(counterMode_object, "counter", "[B"); 6264 if (obj_counter == NULL) return false; 6265 Node* cnt_start = array_element_address(obj_counter, intcon(0), T_BYTE); 6266 6267 Node* saved_encCounter = load_field_from_object(counterMode_object, "encryptedCounter", "[B"); 6268 if (saved_encCounter == NULL) return false; 6269 Node* saved_encCounter_start = array_element_address(saved_encCounter, intcon(0), T_BYTE); 6270 Node* used = field_address_from_object(counterMode_object, "used", "I", /*is_exact*/ false); 6271 6272 // Call the stub, passing src_start, dest_start, k_start, r_start and src_len 6273 Node* ctrCrypt = make_runtime_call(RC_LEAF|RC_NO_FP, 6274 OptoRuntime::counterMode_aescrypt_Type(), 6275 stubAddr, stubName, TypePtr::BOTTOM, 6276 src_start, dest_start, k_start, cnt_start, len, saved_encCounter_start, used); 6277 6278 // return cipher length (int) 6279 Node* retvalue = _gvn.transform(new ProjNode(ctrCrypt, TypeFunc::Parms)); 6280 set_result(retvalue); 6281 return true; 6282 } 6283 6284 //------------------------------get_key_start_from_aescrypt_object----------------------- 6285 Node * LibraryCallKit::get_key_start_from_aescrypt_object(Node *aescrypt_object) { 6286 #if defined(PPC64) || defined(S390) 6287 // MixColumns for decryption can be reduced by preprocessing MixColumns with round keys. 6288 // Intel's extension is based on this optimization and AESCrypt generates round keys by preprocessing MixColumns. 6289 // However, ppc64 vncipher processes MixColumns and requires the same round keys with encryption. 6290 // The ppc64 stubs of encryption and decryption use the same round keys (sessionK[0]). 6291 Node* objSessionK = load_field_from_object(aescrypt_object, "sessionK", "[[I"); 6292 assert (objSessionK != NULL, "wrong version of com.sun.crypto.provider.AESCrypt"); 6293 if (objSessionK == NULL) { 6294 return (Node *) NULL; 6295 } 6296 Node* objAESCryptKey = load_array_element(objSessionK, intcon(0), TypeAryPtr::OOPS, /* set_ctrl */ true); 6297 #else 6298 Node* objAESCryptKey = load_field_from_object(aescrypt_object, "K", "[I"); 6299 #endif // PPC64 6300 assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt"); 6301 if (objAESCryptKey == NULL) return (Node *) NULL; 6302 6303 // now have the array, need to get the start address of the K array 6304 Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT); 6305 return k_start; 6306 } 6307 6308 //----------------------------inline_cipherBlockChaining_AESCrypt_predicate---------------------------- 6309 // Return node representing slow path of predicate check. 6310 // the pseudo code we want to emulate with this predicate is: 6311 // for encryption: 6312 // if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath 6313 // for decryption: 6314 // if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath 6315 // note cipher==plain is more conservative than the original java code but that's OK 6316 // 6317 Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting) { 6318 // The receiver was checked for NULL already. 6319 Node* objCBC = argument(0); 6320 6321 Node* src = argument(1); 6322 Node* dest = argument(4); 6323 6324 // Load embeddedCipher field of CipherBlockChaining object. 6325 Node* embeddedCipherObj = load_field_from_object(objCBC, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6326 6327 // get AESCrypt klass for instanceOf check 6328 // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point 6329 // will have same classloader as CipherBlockChaining object 6330 const TypeInstPtr* tinst = _gvn.type(objCBC)->isa_instptr(); 6331 assert(tinst != NULL, "CBCobj is null"); 6332 assert(tinst->klass()->is_loaded(), "CBCobj is not loaded"); 6333 6334 // we want to do an instanceof comparison against the AESCrypt class 6335 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6336 if (!klass_AESCrypt->is_loaded()) { 6337 // if AESCrypt is not even loaded, we never take the intrinsic fast path 6338 Node* ctrl = control(); 6339 set_control(top()); // no regular fast path 6340 return ctrl; 6341 } 6342 6343 src = must_be_not_null(src, true); 6344 dest = must_be_not_null(dest, true); 6345 6346 // Resolve oops to stable for CmpP below. 6347 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6348 6349 Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt))); 6350 Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); 6351 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); 6352 6353 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); 6354 6355 // for encryption, we are done 6356 if (!decrypting) 6357 return instof_false; // even if it is NULL 6358 6359 // for decryption, we need to add a further check to avoid 6360 // taking the intrinsic path when cipher and plain are the same 6361 // see the original java code for why. 6362 RegionNode* region = new RegionNode(3); 6363 region->init_req(1, instof_false); 6364 6365 Node* cmp_src_dest = _gvn.transform(new CmpPNode(src, dest)); 6366 Node* bool_src_dest = _gvn.transform(new BoolNode(cmp_src_dest, BoolTest::eq)); 6367 Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN); 6368 region->init_req(2, src_dest_conjoint); 6369 6370 record_for_igvn(region); 6371 return _gvn.transform(region); 6372 } 6373 6374 //----------------------------inline_electronicCodeBook_AESCrypt_predicate---------------------------- 6375 // Return node representing slow path of predicate check. 6376 // the pseudo code we want to emulate with this predicate is: 6377 // for encryption: 6378 // if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath 6379 // for decryption: 6380 // if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath 6381 // note cipher==plain is more conservative than the original java code but that's OK 6382 // 6383 Node* LibraryCallKit::inline_electronicCodeBook_AESCrypt_predicate(bool decrypting) { 6384 // The receiver was checked for NULL already. 6385 Node* objECB = argument(0); 6386 6387 // Load embeddedCipher field of ElectronicCodeBook object. 6388 Node* embeddedCipherObj = load_field_from_object(objECB, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6389 6390 // get AESCrypt klass for instanceOf check 6391 // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point 6392 // will have same classloader as ElectronicCodeBook object 6393 const TypeInstPtr* tinst = _gvn.type(objECB)->isa_instptr(); 6394 assert(tinst != NULL, "ECBobj is null"); 6395 assert(tinst->klass()->is_loaded(), "ECBobj is not loaded"); 6396 6397 // we want to do an instanceof comparison against the AESCrypt class 6398 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6399 if (!klass_AESCrypt->is_loaded()) { 6400 // if AESCrypt is not even loaded, we never take the intrinsic fast path 6401 Node* ctrl = control(); 6402 set_control(top()); // no regular fast path 6403 return ctrl; 6404 } 6405 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6406 6407 Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt))); 6408 Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); 6409 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); 6410 6411 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); 6412 6413 // for encryption, we are done 6414 if (!decrypting) 6415 return instof_false; // even if it is NULL 6416 6417 // for decryption, we need to add a further check to avoid 6418 // taking the intrinsic path when cipher and plain are the same 6419 // see the original java code for why. 6420 RegionNode* region = new RegionNode(3); 6421 region->init_req(1, instof_false); 6422 Node* src = argument(1); 6423 Node* dest = argument(4); 6424 Node* cmp_src_dest = _gvn.transform(new CmpPNode(src, dest)); 6425 Node* bool_src_dest = _gvn.transform(new BoolNode(cmp_src_dest, BoolTest::eq)); 6426 Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN); 6427 region->init_req(2, src_dest_conjoint); 6428 6429 record_for_igvn(region); 6430 return _gvn.transform(region); 6431 } 6432 6433 //----------------------------inline_counterMode_AESCrypt_predicate---------------------------- 6434 // Return node representing slow path of predicate check. 6435 // the pseudo code we want to emulate with this predicate is: 6436 // for encryption: 6437 // if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath 6438 // for decryption: 6439 // if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath 6440 // note cipher==plain is more conservative than the original java code but that's OK 6441 // 6442 6443 Node* LibraryCallKit::inline_counterMode_AESCrypt_predicate() { 6444 // The receiver was checked for NULL already. 6445 Node* objCTR = argument(0); 6446 6447 // Load embeddedCipher field of CipherBlockChaining object. 6448 Node* embeddedCipherObj = load_field_from_object(objCTR, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6449 6450 // get AESCrypt klass for instanceOf check 6451 // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point 6452 // will have same classloader as CipherBlockChaining object 6453 const TypeInstPtr* tinst = _gvn.type(objCTR)->isa_instptr(); 6454 assert(tinst != NULL, "CTRobj is null"); 6455 assert(tinst->klass()->is_loaded(), "CTRobj is not loaded"); 6456 6457 // we want to do an instanceof comparison against the AESCrypt class 6458 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6459 if (!klass_AESCrypt->is_loaded()) { 6460 // if AESCrypt is not even loaded, we never take the intrinsic fast path 6461 Node* ctrl = control(); 6462 set_control(top()); // no regular fast path 6463 return ctrl; 6464 } 6465 6466 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6467 Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt))); 6468 Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); 6469 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); 6470 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); 6471 6472 return instof_false; // even if it is NULL 6473 } 6474 6475 //------------------------------inline_ghash_processBlocks 6476 bool LibraryCallKit::inline_ghash_processBlocks() { 6477 address stubAddr; 6478 const char *stubName; 6479 assert(UseGHASHIntrinsics, "need GHASH intrinsics support"); 6480 6481 stubAddr = StubRoutines::ghash_processBlocks(); 6482 stubName = "ghash_processBlocks"; 6483 6484 Node* data = argument(0); 6485 Node* offset = argument(1); 6486 Node* len = argument(2); 6487 Node* state = argument(3); 6488 Node* subkeyH = argument(4); 6489 6490 state = must_be_not_null(state, true); 6491 subkeyH = must_be_not_null(subkeyH, true); 6492 data = must_be_not_null(data, true); 6493 6494 Node* state_start = array_element_address(state, intcon(0), T_LONG); 6495 assert(state_start, "state is NULL"); 6496 Node* subkeyH_start = array_element_address(subkeyH, intcon(0), T_LONG); 6497 assert(subkeyH_start, "subkeyH is NULL"); 6498 Node* data_start = array_element_address(data, offset, T_BYTE); 6499 assert(data_start, "data is NULL"); 6500 6501 Node* ghash = make_runtime_call(RC_LEAF|RC_NO_FP, 6502 OptoRuntime::ghash_processBlocks_Type(), 6503 stubAddr, stubName, TypePtr::BOTTOM, 6504 state_start, subkeyH_start, data_start, len); 6505 return true; 6506 } 6507 6508 bool LibraryCallKit::inline_base64_encodeBlock() { 6509 address stubAddr; 6510 const char *stubName; 6511 assert(UseBASE64Intrinsics, "need Base64 intrinsics support"); 6512 assert(callee()->signature()->size() == 6, "base64_encodeBlock has 6 parameters"); 6513 stubAddr = StubRoutines::base64_encodeBlock(); 6514 stubName = "encodeBlock"; 6515 6516 if (!stubAddr) return false; 6517 Node* base64obj = argument(0); 6518 Node* src = argument(1); 6519 Node* offset = argument(2); 6520 Node* len = argument(3); 6521 Node* dest = argument(4); 6522 Node* dp = argument(5); 6523 Node* isURL = argument(6); 6524 6525 src = must_be_not_null(src, true); 6526 dest = must_be_not_null(dest, true); 6527 6528 Node* src_start = array_element_address(src, intcon(0), T_BYTE); 6529 assert(src_start, "source array is NULL"); 6530 Node* dest_start = array_element_address(dest, intcon(0), T_BYTE); 6531 assert(dest_start, "destination array is NULL"); 6532 6533 Node* base64 = make_runtime_call(RC_LEAF, 6534 OptoRuntime::base64_encodeBlock_Type(), 6535 stubAddr, stubName, TypePtr::BOTTOM, 6536 src_start, offset, len, dest_start, dp, isURL); 6537 return true; 6538 } 6539 6540 bool LibraryCallKit::inline_base64_decodeBlock() { 6541 address stubAddr; 6542 const char *stubName; 6543 assert(UseBASE64Intrinsics, "need Base64 intrinsics support"); 6544 assert(callee()->signature()->size() == 7, "base64_decodeBlock has 7 parameters"); 6545 stubAddr = StubRoutines::base64_decodeBlock(); 6546 stubName = "decodeBlock"; 6547 6548 if (!stubAddr) return false; 6549 Node* base64obj = argument(0); 6550 Node* src = argument(1); 6551 Node* src_offset = argument(2); 6552 Node* len = argument(3); 6553 Node* dest = argument(4); 6554 Node* dest_offset = argument(5); 6555 Node* isURL = argument(6); 6556 Node* isMIME = argument(7); 6557 6558 src = must_be_not_null(src, true); 6559 dest = must_be_not_null(dest, true); 6560 6561 Node* src_start = array_element_address(src, intcon(0), T_BYTE); 6562 assert(src_start, "source array is NULL"); 6563 Node* dest_start = array_element_address(dest, intcon(0), T_BYTE); 6564 assert(dest_start, "destination array is NULL"); 6565 6566 Node* call = make_runtime_call(RC_LEAF, 6567 OptoRuntime::base64_decodeBlock_Type(), 6568 stubAddr, stubName, TypePtr::BOTTOM, 6569 src_start, src_offset, len, dest_start, dest_offset, isURL, isMIME); 6570 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 6571 set_result(result); 6572 return true; 6573 } 6574 6575 //------------------------------inline_digestBase_implCompress----------------------- 6576 // 6577 // Calculate MD5 for single-block byte[] array. 6578 // void com.sun.security.provider.MD5.implCompress(byte[] buf, int ofs) 6579 // 6580 // Calculate SHA (i.e., SHA-1) for single-block byte[] array. 6581 // void com.sun.security.provider.SHA.implCompress(byte[] buf, int ofs) 6582 // 6583 // Calculate SHA2 (i.e., SHA-244 or SHA-256) for single-block byte[] array. 6584 // void com.sun.security.provider.SHA2.implCompress(byte[] buf, int ofs) 6585 // 6586 // Calculate SHA5 (i.e., SHA-384 or SHA-512) for single-block byte[] array. 6587 // void com.sun.security.provider.SHA5.implCompress(byte[] buf, int ofs) 6588 // 6589 // Calculate SHA3 (i.e., SHA3-224 or SHA3-256 or SHA3-384 or SHA3-512) for single-block byte[] array. 6590 // void com.sun.security.provider.SHA3.implCompress(byte[] buf, int ofs) 6591 // 6592 bool LibraryCallKit::inline_digestBase_implCompress(vmIntrinsics::ID id) { 6593 assert(callee()->signature()->size() == 2, "sha_implCompress has 2 parameters"); 6594 6595 Node* digestBase_obj = argument(0); 6596 Node* src = argument(1); // type oop 6597 Node* ofs = argument(2); // type int 6598 6599 const Type* src_type = src->Value(&_gvn); 6600 const TypeAryPtr* top_src = src_type->isa_aryptr(); 6601 if (top_src == NULL || top_src->klass() == NULL) { 6602 // failed array check 6603 return false; 6604 } 6605 // Figure out the size and type of the elements we will be copying. 6606 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 6607 if (src_elem != T_BYTE) { 6608 return false; 6609 } 6610 // 'src_start' points to src array + offset 6611 src = must_be_not_null(src, true); 6612 Node* src_start = array_element_address(src, ofs, src_elem); 6613 Node* state = NULL; 6614 Node* digest_length = NULL; 6615 address stubAddr; 6616 const char *stubName; 6617 6618 switch(id) { 6619 case vmIntrinsics::_md5_implCompress: 6620 assert(UseMD5Intrinsics, "need MD5 instruction support"); 6621 state = get_state_from_digest_object(digestBase_obj, T_INT); 6622 stubAddr = StubRoutines::md5_implCompress(); 6623 stubName = "md5_implCompress"; 6624 break; 6625 case vmIntrinsics::_sha_implCompress: 6626 assert(UseSHA1Intrinsics, "need SHA1 instruction support"); 6627 state = get_state_from_digest_object(digestBase_obj, T_INT); 6628 stubAddr = StubRoutines::sha1_implCompress(); 6629 stubName = "sha1_implCompress"; 6630 break; 6631 case vmIntrinsics::_sha2_implCompress: 6632 assert(UseSHA256Intrinsics, "need SHA256 instruction support"); 6633 state = get_state_from_digest_object(digestBase_obj, T_INT); 6634 stubAddr = StubRoutines::sha256_implCompress(); 6635 stubName = "sha256_implCompress"; 6636 break; 6637 case vmIntrinsics::_sha5_implCompress: 6638 assert(UseSHA512Intrinsics, "need SHA512 instruction support"); 6639 state = get_state_from_digest_object(digestBase_obj, T_LONG); 6640 stubAddr = StubRoutines::sha512_implCompress(); 6641 stubName = "sha512_implCompress"; 6642 break; 6643 case vmIntrinsics::_sha3_implCompress: 6644 assert(UseSHA3Intrinsics, "need SHA3 instruction support"); 6645 state = get_state_from_digest_object(digestBase_obj, T_BYTE); 6646 stubAddr = StubRoutines::sha3_implCompress(); 6647 stubName = "sha3_implCompress"; 6648 digest_length = get_digest_length_from_digest_object(digestBase_obj); 6649 if (digest_length == NULL) return false; 6650 break; 6651 default: 6652 fatal_unexpected_iid(id); 6653 return false; 6654 } 6655 if (state == NULL) return false; 6656 6657 assert(stubAddr != NULL, "Stub is generated"); 6658 if (stubAddr == NULL) return false; 6659 6660 // Call the stub. 6661 Node* call; 6662 if (digest_length == NULL) { 6663 call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::digestBase_implCompress_Type(false), 6664 stubAddr, stubName, TypePtr::BOTTOM, 6665 src_start, state); 6666 } else { 6667 call = make_runtime_call(RC_LEAF|RC_NO_FP, OptoRuntime::digestBase_implCompress_Type(true), 6668 stubAddr, stubName, TypePtr::BOTTOM, 6669 src_start, state, digest_length); 6670 } 6671 6672 return true; 6673 } 6674 6675 //------------------------------inline_digestBase_implCompressMB----------------------- 6676 // 6677 // Calculate MD5/SHA/SHA2/SHA5/SHA3 for multi-block byte[] array. 6678 // int com.sun.security.provider.DigestBase.implCompressMultiBlock(byte[] b, int ofs, int limit) 6679 // 6680 bool LibraryCallKit::inline_digestBase_implCompressMB(int predicate) { 6681 assert(UseMD5Intrinsics || UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics || UseSHA3Intrinsics, 6682 "need MD5/SHA1/SHA256/SHA512/SHA3 instruction support"); 6683 assert((uint)predicate < 5, "sanity"); 6684 assert(callee()->signature()->size() == 3, "digestBase_implCompressMB has 3 parameters"); 6685 6686 Node* digestBase_obj = argument(0); // The receiver was checked for NULL already. 6687 Node* src = argument(1); // byte[] array 6688 Node* ofs = argument(2); // type int 6689 Node* limit = argument(3); // type int 6690 6691 const Type* src_type = src->Value(&_gvn); 6692 const TypeAryPtr* top_src = src_type->isa_aryptr(); 6693 if (top_src == NULL || top_src->klass() == NULL) { 6694 // failed array check 6695 return false; 6696 } 6697 // Figure out the size and type of the elements we will be copying. 6698 BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); 6699 if (src_elem != T_BYTE) { 6700 return false; 6701 } 6702 // 'src_start' points to src array + offset 6703 src = must_be_not_null(src, false); 6704 Node* src_start = array_element_address(src, ofs, src_elem); 6705 6706 const char* klass_digestBase_name = NULL; 6707 const char* stub_name = NULL; 6708 address stub_addr = NULL; 6709 BasicType elem_type = T_INT; 6710 6711 switch (predicate) { 6712 case 0: 6713 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_md5_implCompress)) { 6714 klass_digestBase_name = "sun/security/provider/MD5"; 6715 stub_name = "md5_implCompressMB"; 6716 stub_addr = StubRoutines::md5_implCompressMB(); 6717 } 6718 break; 6719 case 1: 6720 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_sha_implCompress)) { 6721 klass_digestBase_name = "sun/security/provider/SHA"; 6722 stub_name = "sha1_implCompressMB"; 6723 stub_addr = StubRoutines::sha1_implCompressMB(); 6724 } 6725 break; 6726 case 2: 6727 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_sha2_implCompress)) { 6728 klass_digestBase_name = "sun/security/provider/SHA2"; 6729 stub_name = "sha256_implCompressMB"; 6730 stub_addr = StubRoutines::sha256_implCompressMB(); 6731 } 6732 break; 6733 case 3: 6734 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_sha5_implCompress)) { 6735 klass_digestBase_name = "sun/security/provider/SHA5"; 6736 stub_name = "sha512_implCompressMB"; 6737 stub_addr = StubRoutines::sha512_implCompressMB(); 6738 elem_type = T_LONG; 6739 } 6740 break; 6741 case 4: 6742 if (vmIntrinsics::is_intrinsic_available(vmIntrinsics::_sha3_implCompress)) { 6743 klass_digestBase_name = "sun/security/provider/SHA3"; 6744 stub_name = "sha3_implCompressMB"; 6745 stub_addr = StubRoutines::sha3_implCompressMB(); 6746 elem_type = T_BYTE; 6747 } 6748 break; 6749 default: 6750 fatal("unknown DigestBase intrinsic predicate: %d", predicate); 6751 } 6752 if (klass_digestBase_name != NULL) { 6753 assert(stub_addr != NULL, "Stub is generated"); 6754 if (stub_addr == NULL) return false; 6755 6756 // get DigestBase klass to lookup for SHA klass 6757 const TypeInstPtr* tinst = _gvn.type(digestBase_obj)->isa_instptr(); 6758 assert(tinst != NULL, "digestBase_obj is not instance???"); 6759 assert(tinst->klass()->is_loaded(), "DigestBase is not loaded"); 6760 6761 ciKlass* klass_digestBase = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make(klass_digestBase_name)); 6762 assert(klass_digestBase->is_loaded(), "predicate checks that this class is loaded"); 6763 ciInstanceKlass* instklass_digestBase = klass_digestBase->as_instance_klass(); 6764 return inline_digestBase_implCompressMB(digestBase_obj, instklass_digestBase, elem_type, stub_addr, stub_name, src_start, ofs, limit); 6765 } 6766 return false; 6767 } 6768 6769 //------------------------------inline_digestBase_implCompressMB----------------------- 6770 bool LibraryCallKit::inline_digestBase_implCompressMB(Node* digestBase_obj, ciInstanceKlass* instklass_digestBase, 6771 BasicType elem_type, address stubAddr, const char *stubName, 6772 Node* src_start, Node* ofs, Node* limit) { 6773 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_digestBase); 6774 const TypeOopPtr* xtype = aklass->as_instance_type()->cast_to_ptr_type(TypePtr::NotNull); 6775 Node* digest_obj = new CheckCastPPNode(control(), digestBase_obj, xtype); 6776 digest_obj = _gvn.transform(digest_obj); 6777 6778 Node* state = get_state_from_digest_object(digest_obj, elem_type); 6779 if (state == NULL) return false; 6780 6781 Node* digest_length = NULL; 6782 if (strcmp("sha3_implCompressMB", stubName) == 0) { 6783 digest_length = get_digest_length_from_digest_object(digest_obj); 6784 if (digest_length == NULL) return false; 6785 } 6786 6787 // Call the stub. 6788 Node* call; 6789 if (digest_length == NULL) { 6790 call = make_runtime_call(RC_LEAF|RC_NO_FP, 6791 OptoRuntime::digestBase_implCompressMB_Type(false), 6792 stubAddr, stubName, TypePtr::BOTTOM, 6793 src_start, state, ofs, limit); 6794 } else { 6795 call = make_runtime_call(RC_LEAF|RC_NO_FP, 6796 OptoRuntime::digestBase_implCompressMB_Type(true), 6797 stubAddr, stubName, TypePtr::BOTTOM, 6798 src_start, state, digest_length, ofs, limit); 6799 } 6800 6801 // return ofs (int) 6802 Node* result = _gvn.transform(new ProjNode(call, TypeFunc::Parms)); 6803 set_result(result); 6804 6805 return true; 6806 } 6807 6808 //------------------------------inline_galoisCounterMode_AESCrypt----------------------- 6809 bool LibraryCallKit::inline_galoisCounterMode_AESCrypt() { 6810 assert(UseAES, "need AES instruction support"); 6811 address stubAddr = NULL; 6812 const char *stubName = NULL; 6813 stubAddr = StubRoutines::galoisCounterMode_AESCrypt(); 6814 stubName = "galoisCounterMode_AESCrypt"; 6815 6816 if (stubAddr == NULL) return false; 6817 6818 Node* in = argument(0); 6819 Node* inOfs = argument(1); 6820 Node* len = argument(2); 6821 Node* ct = argument(3); 6822 Node* ctOfs = argument(4); 6823 Node* out = argument(5); 6824 Node* outOfs = argument(6); 6825 Node* gctr_object = argument(7); 6826 Node* ghash_object = argument(8); 6827 6828 // (1) in, ct and out are arrays. 6829 const Type* in_type = in->Value(&_gvn); 6830 const Type* ct_type = ct->Value(&_gvn); 6831 const Type* out_type = out->Value(&_gvn); 6832 const TypeAryPtr* top_in = in_type->isa_aryptr(); 6833 const TypeAryPtr* top_ct = ct_type->isa_aryptr(); 6834 const TypeAryPtr* top_out = out_type->isa_aryptr(); 6835 assert(top_in != NULL && top_in->klass() != NULL && 6836 top_ct != NULL && top_ct->klass() != NULL && 6837 top_out != NULL && top_out->klass() != NULL, "args are strange"); 6838 6839 // checks are the responsibility of the caller 6840 Node* in_start = in; 6841 Node* ct_start = ct; 6842 Node* out_start = out; 6843 if (inOfs != NULL || ctOfs != NULL || outOfs != NULL) { 6844 assert(inOfs != NULL && ctOfs != NULL && outOfs != NULL, ""); 6845 in_start = array_element_address(in, inOfs, T_BYTE); 6846 ct_start = array_element_address(ct, ctOfs, T_BYTE); 6847 out_start = array_element_address(out, outOfs, T_BYTE); 6848 } 6849 6850 // if we are in this set of code, we "know" the embeddedCipher is an AESCrypt object 6851 // (because of the predicated logic executed earlier). 6852 // so we cast it here safely. 6853 // this requires a newer class file that has this array as littleEndian ints, otherwise we revert to java 6854 Node* embeddedCipherObj = load_field_from_object(gctr_object, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6855 Node* counter = load_field_from_object(gctr_object, "counter", "[B"); 6856 Node* subkeyHtbl = load_field_from_object(ghash_object, "subkeyHtbl", "[J"); 6857 Node* state = load_field_from_object(ghash_object, "state", "[J"); 6858 6859 if (embeddedCipherObj == NULL || counter == NULL || subkeyHtbl == NULL || state == NULL) { 6860 return false; 6861 } 6862 // cast it to what we know it will be at runtime 6863 const TypeInstPtr* tinst = _gvn.type(gctr_object)->isa_instptr(); 6864 assert(tinst != NULL, "GCTR obj is null"); 6865 assert(tinst->klass()->is_loaded(), "GCTR obj is not loaded"); 6866 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6867 assert(klass_AESCrypt->is_loaded(), "predicate checks that this class is loaded"); 6868 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6869 const TypeKlassPtr* aklass = TypeKlassPtr::make(instklass_AESCrypt); 6870 const TypeOopPtr* xtype = aklass->as_instance_type(); 6871 Node* aescrypt_object = new CheckCastPPNode(control(), embeddedCipherObj, xtype); 6872 aescrypt_object = _gvn.transform(aescrypt_object); 6873 // we need to get the start of the aescrypt_object's expanded key array 6874 Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); 6875 if (k_start == NULL) return false; 6876 // similarly, get the start address of the r vector 6877 Node* cnt_start = array_element_address(counter, intcon(0), T_BYTE); 6878 Node* state_start = array_element_address(state, intcon(0), T_LONG); 6879 Node* subkeyHtbl_start = array_element_address(subkeyHtbl, intcon(0), T_LONG); 6880 6881 6882 // Call the stub, passing params 6883 Node* gcmCrypt = make_runtime_call(RC_LEAF|RC_NO_FP, 6884 OptoRuntime::galoisCounterMode_aescrypt_Type(), 6885 stubAddr, stubName, TypePtr::BOTTOM, 6886 in_start, len, ct_start, out_start, k_start, state_start, subkeyHtbl_start, cnt_start); 6887 6888 // return cipher length (int) 6889 Node* retvalue = _gvn.transform(new ProjNode(gcmCrypt, TypeFunc::Parms)); 6890 set_result(retvalue); 6891 6892 return true; 6893 } 6894 6895 //----------------------------inline_galoisCounterMode_AESCrypt_predicate---------------------------- 6896 // Return node representing slow path of predicate check. 6897 // the pseudo code we want to emulate with this predicate is: 6898 // for encryption: 6899 // if (embeddedCipherObj instanceof AESCrypt) do_intrinsic, else do_javapath 6900 // for decryption: 6901 // if ((embeddedCipherObj instanceof AESCrypt) && (cipher!=plain)) do_intrinsic, else do_javapath 6902 // note cipher==plain is more conservative than the original java code but that's OK 6903 // 6904 6905 Node* LibraryCallKit::inline_galoisCounterMode_AESCrypt_predicate() { 6906 // The receiver was checked for NULL already. 6907 Node* objGCTR = argument(7); 6908 // Load embeddedCipher field of GCTR object. 6909 Node* embeddedCipherObj = load_field_from_object(objGCTR, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;"); 6910 assert(embeddedCipherObj != NULL, "embeddedCipherObj is null"); 6911 6912 // get AESCrypt klass for instanceOf check 6913 // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point 6914 // will have same classloader as CipherBlockChaining object 6915 const TypeInstPtr* tinst = _gvn.type(objGCTR)->isa_instptr(); 6916 assert(tinst != NULL, "GCTR obj is null"); 6917 assert(tinst->klass()->is_loaded(), "GCTR obj is not loaded"); 6918 6919 // we want to do an instanceof comparison against the AESCrypt class 6920 ciKlass* klass_AESCrypt = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make("com/sun/crypto/provider/AESCrypt")); 6921 if (!klass_AESCrypt->is_loaded()) { 6922 // if AESCrypt is not even loaded, we never take the intrinsic fast path 6923 Node* ctrl = control(); 6924 set_control(top()); // no regular fast path 6925 return ctrl; 6926 } 6927 6928 ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); 6929 Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt))); 6930 Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); 6931 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); 6932 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); 6933 6934 return instof_false; // even if it is NULL 6935 } 6936 6937 //------------------------------get_state_from_digest_object----------------------- 6938 Node * LibraryCallKit::get_state_from_digest_object(Node *digest_object, BasicType elem_type) { 6939 const char* state_type; 6940 switch (elem_type) { 6941 case T_BYTE: state_type = "[B"; break; 6942 case T_INT: state_type = "[I"; break; 6943 case T_LONG: state_type = "[J"; break; 6944 default: ShouldNotReachHere(); 6945 } 6946 Node* digest_state = load_field_from_object(digest_object, "state", state_type); 6947 assert (digest_state != NULL, "wrong version of sun.security.provider.MD5/SHA/SHA2/SHA5/SHA3"); 6948 if (digest_state == NULL) return (Node *) NULL; 6949 6950 // now have the array, need to get the start address of the state array 6951 Node* state = array_element_address(digest_state, intcon(0), elem_type); 6952 return state; 6953 } 6954 6955 //------------------------------get_digest_length_from_sha3_object---------------------------------- 6956 Node * LibraryCallKit::get_digest_length_from_digest_object(Node *digest_object) { 6957 Node* digest_length = load_field_from_object(digest_object, "digestLength", "I"); 6958 assert (digest_length != NULL, "sanity"); 6959 return digest_length; 6960 } 6961 6962 //----------------------------inline_digestBase_implCompressMB_predicate---------------------------- 6963 // Return node representing slow path of predicate check. 6964 // the pseudo code we want to emulate with this predicate is: 6965 // if (digestBaseObj instanceof MD5/SHA/SHA2/SHA5/SHA3) do_intrinsic, else do_javapath 6966 // 6967 Node* LibraryCallKit::inline_digestBase_implCompressMB_predicate(int predicate) { 6968 assert(UseMD5Intrinsics || UseSHA1Intrinsics || UseSHA256Intrinsics || UseSHA512Intrinsics || UseSHA3Intrinsics, 6969 "need MD5/SHA1/SHA256/SHA512/SHA3 instruction support"); 6970 assert((uint)predicate < 5, "sanity"); 6971 6972 // The receiver was checked for NULL already. 6973 Node* digestBaseObj = argument(0); 6974 6975 // get DigestBase klass for instanceOf check 6976 const TypeInstPtr* tinst = _gvn.type(digestBaseObj)->isa_instptr(); 6977 assert(tinst != NULL, "digestBaseObj is null"); 6978 assert(tinst->klass()->is_loaded(), "DigestBase is not loaded"); 6979 6980 const char* klass_name = NULL; 6981 switch (predicate) { 6982 case 0: 6983 if (UseMD5Intrinsics) { 6984 // we want to do an instanceof comparison against the MD5 class 6985 klass_name = "sun/security/provider/MD5"; 6986 } 6987 break; 6988 case 1: 6989 if (UseSHA1Intrinsics) { 6990 // we want to do an instanceof comparison against the SHA class 6991 klass_name = "sun/security/provider/SHA"; 6992 } 6993 break; 6994 case 2: 6995 if (UseSHA256Intrinsics) { 6996 // we want to do an instanceof comparison against the SHA2 class 6997 klass_name = "sun/security/provider/SHA2"; 6998 } 6999 break; 7000 case 3: 7001 if (UseSHA512Intrinsics) { 7002 // we want to do an instanceof comparison against the SHA5 class 7003 klass_name = "sun/security/provider/SHA5"; 7004 } 7005 break; 7006 case 4: 7007 if (UseSHA3Intrinsics) { 7008 // we want to do an instanceof comparison against the SHA3 class 7009 klass_name = "sun/security/provider/SHA3"; 7010 } 7011 break; 7012 default: 7013 fatal("unknown SHA intrinsic predicate: %d", predicate); 7014 } 7015 7016 ciKlass* klass = NULL; 7017 if (klass_name != NULL) { 7018 klass = tinst->klass()->as_instance_klass()->find_klass(ciSymbol::make(klass_name)); 7019 } 7020 if ((klass == NULL) || !klass->is_loaded()) { 7021 // if none of MD5/SHA/SHA2/SHA5 is loaded, we never take the intrinsic fast path 7022 Node* ctrl = control(); 7023 set_control(top()); // no intrinsic path 7024 return ctrl; 7025 } 7026 ciInstanceKlass* instklass = klass->as_instance_klass(); 7027 7028 Node* instof = gen_instanceof(digestBaseObj, makecon(TypeKlassPtr::make(instklass))); 7029 Node* cmp_instof = _gvn.transform(new CmpINode(instof, intcon(1))); 7030 Node* bool_instof = _gvn.transform(new BoolNode(cmp_instof, BoolTest::ne)); 7031 Node* instof_false = generate_guard(bool_instof, NULL, PROB_MIN); 7032 7033 return instof_false; // even if it is NULL 7034 } 7035 7036 //-------------inline_fma----------------------------------- 7037 bool LibraryCallKit::inline_fma(vmIntrinsics::ID id) { 7038 Node *a = NULL; 7039 Node *b = NULL; 7040 Node *c = NULL; 7041 Node* result = NULL; 7042 switch (id) { 7043 case vmIntrinsics::_fmaD: 7044 assert(callee()->signature()->size() == 6, "fma has 3 parameters of size 2 each."); 7045 // no receiver since it is static method 7046 a = round_double_node(argument(0)); 7047 b = round_double_node(argument(2)); 7048 c = round_double_node(argument(4)); 7049 result = _gvn.transform(new FmaDNode(control(), a, b, c)); 7050 break; 7051 case vmIntrinsics::_fmaF: 7052 assert(callee()->signature()->size() == 3, "fma has 3 parameters of size 1 each."); 7053 a = argument(0); 7054 b = argument(1); 7055 c = argument(2); 7056 result = _gvn.transform(new FmaFNode(control(), a, b, c)); 7057 break; 7058 default: 7059 fatal_unexpected_iid(id); break; 7060 } 7061 set_result(result); 7062 return true; 7063 } 7064 7065 bool LibraryCallKit::inline_character_compare(vmIntrinsics::ID id) { 7066 // argument(0) is receiver 7067 Node* codePoint = argument(1); 7068 Node* n = NULL; 7069 7070 switch (id) { 7071 case vmIntrinsics::_isDigit : 7072 n = new DigitNode(control(), codePoint); 7073 break; 7074 case vmIntrinsics::_isLowerCase : 7075 n = new LowerCaseNode(control(), codePoint); 7076 break; 7077 case vmIntrinsics::_isUpperCase : 7078 n = new UpperCaseNode(control(), codePoint); 7079 break; 7080 case vmIntrinsics::_isWhitespace : 7081 n = new WhitespaceNode(control(), codePoint); 7082 break; 7083 default: 7084 fatal_unexpected_iid(id); 7085 } 7086 7087 set_result(_gvn.transform(n)); 7088 return true; 7089 } 7090 7091 //------------------------------inline_fp_min_max------------------------------ 7092 bool LibraryCallKit::inline_fp_min_max(vmIntrinsics::ID id) { 7093 /* DISABLED BECAUSE METHOD DATA ISN'T COLLECTED PER CALL-SITE, SEE JDK-8015416. 7094 7095 // The intrinsic should be used only when the API branches aren't predictable, 7096 // the last one performing the most important comparison. The following heuristic 7097 // uses the branch statistics to eventually bail out if necessary. 7098 7099 ciMethodData *md = callee()->method_data(); 7100 7101 if ( md != NULL && md->is_mature() && md->invocation_count() > 0 ) { 7102 ciCallProfile cp = caller()->call_profile_at_bci(bci()); 7103 7104 if ( ((double)cp.count()) / ((double)md->invocation_count()) < 0.8 ) { 7105 // Bail out if the call-site didn't contribute enough to the statistics. 7106 return false; 7107 } 7108 7109 uint taken = 0, not_taken = 0; 7110 7111 for (ciProfileData *p = md->first_data(); md->is_valid(p); p = md->next_data(p)) { 7112 if (p->is_BranchData()) { 7113 taken = ((ciBranchData*)p)->taken(); 7114 not_taken = ((ciBranchData*)p)->not_taken(); 7115 } 7116 } 7117 7118 double balance = (((double)taken) - ((double)not_taken)) / ((double)md->invocation_count()); 7119 balance = balance < 0 ? -balance : balance; 7120 if ( balance > 0.2 ) { 7121 // Bail out if the most important branch is predictable enough. 7122 return false; 7123 } 7124 } 7125 */ 7126 7127 Node *a = NULL; 7128 Node *b = NULL; 7129 Node *n = NULL; 7130 switch (id) { 7131 case vmIntrinsics::_maxF: 7132 case vmIntrinsics::_minF: 7133 case vmIntrinsics::_maxF_strict: 7134 case vmIntrinsics::_minF_strict: 7135 assert(callee()->signature()->size() == 2, "minF/maxF has 2 parameters of size 1 each."); 7136 a = argument(0); 7137 b = argument(1); 7138 break; 7139 case vmIntrinsics::_maxD: 7140 case vmIntrinsics::_minD: 7141 case vmIntrinsics::_maxD_strict: 7142 case vmIntrinsics::_minD_strict: 7143 assert(callee()->signature()->size() == 4, "minD/maxD has 2 parameters of size 2 each."); 7144 a = round_double_node(argument(0)); 7145 b = round_double_node(argument(2)); 7146 break; 7147 default: 7148 fatal_unexpected_iid(id); 7149 break; 7150 } 7151 switch (id) { 7152 case vmIntrinsics::_maxF: 7153 case vmIntrinsics::_maxF_strict: 7154 n = new MaxFNode(a, b); 7155 break; 7156 case vmIntrinsics::_minF: 7157 case vmIntrinsics::_minF_strict: 7158 n = new MinFNode(a, b); 7159 break; 7160 case vmIntrinsics::_maxD: 7161 case vmIntrinsics::_maxD_strict: 7162 n = new MaxDNode(a, b); 7163 break; 7164 case vmIntrinsics::_minD: 7165 case vmIntrinsics::_minD_strict: 7166 n = new MinDNode(a, b); 7167 break; 7168 default: 7169 fatal_unexpected_iid(id); 7170 break; 7171 } 7172 set_result(_gvn.transform(n)); 7173 return true; 7174 } 7175 7176 bool LibraryCallKit::inline_profileBoolean() { 7177 Node* counts = argument(1); 7178 const TypeAryPtr* ary = NULL; 7179 ciArray* aobj = NULL; 7180 if (counts->is_Con() 7181 && (ary = counts->bottom_type()->isa_aryptr()) != NULL 7182 && (aobj = ary->const_oop()->as_array()) != NULL 7183 && (aobj->length() == 2)) { 7184 // Profile is int[2] where [0] and [1] correspond to false and true value occurrences respectively. 7185 jint false_cnt = aobj->element_value(0).as_int(); 7186 jint true_cnt = aobj->element_value(1).as_int(); 7187 7188 if (C->log() != NULL) { 7189 C->log()->elem("observe source='profileBoolean' false='%d' true='%d'", 7190 false_cnt, true_cnt); 7191 } 7192 7193 if (false_cnt + true_cnt == 0) { 7194 // According to profile, never executed. 7195 uncommon_trap_exact(Deoptimization::Reason_intrinsic, 7196 Deoptimization::Action_reinterpret); 7197 return true; 7198 } 7199 7200 // result is a boolean (0 or 1) and its profile (false_cnt & true_cnt) 7201 // is a number of each value occurrences. 7202 Node* result = argument(0); 7203 if (false_cnt == 0 || true_cnt == 0) { 7204 // According to profile, one value has been never seen. 7205 int expected_val = (false_cnt == 0) ? 1 : 0; 7206 7207 Node* cmp = _gvn.transform(new CmpINode(result, intcon(expected_val))); 7208 Node* test = _gvn.transform(new BoolNode(cmp, BoolTest::eq)); 7209 7210 IfNode* check = create_and_map_if(control(), test, PROB_ALWAYS, COUNT_UNKNOWN); 7211 Node* fast_path = _gvn.transform(new IfTrueNode(check)); 7212 Node* slow_path = _gvn.transform(new IfFalseNode(check)); 7213 7214 { // Slow path: uncommon trap for never seen value and then reexecute 7215 // MethodHandleImpl::profileBoolean() to bump the count, so JIT knows 7216 // the value has been seen at least once. 7217 PreserveJVMState pjvms(this); 7218 PreserveReexecuteState preexecs(this); 7219 jvms()->set_should_reexecute(true); 7220 7221 set_control(slow_path); 7222 set_i_o(i_o()); 7223 7224 uncommon_trap_exact(Deoptimization::Reason_intrinsic, 7225 Deoptimization::Action_reinterpret); 7226 } 7227 // The guard for never seen value enables sharpening of the result and 7228 // returning a constant. It allows to eliminate branches on the same value 7229 // later on. 7230 set_control(fast_path); 7231 result = intcon(expected_val); 7232 } 7233 // Stop profiling. 7234 // MethodHandleImpl::profileBoolean() has profiling logic in its bytecode. 7235 // By replacing method body with profile data (represented as ProfileBooleanNode 7236 // on IR level) we effectively disable profiling. 7237 // It enables full speed execution once optimized code is generated. 7238 Node* profile = _gvn.transform(new ProfileBooleanNode(result, false_cnt, true_cnt)); 7239 C->record_for_igvn(profile); 7240 set_result(profile); 7241 return true; 7242 } else { 7243 // Continue profiling. 7244 // Profile data isn't available at the moment. So, execute method's bytecode version. 7245 // Usually, when GWT LambdaForms are profiled it means that a stand-alone nmethod 7246 // is compiled and counters aren't available since corresponding MethodHandle 7247 // isn't a compile-time constant. 7248 return false; 7249 } 7250 } 7251 7252 bool LibraryCallKit::inline_isCompileConstant() { 7253 Node* n = argument(0); 7254 set_result(n->is_Con() ? intcon(1) : intcon(0)); 7255 return true; 7256 } 7257 7258 //------------------------------- inline_getObjectSize -------------------------------------- 7259 // 7260 // Calculate the runtime size of the object/array. 7261 // native long sun.instrument.InstrumentationImpl.getObjectSize0(long nativeAgent, Object objectToSize); 7262 // 7263 bool LibraryCallKit::inline_getObjectSize() { 7264 Node* obj = argument(3); 7265 Node* klass_node = load_object_klass(obj); 7266 7267 jint layout_con = Klass::_lh_neutral_value; 7268 Node* layout_val = get_layout_helper(klass_node, layout_con); 7269 int layout_is_con = (layout_val == NULL); 7270 7271 if (layout_is_con) { 7272 // Layout helper is constant, can figure out things at compile time. 7273 7274 if (Klass::layout_helper_is_instance(layout_con)) { 7275 // Instance case: layout_con contains the size itself. 7276 Node *size = longcon(Klass::layout_helper_size_in_bytes(layout_con)); 7277 set_result(size); 7278 } else { 7279 // Array case: size is round(header + element_size*arraylength). 7280 // Since arraylength is different for every array instance, we have to 7281 // compute the whole thing at runtime. 7282 7283 Node* arr_length = load_array_length(obj); 7284 7285 int round_mask = MinObjAlignmentInBytes - 1; 7286 int hsize = Klass::layout_helper_header_size(layout_con); 7287 int eshift = Klass::layout_helper_log2_element_size(layout_con); 7288 7289 if ((round_mask & ~right_n_bits(eshift)) == 0) { 7290 round_mask = 0; // strength-reduce it if it goes away completely 7291 } 7292 assert((hsize & right_n_bits(eshift)) == 0, "hsize is pre-rounded"); 7293 Node* header_size = intcon(hsize + round_mask); 7294 7295 Node* lengthx = ConvI2X(arr_length); 7296 Node* headerx = ConvI2X(header_size); 7297 7298 Node* abody = lengthx; 7299 if (eshift != 0) { 7300 abody = _gvn.transform(new LShiftXNode(lengthx, intcon(eshift))); 7301 } 7302 Node* size = _gvn.transform( new AddXNode(headerx, abody) ); 7303 if (round_mask != 0) { 7304 size = _gvn.transform( new AndXNode(size, MakeConX(~round_mask)) ); 7305 } 7306 size = ConvX2L(size); 7307 set_result(size); 7308 } 7309 } else { 7310 // Layout helper is not constant, need to test for array-ness at runtime. 7311 7312 enum { _instance_path = 1, _array_path, PATH_LIMIT }; 7313 RegionNode* result_reg = new RegionNode(PATH_LIMIT); 7314 PhiNode* result_val = new PhiNode(result_reg, TypeLong::LONG); 7315 record_for_igvn(result_reg); 7316 7317 Node* array_ctl = generate_array_guard(klass_node, NULL); 7318 if (array_ctl != NULL) { 7319 // Array case: size is round(header + element_size*arraylength). 7320 // Since arraylength is different for every array instance, we have to 7321 // compute the whole thing at runtime. 7322 7323 PreserveJVMState pjvms(this); 7324 set_control(array_ctl); 7325 Node* arr_length = load_array_length(obj); 7326 7327 int round_mask = MinObjAlignmentInBytes - 1; 7328 Node* mask = intcon(round_mask); 7329 7330 Node* hss = intcon(Klass::_lh_header_size_shift); 7331 Node* hsm = intcon(Klass::_lh_header_size_mask); 7332 Node* header_size = _gvn.transform(new URShiftINode(layout_val, hss)); 7333 header_size = _gvn.transform(new AndINode(header_size, hsm)); 7334 header_size = _gvn.transform(new AddINode(header_size, mask)); 7335 7336 // There is no need to mask or shift this value. 7337 // The semantics of LShiftINode include an implicit mask to 0x1F. 7338 assert(Klass::_lh_log2_element_size_shift == 0, "use shift in place"); 7339 Node* elem_shift = layout_val; 7340 7341 Node* lengthx = ConvI2X(arr_length); 7342 Node* headerx = ConvI2X(header_size); 7343 7344 Node* abody = _gvn.transform(new LShiftXNode(lengthx, elem_shift)); 7345 Node* size = _gvn.transform(new AddXNode(headerx, abody)); 7346 if (round_mask != 0) { 7347 size = _gvn.transform(new AndXNode(size, MakeConX(~round_mask))); 7348 } 7349 size = ConvX2L(size); 7350 7351 result_reg->init_req(_array_path, control()); 7352 result_val->init_req(_array_path, size); 7353 } 7354 7355 if (!stopped()) { 7356 // Instance case: the layout helper gives us instance size almost directly, 7357 // but we need to mask out the _lh_instance_slow_path_bit. 7358 Node* size = ConvI2X(layout_val); 7359 assert((int) Klass::_lh_instance_slow_path_bit < BytesPerLong, "clear bit"); 7360 Node* mask = MakeConX(~(intptr_t) right_n_bits(LogBytesPerLong)); 7361 size = _gvn.transform(new AndXNode(size, mask)); 7362 size = ConvX2L(size); 7363 7364 result_reg->init_req(_instance_path, control()); 7365 result_val->init_req(_instance_path, size); 7366 } 7367 7368 set_result(result_reg, result_val); 7369 } 7370 7371 return true; 7372 } 7373 7374 //------------------------------- inline_blackhole -------------------------------------- 7375 // 7376 // Make sure all arguments to this node are alive. 7377 // This matches methods that were requested to be blackholed through compile commands. 7378 // 7379 bool LibraryCallKit::inline_blackhole() { 7380 assert(callee()->is_static(), "Should have been checked before: only static methods here"); 7381 assert(callee()->is_empty(), "Should have been checked before: only empty methods here"); 7382 assert(callee()->holder()->is_loaded(), "Should have been checked before: only methods for loaded classes here"); 7383 7384 // Bind call arguments as blackhole arguments to keep them alive 7385 Node* bh = insert_mem_bar(Op_Blackhole); 7386 uint nargs = callee()->arg_size(); 7387 for (uint i = 0; i < nargs; i++) { 7388 bh->add_req(argument(i)); 7389 } 7390 7391 return true; 7392 }