1 /*
   2  * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "ci/ciInlineKlass.hpp"
  26 #include "ci/ciMethodData.hpp"
  27 #include "ci/ciSymbols.hpp"
  28 #include "classfile/vmSymbols.hpp"
  29 #include "compiler/compileLog.hpp"
  30 #include "interpreter/linkResolver.hpp"
  31 #include "jvm_io.h"
  32 #include "memory/resourceArea.hpp"
  33 #include "memory/universe.hpp"
  34 #include "oops/oop.inline.hpp"
  35 #include "opto/addnode.hpp"
  36 #include "opto/castnode.hpp"
  37 #include "opto/convertnode.hpp"
  38 #include "opto/divnode.hpp"
  39 #include "opto/idealGraphPrinter.hpp"
  40 #include "opto/idealKit.hpp"
  41 #include "opto/inlinetypenode.hpp"
  42 #include "opto/matcher.hpp"
  43 #include "opto/memnode.hpp"
  44 #include "opto/mulnode.hpp"
  45 #include "opto/opaquenode.hpp"
  46 #include "opto/parse.hpp"
  47 #include "opto/runtime.hpp"
  48 #include "runtime/deoptimization.hpp"
  49 #include "runtime/sharedRuntime.hpp"
  50 
  51 #ifndef PRODUCT
  52 extern uint explicit_null_checks_inserted,
  53             explicit_null_checks_elided;
  54 #endif
  55 
  56 Node* Parse::record_profile_for_speculation_at_array_load(Node* ld) {
  57   // Feed unused profile data to type speculation
  58   if (UseTypeSpeculation && UseArrayLoadStoreProfile) {
  59     ciKlass* array_type = nullptr;
  60     ciKlass* element_type = nullptr;
  61     ProfilePtrKind element_ptr = ProfileMaybeNull;
  62     bool flat_array = true;
  63     bool null_free_array = true;
  64     method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
  65     if (element_type != nullptr || element_ptr != ProfileMaybeNull) {
  66       ld = record_profile_for_speculation(ld, element_type, element_ptr);
  67     }
  68   }
  69   return ld;
  70 }
  71 
  72 
  73 //---------------------------------array_load----------------------------------
  74 void Parse::array_load(BasicType bt) {
  75   const Type* elemtype = Type::TOP;
  76   Node* adr = array_addressing(bt, 0, elemtype);
  77   if (stopped())  return;     // guaranteed null or range check
  78 
  79   Node* array_index = pop();
  80   Node* array = pop();
  81 
  82   // Handle inline type arrays
  83   const TypeOopPtr* element_ptr = elemtype->make_oopptr();
  84   const TypeAryPtr* array_type = _gvn.type(array)->is_aryptr();
  85 
  86   if (!array_type->is_not_flat()) {
  87     // Cannot statically determine if array is a flat array, emit runtime check
  88     assert(UseArrayFlattening && is_reference_type(bt) && element_ptr->can_be_inline_type() &&
  89            (!element_ptr->is_inlinetypeptr() || element_ptr->inline_klass()->maybe_flat_in_array()), "array can't be flat");
  90     IdealKit ideal(this);
  91     IdealVariable res(ideal);
  92     ideal.declarations_done();
  93     ideal.if_then(flat_array_test(array, /* flat = */ false)); {
  94       // Non-flat array
  95       sync_kit(ideal);
  96       if (!array_type->is_flat()) {
  97         assert(array_type->is_flat() || control()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found");
  98         const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt);
  99         DecoratorSet decorator_set = IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD;
 100         if (needs_range_check(array_type->size(), array_index)) {
 101           // We've emitted a RangeCheck but now insert an additional check between the range check and the actual load.
 102           // We cannot pin the load to two separate nodes. Instead, we pin it conservatively here such that it cannot
 103           // possibly float above the range check at any point.
 104           decorator_set |= C2_UNKNOWN_CONTROL_LOAD;
 105         }
 106         Node* ld = access_load_at(array, adr, adr_type, element_ptr, bt, decorator_set);
 107         if (element_ptr->is_inlinetypeptr()) {
 108           ld = InlineTypeNode::make_from_oop(this, ld, element_ptr->inline_klass());
 109         }
 110         ideal.set(res, ld);
 111       }
 112       ideal.sync_kit(this);
 113     } ideal.else_(); {
 114       // Flat array
 115       sync_kit(ideal);
 116       if (!array_type->is_not_flat()) {
 117         if (element_ptr->is_inlinetypeptr()) {
 118           ciInlineKlass* vk = element_ptr->inline_klass();
 119           Node* flat_array = cast_to_flat_array(array, vk, false, false, false);
 120           Node* vt = InlineTypeNode::make_from_flat_array(this, vk, flat_array, array_index);
 121           ideal.set(res, vt);
 122         } else {
 123           // Element type is unknown, and thus we cannot statically determine the exact flat array layout. Emit a
 124           // runtime call to correctly load the inline type element from the flat array.
 125           Node* inline_type = load_from_unknown_flat_array(array, array_index, element_ptr);
 126           bool is_null_free = array_type->is_null_free() || !UseNullableValueFlattening;
 127           if (is_null_free) {
 128             inline_type = cast_not_null(inline_type);
 129           }
 130           ideal.set(res, inline_type);
 131         }
 132       }
 133       ideal.sync_kit(this);
 134     } ideal.end_if();
 135     sync_kit(ideal);
 136     Node* ld = _gvn.transform(ideal.value(res));
 137     ld = record_profile_for_speculation_at_array_load(ld);
 138     push_node(bt, ld);
 139     return;
 140   }
 141 
 142   if (elemtype == TypeInt::BOOL) {
 143     bt = T_BOOLEAN;
 144   }
 145   const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt);
 146   Node* ld = access_load_at(array, adr, adr_type, elemtype, bt,
 147                             IN_HEAP | IS_ARRAY | C2_CONTROL_DEPENDENT_LOAD);
 148   ld = record_profile_for_speculation_at_array_load(ld);
 149   // Loading an inline type from a non-flat array
 150   if (element_ptr != nullptr && element_ptr->is_inlinetypeptr()) {
 151     assert(!array_type->is_null_free() || !element_ptr->maybe_null(), "inline type array elements should never be null");
 152     ld = InlineTypeNode::make_from_oop(this, ld, element_ptr->inline_klass());
 153   }
 154   push_node(bt, ld);
 155 }
 156 
 157 Node* Parse::load_from_unknown_flat_array(Node* array, Node* array_index, const TypeOopPtr* element_ptr) {
 158   // Below membars keep this access to an unknown flat array correctly
 159   // ordered with other unknown and known flat array accesses.
 160   insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::INLINES));
 161 
 162   Node* call = nullptr;
 163   {
 164     // Re-execute flat array load if runtime call triggers deoptimization
 165     PreserveReexecuteState preexecs(this);
 166     jvms()->set_bci(_bci);
 167     jvms()->set_should_reexecute(true);
 168     inc_sp(2);
 169     kill_dead_locals();
 170     call = make_runtime_call(RC_NO_LEAF | RC_NO_IO,
 171                              OptoRuntime::load_unknown_inline_Type(),
 172                              OptoRuntime::load_unknown_inline_Java(),
 173                              nullptr, TypeRawPtr::BOTTOM,
 174                              array, array_index);
 175   }
 176   make_slow_call_ex(call, env()->Throwable_klass(), false);
 177   Node* buffer = _gvn.transform(new ProjNode(call, TypeFunc::Parms));
 178 
 179   insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::INLINES));
 180 
 181   // Keep track of the information that the inline type is in flat arrays
 182   const Type* unknown_value = element_ptr->is_instptr()->cast_to_flat_in_array();
 183   return _gvn.transform(new CheckCastPPNode(control(), buffer, unknown_value));
 184 }
 185 
 186 //--------------------------------array_store----------------------------------
 187 void Parse::array_store(BasicType bt) {
 188   const Type* elemtype = Type::TOP;
 189   Node* adr = array_addressing(bt, type2size[bt], elemtype);
 190   if (stopped())  return;     // guaranteed null or range check
 191   Node* stored_value_casted = nullptr;
 192   if (bt == T_OBJECT) {
 193     stored_value_casted = array_store_check(adr, elemtype);
 194     if (stopped()) {
 195       return;
 196     }
 197   }
 198   Node* const stored_value = pop_node(bt); // Value to store
 199   Node* const array_index = pop();         // Index in the array
 200   Node* array = pop();                     // The array itself
 201 
 202   const TypeAryPtr* array_type = _gvn.type(array)->is_aryptr();
 203   const TypeAryPtr* adr_type = TypeAryPtr::get_array_body_type(bt);
 204 
 205   if (elemtype == TypeInt::BOOL) {
 206     bt = T_BOOLEAN;
 207   } else if (bt == T_OBJECT) {
 208     elemtype = elemtype->make_oopptr();
 209     const Type* stored_value_casted_type = _gvn.type(stored_value_casted);
 210     // Based on the value to be stored, try to determine if the array is not null-free and/or not flat.
 211     // This is only legal for non-null stores because the array_store_check always passes for null, even
 212     // if the array is null-free. Null stores are handled in GraphKit::inline_array_null_guard().
 213     bool not_inline = !stored_value_casted_type->maybe_null() && !stored_value_casted_type->is_oopptr()->can_be_inline_type();
 214     bool not_null_free = not_inline;
 215     bool not_flat = not_inline || ( stored_value_casted_type->is_inlinetypeptr() &&
 216                                    !stored_value_casted_type->inline_klass()->maybe_flat_in_array());
 217     if (!array_type->is_not_null_free() && not_null_free) {
 218       // Storing a non-inline type, mark array as not null-free.
 219       array_type = array_type->cast_to_not_null_free();
 220       Node* cast = _gvn.transform(new CheckCastPPNode(control(), array, array_type));
 221       replace_in_map(array, cast);
 222       array = cast;
 223     }
 224     if (!array_type->is_not_flat() && not_flat) {
 225       // Storing to a non-flat array, mark array as not flat.
 226       array_type = array_type->cast_to_not_flat();
 227       Node* cast = _gvn.transform(new CheckCastPPNode(control(), array, array_type));
 228       replace_in_map(array, cast);
 229       array = cast;
 230     }
 231 
 232     if (!array_type->is_flat() && array_type->is_null_free()) {
 233       // Store to non-flat null-free inline type array (elements can never be null)
 234       assert(!stored_value_casted_type->maybe_null(), "should be guaranteed by array store check");
 235       if (elemtype->is_inlinetypeptr() && elemtype->inline_klass()->is_empty()) {
 236         // Ignore empty inline stores, array is already initialized.
 237         return;
 238       }
 239     } else if (!array_type->is_not_flat()) {
 240       // Array might be a flat array, emit runtime checks (for nullptr, a simple inline_array_null_guard is sufficient).
 241       assert(UseArrayFlattening && !not_flat && elemtype->is_oopptr()->can_be_inline_type() &&
 242              (!array_type->klass_is_exact() || array_type->is_flat()), "array can't be a flat array");
 243       // TODO 8350865 Depending on the available layouts, we can avoid this check in below flat/not-flat branches. Also the safe_for_replace arg is now always true.
 244       array = inline_array_null_guard(array, stored_value_casted, 3, true);
 245       IdealKit ideal(this);
 246       ideal.if_then(flat_array_test(array, /* flat = */ false)); {
 247         // Non-flat array
 248         if (!array_type->is_flat()) {
 249           sync_kit(ideal);
 250           assert(array_type->is_flat() || ideal.ctrl()->in(0)->as_If()->is_flat_array_check(&_gvn), "Should be found");
 251           inc_sp(3);
 252           access_store_at(array, adr, adr_type, stored_value_casted, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY, false);
 253           dec_sp(3);
 254           ideal.sync_kit(this);
 255         }
 256       } ideal.else_(); {
 257         // Flat array
 258         sync_kit(ideal);
 259         if (!array_type->is_not_flat()) {
 260           // Try to determine the inline klass type of the stored value
 261           ciInlineKlass* vk = nullptr;
 262           if (stored_value_casted_type->is_inlinetypeptr()) {
 263             vk = stored_value_casted_type->inline_klass();
 264           } else if (elemtype->is_inlinetypeptr()) {
 265             vk = elemtype->inline_klass();
 266           }
 267 
 268           if (vk != nullptr) {
 269             // Element type is known, cast and store to flat array layout.
 270             Node* flat_array = cast_to_flat_array(array, vk, false, false, false);
 271 
 272             // Re-execute flat array store if buffering triggers deoptimization
 273             PreserveReexecuteState preexecs(this);
 274             jvms()->set_should_reexecute(true);
 275             inc_sp(3);
 276 
 277             if (!stored_value_casted->is_InlineType()) {
 278               assert(_gvn.type(stored_value_casted) == TypePtr::NULL_PTR, "Unexpected value");
 279               stored_value_casted = InlineTypeNode::make_null(_gvn, vk);
 280             }
 281 
 282             stored_value_casted->as_InlineType()->store_flat_array(this, flat_array, array_index);
 283           } else {
 284             // Element type is unknown, emit a runtime call since the flat array layout is not statically known.
 285             store_to_unknown_flat_array(array, array_index, stored_value_casted);
 286           }
 287         }
 288         ideal.sync_kit(this);
 289       }
 290       ideal.end_if();
 291       sync_kit(ideal);
 292       return;
 293     } else if (!array_type->is_not_null_free()) {
 294       // Array is not flat but may be null free
 295       assert(elemtype->is_oopptr()->can_be_inline_type(), "array can't be null-free");
 296       array = inline_array_null_guard(array, stored_value_casted, 3, true);
 297     }
 298   }
 299   inc_sp(3);
 300   access_store_at(array, adr, adr_type, stored_value, elemtype, bt, MO_UNORDERED | IN_HEAP | IS_ARRAY);
 301   dec_sp(3);
 302 }
 303 
 304 // Emit a runtime call to store to a flat array whose element type is either unknown (i.e. we do not know the flat
 305 // array layout) or not exact (could have different flat array layouts at runtime).
 306 void Parse::store_to_unknown_flat_array(Node* array, Node* const idx, Node* non_null_stored_value) {
 307   // Below membars keep this access to an unknown flat array correctly
 308   // ordered with other unknown and known flat array accesses.
 309   insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::INLINES));
 310 
 311   Node* call = nullptr;
 312   {
 313     // Re-execute flat array store if runtime call triggers deoptimization
 314     PreserveReexecuteState preexecs(this);
 315     jvms()->set_bci(_bci);
 316     jvms()->set_should_reexecute(true);
 317     inc_sp(3);
 318     kill_dead_locals();
 319     call = make_runtime_call(RC_NO_LEAF | RC_NO_IO,
 320                       OptoRuntime::store_unknown_inline_Type(),
 321                       OptoRuntime::store_unknown_inline_Java(),
 322                       nullptr, TypeRawPtr::BOTTOM,
 323                       non_null_stored_value, array, idx);
 324   }
 325   make_slow_call_ex(call, env()->Throwable_klass(), false);
 326 
 327   insert_mem_bar_volatile(Op_MemBarCPUOrder, C->get_alias_index(TypeAryPtr::INLINES));
 328 }
 329 
 330 //------------------------------array_addressing-------------------------------
 331 // Pull array and index from the stack.  Compute pointer-to-element.
 332 Node* Parse::array_addressing(BasicType type, int vals, const Type*& elemtype) {
 333   Node *idx   = peek(0+vals);   // Get from stack without popping
 334   Node *ary   = peek(1+vals);   // in case of exception
 335 
 336   // Null check the array base, with correct stack contents
 337   ary = null_check(ary, T_ARRAY);
 338   // Compile-time detect of null-exception?
 339   if (stopped())  return top();
 340 
 341   const TypeAryPtr* arytype  = _gvn.type(ary)->is_aryptr();
 342   const TypeInt*    sizetype = arytype->size();
 343   elemtype = arytype->elem();
 344 
 345   if (UseUniqueSubclasses) {
 346     const Type* el = elemtype->make_ptr();
 347     if (el && el->isa_instptr()) {
 348       const TypeInstPtr* toop = el->is_instptr();
 349       if (toop->instance_klass()->unique_concrete_subklass()) {
 350         // If we load from "AbstractClass[]" we must see "ConcreteSubClass".
 351         const Type* subklass = Type::get_const_type(toop->instance_klass());
 352         elemtype = subklass->join_speculative(el);
 353       }
 354     }
 355   }
 356 
 357   if (!arytype->is_loaded()) {
 358     // Only fails for some -Xcomp runs
 359     // The class is unloaded.  We have to run this bytecode in the interpreter.
 360     ciKlass* klass = arytype->unloaded_klass();
 361 
 362     uncommon_trap(Deoptimization::Reason_unloaded,
 363                   Deoptimization::Action_reinterpret,
 364                   klass, "!loaded array");
 365     return top();
 366   }
 367 
 368   ary = create_speculative_inline_type_array_checks(ary, arytype, elemtype);
 369 
 370   if (needs_range_check(sizetype, idx)) {
 371     create_range_check(idx, ary, sizetype);
 372   } else if (C->log() != nullptr) {
 373     C->log()->elem("observe that='!need_range_check'");
 374   }
 375 
 376   // Check for always knowing you are throwing a range-check exception
 377   if (stopped())  return top();
 378 
 379   // Make array address computation control dependent to prevent it
 380   // from floating above the range check during loop optimizations.
 381   Node* ptr = array_element_address(ary, idx, type, sizetype, control());
 382   assert(ptr != top(), "top should go hand-in-hand with stopped");
 383 
 384   return ptr;
 385 }
 386 
 387 // Check if we need a range check for an array access. This is the case if the index is either negative or if it could
 388 // be greater or equal the smallest possible array size (i.e. out-of-bounds).
 389 bool Parse::needs_range_check(const TypeInt* size_type, const Node* index) const {
 390   const TypeInt* index_type = _gvn.type(index)->is_int();
 391   return index_type->_hi >= size_type->_lo || index_type->_lo < 0;
 392 }
 393 
 394 void Parse::create_range_check(Node* idx, Node* ary, const TypeInt* sizetype) {
 395   Node* tst;
 396   if (sizetype->_hi <= 0) {
 397     // The greatest array bound is negative, so we can conclude that we're
 398     // compiling unreachable code, but the unsigned compare trick used below
 399     // only works with non-negative lengths.  Instead, hack "tst" to be zero so
 400     // the uncommon_trap path will always be taken.
 401     tst = _gvn.intcon(0);
 402   } else {
 403     // Range is constant in array-oop, so we can use the original state of mem
 404     Node* len = load_array_length(ary);
 405 
 406     // Test length vs index (standard trick using unsigned compare)
 407     Node* chk = _gvn.transform(new CmpUNode(idx, len) );
 408     BoolTest::mask btest = BoolTest::lt;
 409     tst = _gvn.transform(new BoolNode(chk, btest) );
 410   }
 411   RangeCheckNode* rc = new RangeCheckNode(control(), tst, PROB_MAX, COUNT_UNKNOWN);
 412   _gvn.set_type(rc, rc->Value(&_gvn));
 413   if (!tst->is_Con()) {
 414     record_for_igvn(rc);
 415   }
 416   set_control(_gvn.transform(new IfTrueNode(rc)));
 417   // Branch to failure if out of bounds
 418   {
 419     PreserveJVMState pjvms(this);
 420     set_control(_gvn.transform(new IfFalseNode(rc)));
 421     if (C->allow_range_check_smearing()) {
 422       // Do not use builtin_throw, since range checks are sometimes
 423       // made more stringent by an optimistic transformation.
 424       // This creates "tentative" range checks at this point,
 425       // which are not guaranteed to throw exceptions.
 426       // See IfNode::Ideal, is_range_check, adjust_check.
 427       uncommon_trap(Deoptimization::Reason_range_check,
 428                     Deoptimization::Action_make_not_entrant,
 429                     nullptr, "range_check");
 430     } else {
 431       // If we have already recompiled with the range-check-widening
 432       // heroic optimization turned off, then we must really be throwing
 433       // range check exceptions.
 434       builtin_throw(Deoptimization::Reason_range_check);
 435     }
 436   }
 437 }
 438 
 439 // For inline type arrays, we can use the profiling information for array accesses to speculate on the type, flatness,
 440 // and null-freeness. We can either prepare the speculative type for later uses or emit explicit speculative checks with
 441 // traps now. In the latter case, the speculative type guarantees can avoid additional runtime checks later (e.g.
 442 // non-null-free implies non-flat which allows us to remove flatness checks). This makes the graph simpler.
 443 Node* Parse::create_speculative_inline_type_array_checks(Node* array, const TypeAryPtr* array_type,
 444                                                          const Type*& element_type) {
 445   if (!array_type->is_flat() && !array_type->is_not_flat()) {
 446     // For arrays that might be flat, speculate that the array has the exact type reported in the profile data such that
 447     // we can rely on a fixed memory layout (i.e. either a flat layout or not).
 448     array = cast_to_speculative_array_type(array, array_type, element_type);
 449   } else if (UseTypeSpeculation && UseArrayLoadStoreProfile) {
 450     // Array is known to be either flat or not flat. If possible, update the speculative type by using the profile data
 451     // at this bci.
 452     array = cast_to_profiled_array_type(array);
 453   }
 454 
 455   // Even though the type does not tell us whether we have an inline type array or not, we can still check the profile data
 456   // whether we have a non-null-free or non-flat array. Speculating on a non-null-free array doesn't help aaload but could
 457   // be profitable for a subsequent aastore.
 458   if (!array_type->is_null_free() && !array_type->is_not_null_free()) {
 459     array = speculate_non_null_free_array(array, array_type);
 460   }
 461   if (!array_type->is_flat() && !array_type->is_not_flat()) {
 462     array = speculate_non_flat_array(array, array_type);
 463   }
 464   return array;
 465 }
 466 
 467 // Speculate that the array has the exact type reported in the profile data. We emit a trap when this turns out to be
 468 // wrong. On the fast path, we add a CheckCastPP to use the exact type.
 469 Node* Parse::cast_to_speculative_array_type(Node* const array, const TypeAryPtr*& array_type, const Type*& element_type) {
 470   Deoptimization::DeoptReason reason = Deoptimization::Reason_speculate_class_check;
 471   ciKlass* speculative_array_type = array_type->speculative_type();
 472   if (too_many_traps_or_recompiles(reason) || speculative_array_type == nullptr) {
 473     // No speculative type, check profile data at this bci
 474     speculative_array_type = nullptr;
 475     reason = Deoptimization::Reason_class_check;
 476     if (UseArrayLoadStoreProfile && !too_many_traps_or_recompiles(reason)) {
 477       ciKlass* profiled_element_type = nullptr;
 478       ProfilePtrKind element_ptr = ProfileMaybeNull;
 479       bool flat_array = true;
 480       bool null_free_array = true;
 481       method()->array_access_profiled_type(bci(), speculative_array_type, profiled_element_type, element_ptr, flat_array,
 482                                            null_free_array);
 483     }
 484   }
 485   if (speculative_array_type != nullptr) {
 486     // Speculate that this array has the exact type reported by profile data
 487     Node* casted_array = nullptr;
 488     DEBUG_ONLY(Node* old_control = control();)
 489     Node* slow_ctl = type_check_receiver(array, speculative_array_type, 1.0, &casted_array);
 490     if (stopped()) {
 491       // The check always fails and therefore profile information is incorrect. Don't use it.
 492       assert(old_control == slow_ctl, "type check should have been removed");
 493       set_control(slow_ctl);
 494     } else if (!slow_ctl->is_top()) {
 495       { PreserveJVMState pjvms(this);
 496         set_control(slow_ctl);
 497         uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
 498       }
 499       replace_in_map(array, casted_array);
 500       array_type = _gvn.type(casted_array)->is_aryptr();
 501       element_type = array_type->elem();
 502       return casted_array;
 503     }
 504   }
 505   return array;
 506 }
 507 
 508 // Create a CheckCastPP when the speculative type can improve the current type.
 509 Node* Parse::cast_to_profiled_array_type(Node* const array) {
 510   ciKlass* array_type = nullptr;
 511   ciKlass* element_type = nullptr;
 512   ProfilePtrKind element_ptr = ProfileMaybeNull;
 513   bool flat_array = true;
 514   bool null_free_array = true;
 515   method()->array_access_profiled_type(bci(), array_type, element_type, element_ptr, flat_array, null_free_array);
 516   if (array_type != nullptr) {
 517     return record_profile_for_speculation(array, array_type, ProfileMaybeNull);
 518   }
 519   return array;
 520 }
 521 
 522 // Speculate that the array is non-null-free. We emit a trap when this turns out to be
 523 // wrong. On the fast path, we add a CheckCastPP to use the non-null-free type.
 524 Node* Parse::speculate_non_null_free_array(Node* const array, const TypeAryPtr*& array_type) {
 525   bool null_free_array = true;
 526   Deoptimization::DeoptReason reason = Deoptimization::Reason_none;
 527   if (array_type->speculative() != nullptr &&
 528       array_type->speculative()->is_aryptr()->is_not_null_free() &&
 529       !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
 530     null_free_array = false;
 531     reason = Deoptimization::Reason_speculate_class_check;
 532   } else if (UseArrayLoadStoreProfile && !too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) {
 533     ciKlass* profiled_array_type = nullptr;
 534     ciKlass* profiled_element_type = nullptr;
 535     ProfilePtrKind element_ptr = ProfileMaybeNull;
 536     bool flat_array = true;
 537     method()->array_access_profiled_type(bci(), profiled_array_type, profiled_element_type, element_ptr, flat_array,
 538                                          null_free_array);
 539     reason = Deoptimization::Reason_class_check;
 540   }
 541   if (!null_free_array) {
 542     { // Deoptimize if null-free array
 543       BuildCutout unless(this, null_free_array_test(array, /* null_free = */ false), PROB_MAX);
 544       uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
 545     }
 546     assert(!stopped(), "null-free array should have been caught earlier");
 547     Node* casted_array = _gvn.transform(new CheckCastPPNode(control(), array, array_type->cast_to_not_null_free()));
 548     replace_in_map(array, casted_array);
 549     array_type = _gvn.type(casted_array)->is_aryptr();
 550     return casted_array;
 551   }
 552   return array;
 553 }
 554 
 555 // Speculate that the array is non-flat. We emit a trap when this turns out to be wrong.
 556 // On the fast path, we add a CheckCastPP to use the non-flat type.
 557 Node* Parse::speculate_non_flat_array(Node* const array, const TypeAryPtr* const array_type) {
 558   bool flat_array = true;
 559   Deoptimization::DeoptReason reason = Deoptimization::Reason_none;
 560   if (array_type->speculative() != nullptr &&
 561       array_type->speculative()->is_aryptr()->is_not_flat() &&
 562       !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
 563     flat_array = false;
 564     reason = Deoptimization::Reason_speculate_class_check;
 565   } else if (UseArrayLoadStoreProfile && !too_many_traps_or_recompiles(reason)) {
 566     ciKlass* profiled_array_type = nullptr;
 567     ciKlass* profiled_element_type = nullptr;
 568     ProfilePtrKind element_ptr = ProfileMaybeNull;
 569     bool null_free_array = true;
 570     method()->array_access_profiled_type(bci(), profiled_array_type, profiled_element_type, element_ptr, flat_array,
 571                                          null_free_array);
 572     reason = Deoptimization::Reason_class_check;
 573   }
 574   if (!flat_array) {
 575     { // Deoptimize if flat array
 576       BuildCutout unless(this, flat_array_test(array, /* flat = */ false), PROB_MAX);
 577       uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
 578     }
 579     assert(!stopped(), "flat array should have been caught earlier");
 580     Node* casted_array = _gvn.transform(new CheckCastPPNode(control(), array, array_type->cast_to_not_flat()));
 581     replace_in_map(array, casted_array);
 582     return casted_array;
 583   }
 584   return array;
 585 }
 586 
 587 // returns IfNode
 588 IfNode* Parse::jump_if_fork_int(Node* a, Node* b, BoolTest::mask mask, float prob, float cnt) {
 589   Node   *cmp = _gvn.transform(new CmpINode(a, b)); // two cases: shiftcount > 32 and shiftcount <= 32
 590   Node   *tst = _gvn.transform(new BoolNode(cmp, mask));
 591   IfNode *iff = create_and_map_if(control(), tst, prob, cnt);
 592   return iff;
 593 }
 594 
 595 
 596 // sentinel value for the target bci to mark never taken branches
 597 // (according to profiling)
 598 static const int never_reached = INT_MAX;
 599 
 600 //------------------------------helper for tableswitch-------------------------
 601 void Parse::jump_if_true_fork(IfNode *iff, int dest_bci_if_true, bool unc) {
 602   // True branch, use existing map info
 603   { PreserveJVMState pjvms(this);
 604     Node *iftrue  = _gvn.transform( new IfTrueNode (iff) );
 605     set_control( iftrue );
 606     if (unc) {
 607       repush_if_args();
 608       uncommon_trap(Deoptimization::Reason_unstable_if,
 609                     Deoptimization::Action_reinterpret,
 610                     nullptr,
 611                     "taken always");
 612     } else {
 613       assert(dest_bci_if_true != never_reached, "inconsistent dest");
 614       merge_new_path(dest_bci_if_true);
 615     }
 616   }
 617 
 618   // False branch
 619   Node *iffalse = _gvn.transform( new IfFalseNode(iff) );
 620   set_control( iffalse );
 621 }
 622 
 623 void Parse::jump_if_false_fork(IfNode *iff, int dest_bci_if_true, bool unc) {
 624   // True branch, use existing map info
 625   { PreserveJVMState pjvms(this);
 626     Node *iffalse  = _gvn.transform( new IfFalseNode (iff) );
 627     set_control( iffalse );
 628     if (unc) {
 629       repush_if_args();
 630       uncommon_trap(Deoptimization::Reason_unstable_if,
 631                     Deoptimization::Action_reinterpret,
 632                     nullptr,
 633                     "taken never");
 634     } else {
 635       assert(dest_bci_if_true != never_reached, "inconsistent dest");
 636       merge_new_path(dest_bci_if_true);
 637     }
 638   }
 639 
 640   // False branch
 641   Node *iftrue = _gvn.transform( new IfTrueNode(iff) );
 642   set_control( iftrue );
 643 }
 644 
 645 void Parse::jump_if_always_fork(int dest_bci, bool unc) {
 646   // False branch, use existing map and control()
 647   if (unc) {
 648     repush_if_args();
 649     uncommon_trap(Deoptimization::Reason_unstable_if,
 650                   Deoptimization::Action_reinterpret,
 651                   nullptr,
 652                   "taken never");
 653   } else {
 654     assert(dest_bci != never_reached, "inconsistent dest");
 655     merge_new_path(dest_bci);
 656   }
 657 }
 658 
 659 
 660 extern "C" {
 661   static int jint_cmp(const void *i, const void *j) {
 662     int a = *(jint *)i;
 663     int b = *(jint *)j;
 664     return a > b ? 1 : a < b ? -1 : 0;
 665   }
 666 }
 667 
 668 
 669 class SwitchRange : public StackObj {
 670   // a range of integers coupled with a bci destination
 671   jint _lo;                     // inclusive lower limit
 672   jint _hi;                     // inclusive upper limit
 673   int _dest;
 674   float _cnt;                   // how many times this range was hit according to profiling
 675 
 676 public:
 677   jint lo() const              { return _lo;   }
 678   jint hi() const              { return _hi;   }
 679   int  dest() const            { return _dest; }
 680   bool is_singleton() const    { return _lo == _hi; }
 681   float cnt() const            { return _cnt; }
 682 
 683   void setRange(jint lo, jint hi, int dest, float cnt) {
 684     assert(lo <= hi, "must be a non-empty range");
 685     _lo = lo, _hi = hi; _dest = dest; _cnt = cnt;
 686     assert(_cnt >= 0, "");
 687   }
 688   bool adjoinRange(jint lo, jint hi, int dest, float cnt, bool trim_ranges) {
 689     assert(lo <= hi, "must be a non-empty range");
 690     if (lo == _hi+1) {
 691       // see merge_ranges() comment below
 692       if (trim_ranges) {
 693         if (cnt == 0) {
 694           if (_cnt != 0) {
 695             return false;
 696           }
 697           if (dest != _dest) {
 698             _dest = never_reached;
 699           }
 700         } else {
 701           if (_cnt == 0) {
 702             return false;
 703           }
 704           if (dest != _dest) {
 705             return false;
 706           }
 707         }
 708       } else {
 709         if (dest != _dest) {
 710           return false;
 711         }
 712       }
 713       _hi = hi;
 714       _cnt += cnt;
 715       return true;
 716     }
 717     return false;
 718   }
 719 
 720   void set (jint value, int dest, float cnt) {
 721     setRange(value, value, dest, cnt);
 722   }
 723   bool adjoin(jint value, int dest, float cnt, bool trim_ranges) {
 724     return adjoinRange(value, value, dest, cnt, trim_ranges);
 725   }
 726   bool adjoin(SwitchRange& other) {
 727     return adjoinRange(other._lo, other._hi, other._dest, other._cnt, false);
 728   }
 729 
 730   void print() {
 731     if (is_singleton())
 732       tty->print(" {%d}=>%d (cnt=%f)", lo(), dest(), cnt());
 733     else if (lo() == min_jint)
 734       tty->print(" {..%d}=>%d (cnt=%f)", hi(), dest(), cnt());
 735     else if (hi() == max_jint)
 736       tty->print(" {%d..}=>%d (cnt=%f)", lo(), dest(), cnt());
 737     else
 738       tty->print(" {%d..%d}=>%d (cnt=%f)", lo(), hi(), dest(), cnt());
 739   }
 740 };
 741 
 742 // We try to minimize the number of ranges and the size of the taken
 743 // ones using profiling data. When ranges are created,
 744 // SwitchRange::adjoinRange() only allows 2 adjoining ranges to merge
 745 // if both were never hit or both were hit to build longer unreached
 746 // ranges. Here, we now merge adjoining ranges with the same
 747 // destination and finally set destination of unreached ranges to the
 748 // special value never_reached because it can help minimize the number
 749 // of tests that are necessary.
 750 //
 751 // For instance:
 752 // [0, 1] to target1 sometimes taken
 753 // [1, 2] to target1 never taken
 754 // [2, 3] to target2 never taken
 755 // would lead to:
 756 // [0, 1] to target1 sometimes taken
 757 // [1, 3] never taken
 758 //
 759 // (first 2 ranges to target1 are not merged)
 760 static void merge_ranges(SwitchRange* ranges, int& rp) {
 761   if (rp == 0) {
 762     return;
 763   }
 764   int shift = 0;
 765   for (int j = 0; j < rp; j++) {
 766     SwitchRange& r1 = ranges[j-shift];
 767     SwitchRange& r2 = ranges[j+1];
 768     if (r1.adjoin(r2)) {
 769       shift++;
 770     } else if (shift > 0) {
 771       ranges[j+1-shift] = r2;
 772     }
 773   }
 774   rp -= shift;
 775   for (int j = 0; j <= rp; j++) {
 776     SwitchRange& r = ranges[j];
 777     if (r.cnt() == 0 && r.dest() != never_reached) {
 778       r.setRange(r.lo(), r.hi(), never_reached, r.cnt());
 779     }
 780   }
 781 }
 782 
 783 //-------------------------------do_tableswitch--------------------------------
 784 void Parse::do_tableswitch() {
 785   // Get information about tableswitch
 786   int default_dest = iter().get_dest_table(0);
 787   jint lo_index    = iter().get_int_table(1);
 788   jint hi_index    = iter().get_int_table(2);
 789   int len          = hi_index - lo_index + 1;
 790 
 791   if (len < 1) {
 792     // If this is a backward branch, add safepoint
 793     maybe_add_safepoint(default_dest);
 794     pop(); // the effect of the instruction execution on the operand stack
 795     merge(default_dest);
 796     return;
 797   }
 798 
 799   ciMethodData* methodData = method()->method_data();
 800   ciMultiBranchData* profile = nullptr;
 801   if (methodData->is_mature() && UseSwitchProfiling) {
 802     ciProfileData* data = methodData->bci_to_data(bci());
 803     if (data != nullptr && data->is_MultiBranchData()) {
 804       profile = (ciMultiBranchData*)data;
 805     }
 806   }
 807   bool trim_ranges = !C->too_many_traps(method(), bci(), Deoptimization::Reason_unstable_if);
 808 
 809   // generate decision tree, using trichotomy when possible
 810   int rnum = len+2;
 811   bool makes_backward_branch = (default_dest <= bci());
 812   SwitchRange* ranges = NEW_RESOURCE_ARRAY(SwitchRange, rnum);
 813   int rp = -1;
 814   if (lo_index != min_jint) {
 815     float cnt = 1.0F;
 816     if (profile != nullptr) {
 817       cnt = (float)profile->default_count() / (hi_index != max_jint ? 2.0F : 1.0F);
 818     }
 819     ranges[++rp].setRange(min_jint, lo_index-1, default_dest, cnt);
 820   }
 821   for (int j = 0; j < len; j++) {
 822     jint match_int = lo_index+j;
 823     int  dest      = iter().get_dest_table(j+3);
 824     makes_backward_branch |= (dest <= bci());
 825     float cnt = 1.0F;
 826     if (profile != nullptr) {
 827       cnt = (float)profile->count_at(j);
 828     }
 829     if (rp < 0 || !ranges[rp].adjoin(match_int, dest, cnt, trim_ranges)) {
 830       ranges[++rp].set(match_int, dest, cnt);
 831     }
 832   }
 833   jint highest = lo_index+(len-1);
 834   assert(ranges[rp].hi() == highest, "");
 835   if (highest != max_jint) {
 836     float cnt = 1.0F;
 837     if (profile != nullptr) {
 838       cnt = (float)profile->default_count() / (lo_index != min_jint ? 2.0F : 1.0F);
 839     }
 840     if (!ranges[rp].adjoinRange(highest+1, max_jint, default_dest, cnt, trim_ranges)) {
 841       ranges[++rp].setRange(highest+1, max_jint, default_dest, cnt);
 842     }
 843   }
 844   assert(rp < len+2, "not too many ranges");
 845 
 846   if (trim_ranges) {
 847     merge_ranges(ranges, rp);
 848   }
 849 
 850   // Safepoint in case if backward branch observed
 851   if (makes_backward_branch) {
 852     add_safepoint();
 853   }
 854 
 855   Node* lookup = pop(); // lookup value
 856   jump_switch_ranges(lookup, &ranges[0], &ranges[rp]);
 857 }
 858 
 859 
 860 //------------------------------do_lookupswitch--------------------------------
 861 void Parse::do_lookupswitch() {
 862   // Get information about lookupswitch
 863   int default_dest = iter().get_dest_table(0);
 864   jint len          = iter().get_int_table(1);
 865 
 866   if (len < 1) {    // If this is a backward branch, add safepoint
 867     maybe_add_safepoint(default_dest);
 868     pop(); // the effect of the instruction execution on the operand stack
 869     merge(default_dest);
 870     return;
 871   }
 872 
 873   ciMethodData* methodData = method()->method_data();
 874   ciMultiBranchData* profile = nullptr;
 875   if (methodData->is_mature() && UseSwitchProfiling) {
 876     ciProfileData* data = methodData->bci_to_data(bci());
 877     if (data != nullptr && data->is_MultiBranchData()) {
 878       profile = (ciMultiBranchData*)data;
 879     }
 880   }
 881   bool trim_ranges = !C->too_many_traps(method(), bci(), Deoptimization::Reason_unstable_if);
 882 
 883   // generate decision tree, using trichotomy when possible
 884   jint* table = NEW_RESOURCE_ARRAY(jint, len*3);
 885   {
 886     for (int j = 0; j < len; j++) {
 887       table[3*j+0] = iter().get_int_table(2+2*j);
 888       table[3*j+1] = iter().get_dest_table(2+2*j+1);
 889       // Handle overflow when converting from uint to jint
 890       table[3*j+2] = (profile == nullptr) ? 1 : (jint)MIN2<uint>((uint)max_jint, profile->count_at(j));
 891     }
 892     qsort(table, len, 3*sizeof(table[0]), jint_cmp);
 893   }
 894 
 895   float default_cnt = 1.0F;
 896   if (profile != nullptr) {
 897     juint defaults = max_juint - len;
 898     default_cnt = (float)profile->default_count()/(float)defaults;
 899   }
 900 
 901   int rnum = len*2+1;
 902   bool makes_backward_branch = (default_dest <= bci());
 903   SwitchRange* ranges = NEW_RESOURCE_ARRAY(SwitchRange, rnum);
 904   int rp = -1;
 905   for (int j = 0; j < len; j++) {
 906     jint match_int   = table[3*j+0];
 907     jint  dest        = table[3*j+1];
 908     jint  cnt         = table[3*j+2];
 909     jint  next_lo     = rp < 0 ? min_jint : ranges[rp].hi()+1;
 910     makes_backward_branch |= (dest <= bci());
 911     float c = default_cnt * ((float)match_int - (float)next_lo);
 912     if (match_int != next_lo && (rp < 0 || !ranges[rp].adjoinRange(next_lo, match_int-1, default_dest, c, trim_ranges))) {
 913       assert(default_dest != never_reached, "sentinel value for dead destinations");
 914       ranges[++rp].setRange(next_lo, match_int-1, default_dest, c);
 915     }
 916     if (rp < 0 || !ranges[rp].adjoin(match_int, dest, (float)cnt, trim_ranges)) {
 917       assert(dest != never_reached, "sentinel value for dead destinations");
 918       ranges[++rp].set(match_int, dest,  (float)cnt);
 919     }
 920   }
 921   jint highest = table[3*(len-1)];
 922   assert(ranges[rp].hi() == highest, "");
 923   if (highest != max_jint &&
 924       !ranges[rp].adjoinRange(highest+1, max_jint, default_dest, default_cnt * ((float)max_jint - (float)highest), trim_ranges)) {
 925     ranges[++rp].setRange(highest+1, max_jint, default_dest, default_cnt * ((float)max_jint - (float)highest));
 926   }
 927   assert(rp < rnum, "not too many ranges");
 928 
 929   if (trim_ranges) {
 930     merge_ranges(ranges, rp);
 931   }
 932 
 933   // Safepoint in case backward branch observed
 934   if (makes_backward_branch) {
 935     add_safepoint();
 936   }
 937 
 938   Node *lookup = pop(); // lookup value
 939   jump_switch_ranges(lookup, &ranges[0], &ranges[rp]);
 940 }
 941 
 942 static float if_prob(float taken_cnt, float total_cnt) {
 943   assert(taken_cnt <= total_cnt, "");
 944   if (total_cnt == 0) {
 945     return PROB_FAIR;
 946   }
 947   float p = taken_cnt / total_cnt;
 948   return clamp(p, PROB_MIN, PROB_MAX);
 949 }
 950 
 951 static float if_cnt(float cnt) {
 952   if (cnt == 0) {
 953     return COUNT_UNKNOWN;
 954   }
 955   return cnt;
 956 }
 957 
 958 static float sum_of_cnts(SwitchRange *lo, SwitchRange *hi) {
 959   float total_cnt = 0;
 960   for (SwitchRange* sr = lo; sr <= hi; sr++) {
 961     total_cnt += sr->cnt();
 962   }
 963   return total_cnt;
 964 }
 965 
 966 class SwitchRanges : public ResourceObj {
 967 public:
 968   SwitchRange* _lo;
 969   SwitchRange* _hi;
 970   SwitchRange* _mid;
 971   float _cost;
 972 
 973   enum {
 974     Start,
 975     LeftDone,
 976     RightDone,
 977     Done
 978   } _state;
 979 
 980   SwitchRanges(SwitchRange *lo, SwitchRange *hi)
 981     : _lo(lo), _hi(hi), _mid(nullptr),
 982       _cost(0), _state(Start) {
 983   }
 984 
 985   SwitchRanges()
 986     : _lo(nullptr), _hi(nullptr), _mid(nullptr),
 987       _cost(0), _state(Start) {}
 988 };
 989 
 990 // Estimate cost of performing a binary search on lo..hi
 991 static float compute_tree_cost(SwitchRange *lo, SwitchRange *hi, float total_cnt) {
 992   GrowableArray<SwitchRanges> tree;
 993   SwitchRanges root(lo, hi);
 994   tree.push(root);
 995 
 996   float cost = 0;
 997   do {
 998     SwitchRanges& r = *tree.adr_at(tree.length()-1);
 999     if (r._hi != r._lo) {
1000       if (r._mid == nullptr) {
1001         float r_cnt = sum_of_cnts(r._lo, r._hi);
1002 
1003         if (r_cnt == 0) {
1004           tree.pop();
1005           cost = 0;
1006           continue;
1007         }
1008 
1009         SwitchRange* mid = nullptr;
1010         mid = r._lo;
1011         for (float cnt = 0; ; ) {
1012           assert(mid <= r._hi, "out of bounds");
1013           cnt += mid->cnt();
1014           if (cnt > r_cnt / 2) {
1015             break;
1016           }
1017           mid++;
1018         }
1019         assert(mid <= r._hi, "out of bounds");
1020         r._mid = mid;
1021         r._cost = r_cnt / total_cnt;
1022       }
1023       r._cost += cost;
1024       if (r._state < SwitchRanges::LeftDone && r._mid > r._lo) {
1025         cost = 0;
1026         r._state = SwitchRanges::LeftDone;
1027         tree.push(SwitchRanges(r._lo, r._mid-1));
1028       } else if (r._state < SwitchRanges::RightDone) {
1029         cost = 0;
1030         r._state = SwitchRanges::RightDone;
1031         tree.push(SwitchRanges(r._mid == r._lo ? r._mid+1 : r._mid, r._hi));
1032       } else {
1033         tree.pop();
1034         cost = r._cost;
1035       }
1036     } else {
1037       tree.pop();
1038       cost = r._cost;
1039     }
1040   } while (tree.length() > 0);
1041 
1042 
1043   return cost;
1044 }
1045 
1046 // It sometimes pays off to test most common ranges before the binary search
1047 void Parse::linear_search_switch_ranges(Node* key_val, SwitchRange*& lo, SwitchRange*& hi) {
1048   uint nr = hi - lo + 1;
1049   float total_cnt = sum_of_cnts(lo, hi);
1050 
1051   float min = compute_tree_cost(lo, hi, total_cnt);
1052   float extra = 1;
1053   float sub = 0;
1054 
1055   SwitchRange* array1 = lo;
1056   SwitchRange* array2 = NEW_RESOURCE_ARRAY(SwitchRange, nr);
1057 
1058   SwitchRange* ranges = nullptr;
1059 
1060   while (nr >= 2) {
1061     assert(lo == array1 || lo == array2, "one the 2 already allocated arrays");
1062     ranges = (lo == array1) ? array2 : array1;
1063 
1064     // Find highest frequency range
1065     SwitchRange* candidate = lo;
1066     for (SwitchRange* sr = lo+1; sr <= hi; sr++) {
1067       if (sr->cnt() > candidate->cnt()) {
1068         candidate = sr;
1069       }
1070     }
1071     SwitchRange most_freq = *candidate;
1072     if (most_freq.cnt() == 0) {
1073       break;
1074     }
1075 
1076     // Copy remaining ranges into another array
1077     int shift = 0;
1078     for (uint i = 0; i < nr; i++) {
1079       SwitchRange* sr = &lo[i];
1080       if (sr != candidate) {
1081         ranges[i-shift] = *sr;
1082       } else {
1083         shift++;
1084         if (i > 0 && i < nr-1) {
1085           SwitchRange prev = lo[i-1];
1086           prev.setRange(prev.lo(), sr->hi(), prev.dest(), prev.cnt());
1087           if (prev.adjoin(lo[i+1])) {
1088             shift++;
1089             i++;
1090           }
1091           ranges[i-shift] = prev;
1092         }
1093       }
1094     }
1095     nr -= shift;
1096 
1097     // Evaluate cost of testing the most common range and performing a
1098     // binary search on the other ranges
1099     float cost = extra + compute_tree_cost(&ranges[0], &ranges[nr-1], total_cnt);
1100     if (cost >= min) {
1101       break;
1102     }
1103     // swap arrays
1104     lo = &ranges[0];
1105     hi = &ranges[nr-1];
1106 
1107     // It pays off: emit the test for the most common range
1108     assert(most_freq.cnt() > 0, "must be taken");
1109     Node* val = _gvn.transform(new SubINode(key_val, _gvn.intcon(most_freq.lo())));
1110     Node* cmp = _gvn.transform(new CmpUNode(val, _gvn.intcon(java_subtract(most_freq.hi(), most_freq.lo()))));
1111     Node* tst = _gvn.transform(new BoolNode(cmp, BoolTest::le));
1112     IfNode* iff = create_and_map_if(control(), tst, if_prob(most_freq.cnt(), total_cnt), if_cnt(most_freq.cnt()));
1113     jump_if_true_fork(iff, most_freq.dest(), false);
1114 
1115     sub += most_freq.cnt() / total_cnt;
1116     extra += 1 - sub;
1117     min = cost;
1118   }
1119 }
1120 
1121 //----------------------------create_jump_tables-------------------------------
1122 bool Parse::create_jump_tables(Node* key_val, SwitchRange* lo, SwitchRange* hi) {
1123   // Are jumptables enabled
1124   if (!UseJumpTables)  return false;
1125 
1126   // Are jumptables supported
1127   if (!Matcher::has_match_rule(Op_Jump))  return false;
1128 
1129   bool trim_ranges = !C->too_many_traps(method(), bci(), Deoptimization::Reason_unstable_if);
1130 
1131   // Decide if a guard is needed to lop off big ranges at either (or
1132   // both) end(s) of the input set. We'll call this the default target
1133   // even though we can't be sure that it is the true "default".
1134 
1135   bool needs_guard = false;
1136   int default_dest;
1137   int64_t total_outlier_size = 0;
1138   int64_t hi_size = ((int64_t)hi->hi()) - ((int64_t)hi->lo()) + 1;
1139   int64_t lo_size = ((int64_t)lo->hi()) - ((int64_t)lo->lo()) + 1;
1140 
1141   if (lo->dest() == hi->dest()) {
1142     total_outlier_size = hi_size + lo_size;
1143     default_dest = lo->dest();
1144   } else if (lo_size > hi_size) {
1145     total_outlier_size = lo_size;
1146     default_dest = lo->dest();
1147   } else {
1148     total_outlier_size = hi_size;
1149     default_dest = hi->dest();
1150   }
1151 
1152   float total = sum_of_cnts(lo, hi);
1153   float cost = compute_tree_cost(lo, hi, total);
1154 
1155   // If a guard test will eliminate very sparse end ranges, then
1156   // it is worth the cost of an extra jump.
1157   float trimmed_cnt = 0;
1158   if (total_outlier_size > (MaxJumpTableSparseness * 4)) {
1159     needs_guard = true;
1160     if (default_dest == lo->dest()) {
1161       trimmed_cnt += lo->cnt();
1162       lo++;
1163     }
1164     if (default_dest == hi->dest()) {
1165       trimmed_cnt += hi->cnt();
1166       hi--;
1167     }
1168   }
1169 
1170   // Find the total number of cases and ranges
1171   int64_t num_cases = ((int64_t)hi->hi()) - ((int64_t)lo->lo()) + 1;
1172   int num_range = hi - lo + 1;
1173 
1174   // Don't create table if: too large, too small, or too sparse.
1175   if (num_cases > MaxJumpTableSize)
1176     return false;
1177   if (UseSwitchProfiling) {
1178     // MinJumpTableSize is set so with a well balanced binary tree,
1179     // when the number of ranges is MinJumpTableSize, it's cheaper to
1180     // go through a JumpNode that a tree of IfNodes. Average cost of a
1181     // tree of IfNodes with MinJumpTableSize is
1182     // log2f(MinJumpTableSize) comparisons. So if the cost computed
1183     // from profile data is less than log2f(MinJumpTableSize) then
1184     // going with the binary search is cheaper.
1185     if (cost < log2f(MinJumpTableSize)) {
1186       return false;
1187     }
1188   } else {
1189     if (num_cases < MinJumpTableSize)
1190       return false;
1191   }
1192   if (num_cases > (MaxJumpTableSparseness * num_range))
1193     return false;
1194 
1195   // Normalize table lookups to zero
1196   int lowval = lo->lo();
1197   key_val = _gvn.transform( new SubINode(key_val, _gvn.intcon(lowval)) );
1198 
1199   // Generate a guard to protect against input keyvals that aren't
1200   // in the switch domain.
1201   if (needs_guard) {
1202     Node*   size = _gvn.intcon(num_cases);
1203     Node*   cmp = _gvn.transform(new CmpUNode(key_val, size));
1204     Node*   tst = _gvn.transform(new BoolNode(cmp, BoolTest::ge));
1205     IfNode* iff = create_and_map_if(control(), tst, if_prob(trimmed_cnt, total), if_cnt(trimmed_cnt));
1206     jump_if_true_fork(iff, default_dest, trim_ranges && trimmed_cnt == 0);
1207 
1208     total -= trimmed_cnt;
1209   }
1210 
1211   // Create an ideal node JumpTable that has projections
1212   // of all possible ranges for a switch statement
1213   // The key_val input must be converted to a pointer offset and scaled.
1214   // Compare Parse::array_addressing above.
1215 
1216   // Clean the 32-bit int into a real 64-bit offset.
1217   // Otherwise, the jint value 0 might turn into an offset of 0x0800000000.
1218   // Make I2L conversion control dependent to prevent it from
1219   // floating above the range check during loop optimizations.
1220   // Do not use a narrow int type here to prevent the data path from dying
1221   // while the control path is not removed. This can happen if the type of key_val
1222   // is later known to be out of bounds of [0, num_cases] and therefore a narrow cast
1223   // would be replaced by TOP while C2 is not able to fold the corresponding range checks.
1224   // Set _carry_dependency for the cast to avoid being removed by IGVN.
1225 #ifdef _LP64
1226   key_val = C->constrained_convI2L(&_gvn, key_val, TypeInt::INT, control(), true /* carry_dependency */);
1227 #endif
1228 
1229   // Shift the value by wordsize so we have an index into the table, rather
1230   // than a switch value
1231   Node *shiftWord = _gvn.MakeConX(wordSize);
1232   key_val = _gvn.transform( new MulXNode( key_val, shiftWord));
1233 
1234   // Create the JumpNode
1235   Arena* arena = C->comp_arena();
1236   float* probs = (float*)arena->Amalloc(sizeof(float)*num_cases);
1237   int i = 0;
1238   if (total == 0) {
1239     for (SwitchRange* r = lo; r <= hi; r++) {
1240       for (int64_t j = r->lo(); j <= r->hi(); j++, i++) {
1241         probs[i] = 1.0F / num_cases;
1242       }
1243     }
1244   } else {
1245     for (SwitchRange* r = lo; r <= hi; r++) {
1246       float prob = r->cnt()/total;
1247       for (int64_t j = r->lo(); j <= r->hi(); j++, i++) {
1248         probs[i] = prob / (r->hi() - r->lo() + 1);
1249       }
1250     }
1251   }
1252 
1253   ciMethodData* methodData = method()->method_data();
1254   ciMultiBranchData* profile = nullptr;
1255   if (methodData->is_mature()) {
1256     ciProfileData* data = methodData->bci_to_data(bci());
1257     if (data != nullptr && data->is_MultiBranchData()) {
1258       profile = (ciMultiBranchData*)data;
1259     }
1260   }
1261 
1262   Node* jtn = _gvn.transform(new JumpNode(control(), key_val, num_cases, probs, profile == nullptr ? COUNT_UNKNOWN : total));
1263 
1264   // These are the switch destinations hanging off the jumpnode
1265   i = 0;
1266   for (SwitchRange* r = lo; r <= hi; r++) {
1267     for (int64_t j = r->lo(); j <= r->hi(); j++, i++) {
1268       Node* input = _gvn.transform(new JumpProjNode(jtn, i, r->dest(), (int)(j - lowval)));
1269       {
1270         PreserveJVMState pjvms(this);
1271         set_control(input);
1272         jump_if_always_fork(r->dest(), trim_ranges && r->cnt() == 0);
1273       }
1274     }
1275   }
1276   assert(i == num_cases, "miscount of cases");
1277   stop_and_kill_map();  // no more uses for this JVMS
1278   return true;
1279 }
1280 
1281 //----------------------------jump_switch_ranges-------------------------------
1282 void Parse::jump_switch_ranges(Node* key_val, SwitchRange *lo, SwitchRange *hi, int switch_depth) {
1283   Block* switch_block = block();
1284   bool trim_ranges = !C->too_many_traps(method(), bci(), Deoptimization::Reason_unstable_if);
1285 
1286   if (switch_depth == 0) {
1287     // Do special processing for the top-level call.
1288     assert(lo->lo() == min_jint, "initial range must exhaust Type::INT");
1289     assert(hi->hi() == max_jint, "initial range must exhaust Type::INT");
1290 
1291     // Decrement pred-numbers for the unique set of nodes.
1292 #ifdef ASSERT
1293     if (!trim_ranges) {
1294       // Ensure that the block's successors are a (duplicate-free) set.
1295       int successors_counted = 0;  // block occurrences in [hi..lo]
1296       int unique_successors = switch_block->num_successors();
1297       for (int i = 0; i < unique_successors; i++) {
1298         Block* target = switch_block->successor_at(i);
1299 
1300         // Check that the set of successors is the same in both places.
1301         int successors_found = 0;
1302         for (SwitchRange* p = lo; p <= hi; p++) {
1303           if (p->dest() == target->start())  successors_found++;
1304         }
1305         assert(successors_found > 0, "successor must be known");
1306         successors_counted += successors_found;
1307       }
1308       assert(successors_counted == (hi-lo)+1, "no unexpected successors");
1309     }
1310 #endif
1311 
1312     // Maybe prune the inputs, based on the type of key_val.
1313     jint min_val = min_jint;
1314     jint max_val = max_jint;
1315     const TypeInt* ti = key_val->bottom_type()->isa_int();
1316     if (ti != nullptr) {
1317       min_val = ti->_lo;
1318       max_val = ti->_hi;
1319       assert(min_val <= max_val, "invalid int type");
1320     }
1321     while (lo->hi() < min_val) {
1322       lo++;
1323     }
1324     if (lo->lo() < min_val)  {
1325       lo->setRange(min_val, lo->hi(), lo->dest(), lo->cnt());
1326     }
1327     while (hi->lo() > max_val) {
1328       hi--;
1329     }
1330     if (hi->hi() > max_val) {
1331       hi->setRange(hi->lo(), max_val, hi->dest(), hi->cnt());
1332     }
1333 
1334     linear_search_switch_ranges(key_val, lo, hi);
1335   }
1336 
1337 #ifndef PRODUCT
1338   if (switch_depth == 0) {
1339     _max_switch_depth = 0;
1340     _est_switch_depth = log2i_graceful((hi - lo + 1) - 1) + 1;
1341   }
1342 #endif
1343 
1344   assert(lo <= hi, "must be a non-empty set of ranges");
1345   if (lo == hi) {
1346     jump_if_always_fork(lo->dest(), trim_ranges && lo->cnt() == 0);
1347   } else {
1348     assert(lo->hi() == (lo+1)->lo()-1, "contiguous ranges");
1349     assert(hi->lo() == (hi-1)->hi()+1, "contiguous ranges");
1350 
1351     if (create_jump_tables(key_val, lo, hi)) return;
1352 
1353     SwitchRange* mid = nullptr;
1354     float total_cnt = sum_of_cnts(lo, hi);
1355 
1356     int nr = hi - lo + 1;
1357     if (UseSwitchProfiling) {
1358       // Don't keep the binary search tree balanced: pick up mid point
1359       // that split frequencies in half.
1360       float cnt = 0;
1361       for (SwitchRange* sr = lo; sr <= hi; sr++) {
1362         cnt += sr->cnt();
1363         if (cnt >= total_cnt / 2) {
1364           mid = sr;
1365           break;
1366         }
1367       }
1368     } else {
1369       mid = lo + nr/2;
1370 
1371       // if there is an easy choice, pivot at a singleton:
1372       if (nr > 3 && !mid->is_singleton() && (mid-1)->is_singleton())  mid--;
1373 
1374       assert(lo < mid && mid <= hi, "good pivot choice");
1375       assert(nr != 2 || mid == hi,   "should pick higher of 2");
1376       assert(nr != 3 || mid == hi-1, "should pick middle of 3");
1377     }
1378 
1379 
1380     Node *test_val = _gvn.intcon(mid == lo ? mid->hi() : mid->lo());
1381 
1382     if (mid->is_singleton()) {
1383       IfNode *iff_ne = jump_if_fork_int(key_val, test_val, BoolTest::ne, 1-if_prob(mid->cnt(), total_cnt), if_cnt(mid->cnt()));
1384       jump_if_false_fork(iff_ne, mid->dest(), trim_ranges && mid->cnt() == 0);
1385 
1386       // Special Case:  If there are exactly three ranges, and the high
1387       // and low range each go to the same place, omit the "gt" test,
1388       // since it will not discriminate anything.
1389       bool eq_test_only = (hi == lo+2 && hi->dest() == lo->dest() && mid == hi-1) || mid == lo;
1390 
1391       // if there is a higher range, test for it and process it:
1392       if (mid < hi && !eq_test_only) {
1393         // two comparisons of same values--should enable 1 test for 2 branches
1394         // Use BoolTest::lt instead of BoolTest::gt
1395         float cnt = sum_of_cnts(lo, mid-1);
1396         IfNode *iff_lt  = jump_if_fork_int(key_val, test_val, BoolTest::lt, if_prob(cnt, total_cnt), if_cnt(cnt));
1397         Node   *iftrue  = _gvn.transform( new IfTrueNode(iff_lt) );
1398         Node   *iffalse = _gvn.transform( new IfFalseNode(iff_lt) );
1399         { PreserveJVMState pjvms(this);
1400           set_control(iffalse);
1401           jump_switch_ranges(key_val, mid+1, hi, switch_depth+1);
1402         }
1403         set_control(iftrue);
1404       }
1405 
1406     } else {
1407       // mid is a range, not a singleton, so treat mid..hi as a unit
1408       float cnt = sum_of_cnts(mid == lo ? mid+1 : mid, hi);
1409       IfNode *iff_ge = jump_if_fork_int(key_val, test_val, mid == lo ? BoolTest::gt : BoolTest::ge, if_prob(cnt, total_cnt), if_cnt(cnt));
1410 
1411       // if there is a higher range, test for it and process it:
1412       if (mid == hi) {
1413         jump_if_true_fork(iff_ge, mid->dest(), trim_ranges && cnt == 0);
1414       } else {
1415         Node *iftrue  = _gvn.transform( new IfTrueNode(iff_ge) );
1416         Node *iffalse = _gvn.transform( new IfFalseNode(iff_ge) );
1417         { PreserveJVMState pjvms(this);
1418           set_control(iftrue);
1419           jump_switch_ranges(key_val, mid == lo ? mid+1 : mid, hi, switch_depth+1);
1420         }
1421         set_control(iffalse);
1422       }
1423     }
1424 
1425     // in any case, process the lower range
1426     if (mid == lo) {
1427       if (mid->is_singleton()) {
1428         jump_switch_ranges(key_val, lo+1, hi, switch_depth+1);
1429       } else {
1430         jump_if_always_fork(lo->dest(), trim_ranges && lo->cnt() == 0);
1431       }
1432     } else {
1433       jump_switch_ranges(key_val, lo, mid-1, switch_depth+1);
1434     }
1435   }
1436 
1437   // Decrease pred_count for each successor after all is done.
1438   if (switch_depth == 0) {
1439     int unique_successors = switch_block->num_successors();
1440     for (int i = 0; i < unique_successors; i++) {
1441       Block* target = switch_block->successor_at(i);
1442       // Throw away the pre-allocated path for each unique successor.
1443       target->next_path_num();
1444     }
1445   }
1446 
1447 #ifndef PRODUCT
1448   _max_switch_depth = MAX2(switch_depth, _max_switch_depth);
1449   if (TraceOptoParse && Verbose && WizardMode && switch_depth == 0) {
1450     SwitchRange* r;
1451     int nsing = 0;
1452     for( r = lo; r <= hi; r++ ) {
1453       if( r->is_singleton() )  nsing++;
1454     }
1455     tty->print(">>> ");
1456     _method->print_short_name();
1457     tty->print_cr(" switch decision tree");
1458     tty->print_cr("    %d ranges (%d singletons), max_depth=%d, est_depth=%d",
1459                   (int) (hi-lo+1), nsing, _max_switch_depth, _est_switch_depth);
1460     if (_max_switch_depth > _est_switch_depth) {
1461       tty->print_cr("******** BAD SWITCH DEPTH ********");
1462     }
1463     tty->print("   ");
1464     for( r = lo; r <= hi; r++ ) {
1465       r->print();
1466     }
1467     tty->cr();
1468   }
1469 #endif
1470 }
1471 
1472 Node* Parse::floating_point_mod(Node* a, Node* b, BasicType type) {
1473   assert(type == BasicType::T_FLOAT || type == BasicType::T_DOUBLE, "only float and double are floating points");
1474   CallNode* mod = type == BasicType::T_DOUBLE ? static_cast<CallNode*>(new ModDNode(C, a, b)) : new ModFNode(C, a, b);
1475 
1476   Node* prev_mem = set_predefined_input_for_runtime_call(mod);
1477   mod = _gvn.transform(mod)->as_Call();
1478   set_predefined_output_for_runtime_call(mod, prev_mem, TypeRawPtr::BOTTOM);
1479   Node* result = _gvn.transform(new ProjNode(mod, TypeFunc::Parms + 0));
1480   record_for_igvn(mod);
1481   return result;
1482 }
1483 
1484 void Parse::l2f() {
1485   Node* f2 = pop();
1486   Node* f1 = pop();
1487   Node* c = make_runtime_call(RC_LEAF, OptoRuntime::l2f_Type(),
1488                               CAST_FROM_FN_PTR(address, SharedRuntime::l2f),
1489                               "l2f", nullptr, //no memory effects
1490                               f1, f2);
1491   Node* res = _gvn.transform(new ProjNode(c, TypeFunc::Parms + 0));
1492 
1493   push(res);
1494 }
1495 
1496 // Handle jsr and jsr_w bytecode
1497 void Parse::do_jsr() {
1498   assert(bc() == Bytecodes::_jsr || bc() == Bytecodes::_jsr_w, "wrong bytecode");
1499 
1500   // Store information about current state, tagged with new _jsr_bci
1501   int return_bci = iter().next_bci();
1502   int jsr_bci    = (bc() == Bytecodes::_jsr) ? iter().get_dest() : iter().get_far_dest();
1503 
1504   // The way we do things now, there is only one successor block
1505   // for the jsr, because the target code is cloned by ciTypeFlow.
1506   Block* target = successor_for_bci(jsr_bci);
1507 
1508   // What got pushed?
1509   const Type* ret_addr = target->peek();
1510   assert(ret_addr->singleton(), "must be a constant (cloned jsr body)");
1511 
1512   // Effect on jsr on stack
1513   push(_gvn.makecon(ret_addr));
1514 
1515   // Flow to the jsr.
1516   merge(jsr_bci);
1517 }
1518 
1519 // Handle ret bytecode
1520 void Parse::do_ret() {
1521   // Find to whom we return.
1522   assert(block()->num_successors() == 1, "a ret can only go one place now");
1523   Block* target = block()->successor_at(0);
1524   assert(!target->is_ready(), "our arrival must be expected");
1525   int pnum = target->next_path_num();
1526   merge_common(target, pnum);
1527 }
1528 
1529 static bool has_injected_profile(BoolTest::mask btest, Node* test, int& taken, int& not_taken) {
1530   if (btest != BoolTest::eq && btest != BoolTest::ne) {
1531     // Only ::eq and ::ne are supported for profile injection.
1532     return false;
1533   }
1534   if (test->is_Cmp() &&
1535       test->in(1)->Opcode() == Op_ProfileBoolean) {
1536     ProfileBooleanNode* profile = (ProfileBooleanNode*)test->in(1);
1537     int false_cnt = profile->false_count();
1538     int  true_cnt = profile->true_count();
1539 
1540     // Counts matching depends on the actual test operation (::eq or ::ne).
1541     // No need to scale the counts because profile injection was designed
1542     // to feed exact counts into VM.
1543     taken     = (btest == BoolTest::eq) ? false_cnt :  true_cnt;
1544     not_taken = (btest == BoolTest::eq) ?  true_cnt : false_cnt;
1545 
1546     profile->consume();
1547     return true;
1548   }
1549   return false;
1550 }
1551 
1552 // Give up if too few (or too many, in which case the sum will overflow) counts to be meaningful.
1553 // We also check that individual counters are positive first, otherwise the sum can become positive.
1554 // (check for saturation, integer overflow, and immature counts)
1555 static bool counters_are_meaningful(int counter1, int counter2, int min) {
1556   // check for saturation, including "uint" values too big to fit in "int"
1557   if (counter1 < 0 || counter2 < 0) {
1558     return false;
1559   }
1560   // check for integer overflow of the sum
1561   int64_t sum = (int64_t)counter1 + (int64_t)counter2;
1562   STATIC_ASSERT(sizeof(counter1) < sizeof(sum));
1563   if (sum > INT_MAX) {
1564     return false;
1565   }
1566   // check if mature
1567   return (counter1 + counter2) >= min;
1568 }
1569 
1570 //--------------------------dynamic_branch_prediction--------------------------
1571 // Try to gather dynamic branch prediction behavior.  Return a probability
1572 // of the branch being taken and set the "cnt" field.  Returns a -1.0
1573 // if we need to use static prediction for some reason.
1574 float Parse::dynamic_branch_prediction(float &cnt, BoolTest::mask btest, Node* test) {
1575   ResourceMark rm;
1576 
1577   cnt  = COUNT_UNKNOWN;
1578 
1579   int     taken = 0;
1580   int not_taken = 0;
1581 
1582   bool use_mdo = !has_injected_profile(btest, test, taken, not_taken);
1583 
1584   if (use_mdo) {
1585     // Use MethodData information if it is available
1586     // FIXME: free the ProfileData structure
1587     ciMethodData* methodData = method()->method_data();
1588     if (!methodData->is_mature())  return PROB_UNKNOWN;
1589     ciProfileData* data = methodData->bci_to_data(bci());
1590     if (data == nullptr) {
1591       return PROB_UNKNOWN;
1592     }
1593     if (!data->is_JumpData())  return PROB_UNKNOWN;
1594 
1595     // get taken and not taken values
1596     // NOTE: saturated UINT_MAX values become negative,
1597     // as do counts above INT_MAX.
1598     taken = data->as_JumpData()->taken();
1599     not_taken = 0;
1600     if (data->is_BranchData()) {
1601       not_taken = data->as_BranchData()->not_taken();
1602     }
1603 
1604     // scale the counts to be commensurate with invocation counts:
1605     // NOTE: overflow for positive values is clamped at INT_MAX
1606     taken = method()->scale_count(taken);
1607     not_taken = method()->scale_count(not_taken);
1608   }
1609   // At this point, saturation or overflow is indicated by INT_MAX
1610   // or a negative value.
1611 
1612   // Give up if too few (or too many, in which case the sum will overflow) counts to be meaningful.
1613   // We also check that individual counters are positive first, otherwise the sum can become positive.
1614   if (!counters_are_meaningful(taken, not_taken, 40)) {
1615     if (C->log() != nullptr) {
1616       C->log()->elem("branch target_bci='%d' taken='%d' not_taken='%d'", iter().get_dest(), taken, not_taken);
1617     }
1618     return PROB_UNKNOWN;
1619   }
1620 
1621   // Compute frequency that we arrive here
1622   float sum = taken + not_taken;
1623   // Adjust, if this block is a cloned private block but the
1624   // Jump counts are shared.  Taken the private counts for
1625   // just this path instead of the shared counts.
1626   if( block()->count() > 0 )
1627     sum = block()->count();
1628   cnt = sum / FreqCountInvocations;
1629 
1630   // Pin probability to sane limits
1631   float prob;
1632   if( !taken )
1633     prob = (0+PROB_MIN) / 2;
1634   else if( !not_taken )
1635     prob = (1+PROB_MAX) / 2;
1636   else {                         // Compute probability of true path
1637     prob = (float)taken / (float)(taken + not_taken);
1638     if (prob > PROB_MAX)  prob = PROB_MAX;
1639     if (prob < PROB_MIN)   prob = PROB_MIN;
1640   }
1641 
1642   assert((cnt > 0.0f) && (prob > 0.0f),
1643          "Bad frequency assignment in if cnt=%g prob=%g taken=%d not_taken=%d", cnt, prob, taken, not_taken);
1644 
1645   if (C->log() != nullptr) {
1646     const char* prob_str = nullptr;
1647     if (prob >= PROB_MAX)  prob_str = (prob == PROB_MAX) ? "max" : "always";
1648     if (prob <= PROB_MIN)  prob_str = (prob == PROB_MIN) ? "min" : "never";
1649     char prob_str_buf[30];
1650     if (prob_str == nullptr) {
1651       jio_snprintf(prob_str_buf, sizeof(prob_str_buf), "%20.2f", prob);
1652       prob_str = prob_str_buf;
1653     }
1654     C->log()->elem("branch target_bci='%d' taken='%d' not_taken='%d' cnt='%f' prob='%s'",
1655                    iter().get_dest(), taken, not_taken, cnt, prob_str);
1656   }
1657   return prob;
1658 }
1659 
1660 //-----------------------------branch_prediction-------------------------------
1661 float Parse::branch_prediction(float& cnt,
1662                                BoolTest::mask btest,
1663                                int target_bci,
1664                                Node* test) {
1665   float prob = dynamic_branch_prediction(cnt, btest, test);
1666   // If prob is unknown, switch to static prediction
1667   if (prob != PROB_UNKNOWN)  return prob;
1668 
1669   prob = PROB_FAIR;                   // Set default value
1670   if (btest == BoolTest::eq)          // Exactly equal test?
1671     prob = PROB_STATIC_INFREQUENT;    // Assume its relatively infrequent
1672   else if (btest == BoolTest::ne)
1673     prob = PROB_STATIC_FREQUENT;      // Assume its relatively frequent
1674 
1675   // If this is a conditional test guarding a backwards branch,
1676   // assume its a loop-back edge.  Make it a likely taken branch.
1677   if (target_bci < bci()) {
1678     if (is_osr_parse()) {    // Could be a hot OSR'd loop; force deopt
1679       // Since it's an OSR, we probably have profile data, but since
1680       // branch_prediction returned PROB_UNKNOWN, the counts are too small.
1681       // Let's make a special check here for completely zero counts.
1682       ciMethodData* methodData = method()->method_data();
1683       if (!methodData->is_empty()) {
1684         ciProfileData* data = methodData->bci_to_data(bci());
1685         // Only stop for truly zero counts, which mean an unknown part
1686         // of the OSR-ed method, and we want to deopt to gather more stats.
1687         // If you have ANY counts, then this loop is simply 'cold' relative
1688         // to the OSR loop.
1689         if (data == nullptr ||
1690             (data->as_BranchData()->taken() +  data->as_BranchData()->not_taken() == 0)) {
1691           // This is the only way to return PROB_UNKNOWN:
1692           return PROB_UNKNOWN;
1693         }
1694       }
1695     }
1696     prob = PROB_STATIC_FREQUENT;     // Likely to take backwards branch
1697   }
1698 
1699   assert(prob != PROB_UNKNOWN, "must have some guess at this point");
1700   return prob;
1701 }
1702 
1703 // The magic constants are chosen so as to match the output of
1704 // branch_prediction() when the profile reports a zero taken count.
1705 // It is important to distinguish zero counts unambiguously, because
1706 // some branches (e.g., _213_javac.Assembler.eliminate) validly produce
1707 // very small but nonzero probabilities, which if confused with zero
1708 // counts would keep the program recompiling indefinitely.
1709 bool Parse::seems_never_taken(float prob) const {
1710   return prob < PROB_MIN;
1711 }
1712 
1713 //-------------------------------repush_if_args--------------------------------
1714 // Push arguments of an "if" bytecode back onto the stack by adjusting _sp.
1715 inline int Parse::repush_if_args() {
1716   if (PrintOpto && WizardMode) {
1717     tty->print("defending against excessive implicit null exceptions on %s @%d in ",
1718                Bytecodes::name(iter().cur_bc()), iter().cur_bci());
1719     method()->print_name(); tty->cr();
1720   }
1721   int bc_depth = - Bytecodes::depth(iter().cur_bc());
1722   assert(bc_depth == 1 || bc_depth == 2, "only two kinds of branches");
1723   DEBUG_ONLY(sync_jvms());   // argument(n) requires a synced jvms
1724   assert(argument(0) != nullptr, "must exist");
1725   assert(bc_depth == 1 || argument(1) != nullptr, "two must exist");
1726   inc_sp(bc_depth);
1727   return bc_depth;
1728 }
1729 
1730 // Used by StressUnstableIfTraps
1731 static volatile int _trap_stress_counter = 0;
1732 
1733 void Parse::increment_trap_stress_counter(Node*& counter, Node*& incr_store) {
1734   Node* counter_addr = makecon(TypeRawPtr::make((address)&_trap_stress_counter));
1735   counter = make_load(control(), counter_addr, TypeInt::INT, T_INT, MemNode::unordered);
1736   counter = _gvn.transform(new AddINode(counter, intcon(1)));
1737   incr_store = store_to_memory(control(), counter_addr, counter, T_INT, MemNode::unordered);
1738 }
1739 
1740 //----------------------------------do_ifnull----------------------------------
1741 void Parse::do_ifnull(BoolTest::mask btest, Node *c) {
1742   int target_bci = iter().get_dest();
1743 
1744   Node* counter = nullptr;
1745   Node* incr_store = nullptr;
1746   bool do_stress_trap = StressUnstableIfTraps && ((C->random() % 2) == 0);
1747   if (do_stress_trap) {
1748     increment_trap_stress_counter(counter, incr_store);
1749   }
1750 
1751   Block* branch_block = successor_for_bci(target_bci);
1752   Block* next_block   = successor_for_bci(iter().next_bci());
1753 
1754   float cnt;
1755   float prob = branch_prediction(cnt, btest, target_bci, c);
1756   if (prob == PROB_UNKNOWN) {
1757     // (An earlier version of do_ifnull omitted this trap for OSR methods.)
1758     if (PrintOpto && Verbose) {
1759       tty->print_cr("Never-taken edge stops compilation at bci %d", bci());
1760     }
1761     repush_if_args(); // to gather stats on loop
1762     uncommon_trap(Deoptimization::Reason_unreached,
1763                   Deoptimization::Action_reinterpret,
1764                   nullptr, "cold");
1765     if (C->eliminate_boxing()) {
1766       // Mark the successor blocks as parsed
1767       branch_block->next_path_num();
1768       next_block->next_path_num();
1769     }
1770     return;
1771   }
1772 
1773   NOT_PRODUCT(explicit_null_checks_inserted++);
1774 
1775   // Generate real control flow
1776   Node   *tst = _gvn.transform( new BoolNode( c, btest ) );
1777 
1778   // Sanity check the probability value
1779   assert(prob > 0.0f,"Bad probability in Parser");
1780  // Need xform to put node in hash table
1781   IfNode *iff = create_and_xform_if( control(), tst, prob, cnt );
1782   assert(iff->_prob > 0.0f,"Optimizer made bad probability in parser");
1783   // True branch
1784   { PreserveJVMState pjvms(this);
1785     Node* iftrue  = _gvn.transform( new IfTrueNode (iff) );
1786     set_control(iftrue);
1787 
1788     if (stopped()) {            // Path is dead?
1789       NOT_PRODUCT(explicit_null_checks_elided++);
1790       if (C->eliminate_boxing()) {
1791         // Mark the successor block as parsed
1792         branch_block->next_path_num();
1793       }
1794     } else {                    // Path is live.
1795       adjust_map_after_if(btest, c, prob, branch_block);
1796       if (!stopped()) {
1797         merge(target_bci);
1798       }
1799     }
1800   }
1801 
1802   // False branch
1803   Node* iffalse = _gvn.transform( new IfFalseNode(iff) );
1804   set_control(iffalse);
1805 
1806   if (stopped()) {              // Path is dead?
1807     NOT_PRODUCT(explicit_null_checks_elided++);
1808     if (C->eliminate_boxing()) {
1809       // Mark the successor block as parsed
1810       next_block->next_path_num();
1811     }
1812   } else  {                     // Path is live.
1813     adjust_map_after_if(BoolTest(btest).negate(), c, 1.0-prob, next_block);
1814   }
1815 
1816   if (do_stress_trap) {
1817     stress_trap(iff, counter, incr_store);
1818   }
1819 }
1820 
1821 //------------------------------------do_if------------------------------------
1822 void Parse::do_if(BoolTest::mask btest, Node* c, bool can_trap, bool new_path, Node** ctrl_taken) {
1823   int target_bci = iter().get_dest();
1824 
1825   Block* branch_block = successor_for_bci(target_bci);
1826   Block* next_block   = successor_for_bci(iter().next_bci());
1827 
1828   float cnt;
1829   float prob = branch_prediction(cnt, btest, target_bci, c);
1830   float untaken_prob = 1.0 - prob;
1831 
1832   if (prob == PROB_UNKNOWN) {
1833     if (PrintOpto && Verbose) {
1834       tty->print_cr("Never-taken edge stops compilation at bci %d", bci());
1835     }
1836     repush_if_args(); // to gather stats on loop
1837     uncommon_trap(Deoptimization::Reason_unreached,
1838                   Deoptimization::Action_reinterpret,
1839                   nullptr, "cold");
1840     if (C->eliminate_boxing()) {
1841       // Mark the successor blocks as parsed
1842       branch_block->next_path_num();
1843       next_block->next_path_num();
1844     }
1845     return;
1846   }
1847 
1848   Node* counter = nullptr;
1849   Node* incr_store = nullptr;
1850   bool do_stress_trap = StressUnstableIfTraps && ((C->random() % 2) == 0);
1851   if (do_stress_trap) {
1852     increment_trap_stress_counter(counter, incr_store);
1853   }
1854 
1855   // Sanity check the probability value
1856   assert(0.0f < prob && prob < 1.0f,"Bad probability in Parser");
1857 
1858   bool taken_if_true = true;
1859   // Convert BoolTest to canonical form:
1860   if (!BoolTest(btest).is_canonical()) {
1861     btest         = BoolTest(btest).negate();
1862     taken_if_true = false;
1863     // prob is NOT updated here; it remains the probability of the taken
1864     // path (as opposed to the prob of the path guarded by an 'IfTrueNode').
1865   }
1866   assert(btest != BoolTest::eq, "!= is the only canonical exact test");
1867 
1868   Node* tst0 = new BoolNode(c, btest);
1869   Node* tst = _gvn.transform(tst0);
1870   BoolTest::mask taken_btest   = BoolTest::illegal;
1871   BoolTest::mask untaken_btest = BoolTest::illegal;
1872 
1873   if (tst->is_Bool()) {
1874     // Refresh c from the transformed bool node, since it may be
1875     // simpler than the original c.  Also re-canonicalize btest.
1876     // This wins when (Bool ne (Conv2B p) 0) => (Bool ne (CmpP p null)).
1877     // That can arise from statements like: if (x instanceof C) ...
1878     if (tst != tst0) {
1879       // Canonicalize one more time since transform can change it.
1880       btest = tst->as_Bool()->_test._test;
1881       if (!BoolTest(btest).is_canonical()) {
1882         // Reverse edges one more time...
1883         tst   = _gvn.transform( tst->as_Bool()->negate(&_gvn) );
1884         btest = tst->as_Bool()->_test._test;
1885         assert(BoolTest(btest).is_canonical(), "sanity");
1886         taken_if_true = !taken_if_true;
1887       }
1888       c = tst->in(1);
1889     }
1890     BoolTest::mask neg_btest = BoolTest(btest).negate();
1891     taken_btest   = taken_if_true ?     btest : neg_btest;
1892     untaken_btest = taken_if_true ? neg_btest :     btest;
1893   }
1894 
1895   // Generate real control flow
1896   float true_prob = (taken_if_true ? prob : untaken_prob);
1897   IfNode* iff = create_and_map_if(control(), tst, true_prob, cnt);
1898   assert(iff->_prob > 0.0f,"Optimizer made bad probability in parser");
1899   Node* taken_branch   = new IfTrueNode(iff);
1900   Node* untaken_branch = new IfFalseNode(iff);
1901   if (!taken_if_true) {  // Finish conversion to canonical form
1902     Node* tmp      = taken_branch;
1903     taken_branch   = untaken_branch;
1904     untaken_branch = tmp;
1905   }
1906 
1907   // Branch is taken:
1908   { PreserveJVMState pjvms(this);
1909     taken_branch = _gvn.transform(taken_branch);
1910     set_control(taken_branch);
1911 
1912     if (stopped()) {
1913       if (C->eliminate_boxing() && !new_path) {
1914         // Mark the successor block as parsed (if we haven't created a new path)
1915         branch_block->next_path_num();
1916       }
1917     } else {
1918       adjust_map_after_if(taken_btest, c, prob, branch_block, can_trap);
1919       if (!stopped()) {
1920         if (new_path) {
1921           // Merge by using a new path
1922           merge_new_path(target_bci);
1923         } else if (ctrl_taken != nullptr) {
1924           // Don't merge but save taken branch to be wired by caller
1925           *ctrl_taken = control();
1926         } else {
1927           merge(target_bci);
1928         }
1929       }
1930     }
1931   }
1932 
1933   untaken_branch = _gvn.transform(untaken_branch);
1934   set_control(untaken_branch);
1935 
1936   // Branch not taken.
1937   if (stopped() && ctrl_taken == nullptr) {
1938     if (C->eliminate_boxing()) {
1939       // Mark the successor block as parsed (if caller does not re-wire control flow)
1940       next_block->next_path_num();
1941     }
1942   } else {
1943     adjust_map_after_if(untaken_btest, c, untaken_prob, next_block, can_trap);
1944   }
1945 
1946   if (do_stress_trap) {
1947     stress_trap(iff, counter, incr_store);
1948   }
1949 }
1950 
1951 
1952 static ProfilePtrKind speculative_ptr_kind(const TypeOopPtr* t) {
1953   if (t->speculative() == nullptr) {
1954     return ProfileUnknownNull;
1955   }
1956   if (t->speculative_always_null()) {
1957     return ProfileAlwaysNull;
1958   }
1959   if (t->speculative_maybe_null()) {
1960     return ProfileMaybeNull;
1961   }
1962   return ProfileNeverNull;
1963 }
1964 
1965 void Parse::acmp_always_null_input(Node* input, const TypeOopPtr* tinput, BoolTest::mask btest, Node* eq_region) {
1966   inc_sp(2);
1967   Node* cast = null_check_common(input, T_OBJECT, true, nullptr,
1968                                  !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_check) &&
1969                                  speculative_ptr_kind(tinput) == ProfileAlwaysNull);
1970   dec_sp(2);
1971   if (btest == BoolTest::ne) {
1972     {
1973       PreserveJVMState pjvms(this);
1974       replace_in_map(input, cast);
1975       int target_bci = iter().get_dest();
1976       merge(target_bci);
1977     }
1978     record_for_igvn(eq_region);
1979     set_control(_gvn.transform(eq_region));
1980   } else {
1981     replace_in_map(input, cast);
1982   }
1983 }
1984 
1985 Node* Parse::acmp_null_check(Node* input, const TypeOopPtr* tinput, ProfilePtrKind input_ptr, Node*& null_ctl) {
1986   inc_sp(2);
1987   null_ctl = top();
1988   Node* cast = null_check_oop(input, &null_ctl,
1989                               input_ptr == ProfileNeverNull || (input_ptr == ProfileUnknownNull && !too_many_traps_or_recompiles(Deoptimization::Reason_null_check)),
1990                               false,
1991                               speculative_ptr_kind(tinput) == ProfileNeverNull &&
1992                               !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_check));
1993   dec_sp(2);
1994   assert(!stopped(), "null input should have been caught earlier");
1995   return cast;
1996 }
1997 
1998 void Parse::acmp_known_non_inline_type_input(Node* input, const TypeOopPtr* tinput, ProfilePtrKind input_ptr, ciKlass* input_type, BoolTest::mask btest, Node* eq_region) {
1999   Node* ne_region = new RegionNode(1);
2000   Node* null_ctl;
2001   Node* cast = acmp_null_check(input, tinput, input_ptr, null_ctl);
2002   ne_region->add_req(null_ctl);
2003 
2004   Node* slow_ctl = type_check_receiver(cast, input_type, 1.0, &cast);
2005   {
2006     PreserveJVMState pjvms(this);
2007     inc_sp(2);
2008     set_control(slow_ctl);
2009     Deoptimization::DeoptReason reason;
2010     if (tinput->speculative_type() != nullptr && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
2011       reason = Deoptimization::Reason_speculate_class_check;
2012     } else {
2013       reason = Deoptimization::Reason_class_check;
2014     }
2015     uncommon_trap_exact(reason, Deoptimization::Action_maybe_recompile);
2016   }
2017   ne_region->add_req(control());
2018 
2019   record_for_igvn(ne_region);
2020   set_control(_gvn.transform(ne_region));
2021   if (btest == BoolTest::ne) {
2022     {
2023       PreserveJVMState pjvms(this);
2024       if (null_ctl == top()) {
2025         replace_in_map(input, cast);
2026       }
2027       int target_bci = iter().get_dest();
2028       merge(target_bci);
2029     }
2030     record_for_igvn(eq_region);
2031     set_control(_gvn.transform(eq_region));
2032   } else {
2033     if (null_ctl == top()) {
2034       replace_in_map(input, cast);
2035     }
2036     set_control(_gvn.transform(ne_region));
2037   }
2038 }
2039 
2040 void Parse::acmp_unknown_non_inline_type_input(Node* input, const TypeOopPtr* tinput, ProfilePtrKind input_ptr, BoolTest::mask btest, Node* eq_region) {
2041   Node* ne_region = new RegionNode(1);
2042   Node* null_ctl;
2043   Node* cast = acmp_null_check(input, tinput, input_ptr, null_ctl);
2044   ne_region->add_req(null_ctl);
2045 
2046   {
2047     BuildCutout unless(this, inline_type_test(cast, /* is_inline = */ false), PROB_MAX);
2048     inc_sp(2);
2049     uncommon_trap_exact(Deoptimization::Reason_class_check, Deoptimization::Action_maybe_recompile);
2050   }
2051 
2052   ne_region->add_req(control());
2053 
2054   record_for_igvn(ne_region);
2055   set_control(_gvn.transform(ne_region));
2056   if (btest == BoolTest::ne) {
2057     {
2058       PreserveJVMState pjvms(this);
2059       if (null_ctl == top()) {
2060         replace_in_map(input, cast);
2061       }
2062       int target_bci = iter().get_dest();
2063       merge(target_bci);
2064     }
2065     record_for_igvn(eq_region);
2066     set_control(_gvn.transform(eq_region));
2067   } else {
2068     if (null_ctl == top()) {
2069       replace_in_map(input, cast);
2070     }
2071     set_control(_gvn.transform(ne_region));
2072   }
2073 }
2074 
2075 void Parse::do_acmp(BoolTest::mask btest, Node* left, Node* right) {
2076   ciKlass* left_type = nullptr;
2077   ciKlass* right_type = nullptr;
2078   ProfilePtrKind left_ptr = ProfileUnknownNull;
2079   ProfilePtrKind right_ptr = ProfileUnknownNull;
2080   bool left_inline_type = true;
2081   bool right_inline_type = true;
2082 
2083   // Leverage profiling at acmp
2084   if (UseACmpProfile) {
2085     method()->acmp_profiled_type(bci(), left_type, right_type, left_ptr, right_ptr, left_inline_type, right_inline_type);
2086     if (too_many_traps_or_recompiles(Deoptimization::Reason_class_check)) {
2087       left_type = nullptr;
2088       right_type = nullptr;
2089       left_inline_type = true;
2090       right_inline_type = true;
2091     }
2092     if (too_many_traps_or_recompiles(Deoptimization::Reason_null_check)) {
2093       left_ptr = ProfileUnknownNull;
2094       right_ptr = ProfileUnknownNull;
2095     }
2096   }
2097 
2098   if (UseTypeSpeculation) {
2099     record_profile_for_speculation(left, left_type, left_ptr);
2100     record_profile_for_speculation(right, right_type, right_ptr);
2101   }
2102 
2103   if (!EnableValhalla) {
2104     Node* cmp = CmpP(left, right);
2105     cmp = optimize_cmp_with_klass(cmp);
2106     do_if(btest, cmp);
2107     return;
2108   }
2109 
2110   // Check for equality before potentially allocating
2111   if (left == right) {
2112     do_if(btest, makecon(TypeInt::CC_EQ));
2113     return;
2114   }
2115 
2116   // Allocate inline type operands and re-execute on deoptimization
2117   if (left->is_InlineType()) {
2118     if (_gvn.type(right)->is_zero_type() ||
2119         (right->is_InlineType() && _gvn.type(right->as_InlineType()->get_is_init())->is_zero_type())) {
2120       // Null checking a scalarized but nullable inline type. Check the IsInit
2121       // input instead of the oop input to avoid keeping buffer allocations alive.
2122       Node* cmp = CmpI(left->as_InlineType()->get_is_init(), intcon(0));
2123       do_if(btest, cmp);
2124       return;
2125     } else {
2126       PreserveReexecuteState preexecs(this);
2127       inc_sp(2);
2128       jvms()->set_should_reexecute(true);
2129       left = left->as_InlineType()->buffer(this)->get_oop();
2130     }
2131   }
2132   if (right->is_InlineType()) {
2133     PreserveReexecuteState preexecs(this);
2134     inc_sp(2);
2135     jvms()->set_should_reexecute(true);
2136     right = right->as_InlineType()->buffer(this)->get_oop();
2137   }
2138 
2139   // First, do a normal pointer comparison
2140   const TypeOopPtr* tleft = _gvn.type(left)->isa_oopptr();
2141   const TypeOopPtr* tright = _gvn.type(right)->isa_oopptr();
2142   Node* cmp = CmpP(left, right);
2143   cmp = optimize_cmp_with_klass(cmp);
2144   if (tleft == nullptr || !tleft->can_be_inline_type() ||
2145       tright == nullptr || !tright->can_be_inline_type()) {
2146     // This is sufficient, if one of the operands can't be an inline type
2147     do_if(btest, cmp);
2148     return;
2149   }
2150 
2151   // Don't add traps to unstable if branches because additional checks are required to
2152   // decide if the operands are equal/substitutable and we therefore shouldn't prune
2153   // branches for one if based on the profiling of the acmp branches.
2154   // Also, OptimizeUnstableIf would set an incorrect re-rexecution state because it
2155   // assumes that there is a 1-1 mapping between the if and the acmp branches and that
2156   // hitting a trap means that we will take the corresponding acmp branch on re-execution.
2157   const bool can_trap = true;
2158 
2159   Node* eq_region = nullptr;
2160   if (btest == BoolTest::eq) {
2161     do_if(btest, cmp, !can_trap, true);
2162     if (stopped()) {
2163       // Pointers are equal, operands must be equal
2164       return;
2165     }
2166   } else {
2167     assert(btest == BoolTest::ne, "only eq or ne");
2168     Node* is_not_equal = nullptr;
2169     eq_region = new RegionNode(3);
2170     {
2171       PreserveJVMState pjvms(this);
2172       // Pointers are not equal, but more checks are needed to determine if the operands are (not) substitutable
2173       do_if(btest, cmp, !can_trap, false, &is_not_equal);
2174       if (!stopped()) {
2175         eq_region->init_req(1, control());
2176       }
2177     }
2178     if (is_not_equal == nullptr || is_not_equal->is_top()) {
2179       record_for_igvn(eq_region);
2180       set_control(_gvn.transform(eq_region));
2181       return;
2182     }
2183     set_control(is_not_equal);
2184   }
2185 
2186   // Prefer speculative types if available
2187   if (!too_many_traps_or_recompiles(Deoptimization::Reason_speculate_class_check)) {
2188     if (tleft->speculative_type() != nullptr) {
2189       left_type = tleft->speculative_type();
2190     }
2191     if (tright->speculative_type() != nullptr) {
2192       right_type = tright->speculative_type();
2193     }
2194   }
2195 
2196   if (speculative_ptr_kind(tleft) != ProfileMaybeNull && speculative_ptr_kind(tleft) != ProfileUnknownNull) {
2197     ProfilePtrKind speculative_left_ptr = speculative_ptr_kind(tleft);
2198     if (speculative_left_ptr == ProfileAlwaysNull && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_assert)) {
2199       left_ptr = speculative_left_ptr;
2200     } else if (speculative_left_ptr == ProfileNeverNull && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_check)) {
2201       left_ptr = speculative_left_ptr;
2202     }
2203   }
2204   if (speculative_ptr_kind(tright) != ProfileMaybeNull && speculative_ptr_kind(tright) != ProfileUnknownNull) {
2205     ProfilePtrKind speculative_right_ptr = speculative_ptr_kind(tright);
2206     if (speculative_right_ptr == ProfileAlwaysNull && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_assert)) {
2207       right_ptr = speculative_right_ptr;
2208     } else if (speculative_right_ptr == ProfileNeverNull && !too_many_traps_or_recompiles(Deoptimization::Reason_speculate_null_check)) {
2209       right_ptr = speculative_right_ptr;
2210     }
2211   }
2212 
2213   if (left_ptr == ProfileAlwaysNull) {
2214     // Comparison with null. Assert the input is indeed null and we're done.
2215     acmp_always_null_input(left, tleft, btest, eq_region);
2216     return;
2217   }
2218   if (right_ptr == ProfileAlwaysNull) {
2219     // Comparison with null. Assert the input is indeed null and we're done.
2220     acmp_always_null_input(right, tright, btest, eq_region);
2221     return;
2222   }
2223   if (left_type != nullptr && !left_type->is_inlinetype()) {
2224     // Comparison with an object of known type
2225     acmp_known_non_inline_type_input(left, tleft, left_ptr, left_type, btest, eq_region);
2226     return;
2227   }
2228   if (right_type != nullptr && !right_type->is_inlinetype()) {
2229     // Comparison with an object of known type
2230     acmp_known_non_inline_type_input(right, tright, right_ptr, right_type, btest, eq_region);
2231     return;
2232   }
2233   if (!left_inline_type) {
2234     // Comparison with an object known not to be an inline type
2235     acmp_unknown_non_inline_type_input(left, tleft, left_ptr, btest, eq_region);
2236     return;
2237   }
2238   if (!right_inline_type) {
2239     // Comparison with an object known not to be an inline type
2240     acmp_unknown_non_inline_type_input(right, tright, right_ptr, btest, eq_region);
2241     return;
2242   }
2243 
2244   // Pointers are not equal, check if first operand is non-null
2245   Node* ne_region = new RegionNode(6);
2246   Node* null_ctl;
2247   Node* not_null_right = acmp_null_check(right, tright, right_ptr, null_ctl);
2248   ne_region->init_req(1, null_ctl);
2249 
2250   // First operand is non-null, check if it is an inline type
2251   Node* is_value = inline_type_test(not_null_right);
2252   IfNode* is_value_iff = create_and_map_if(control(), is_value, PROB_FAIR, COUNT_UNKNOWN);
2253   Node* not_value = _gvn.transform(new IfFalseNode(is_value_iff));
2254   ne_region->init_req(2, not_value);
2255   set_control(_gvn.transform(new IfTrueNode(is_value_iff)));
2256 
2257   // The first operand is an inline type, check if the second operand is non-null
2258   Node* not_null_left = acmp_null_check(left, tleft, left_ptr, null_ctl);
2259   ne_region->init_req(3, null_ctl);
2260 
2261   // Check if both operands are of the same class.
2262   Node* kls_left = load_object_klass(not_null_left);
2263   Node* kls_right = load_object_klass(not_null_right);
2264   Node* kls_cmp = CmpP(kls_left, kls_right);
2265   Node* kls_bol = _gvn.transform(new BoolNode(kls_cmp, BoolTest::ne));
2266   IfNode* kls_iff = create_and_map_if(control(), kls_bol, PROB_FAIR, COUNT_UNKNOWN);
2267   Node* kls_ne = _gvn.transform(new IfTrueNode(kls_iff));
2268   set_control(_gvn.transform(new IfFalseNode(kls_iff)));
2269   ne_region->init_req(4, kls_ne);
2270 
2271   if (stopped()) {
2272     record_for_igvn(ne_region);
2273     set_control(_gvn.transform(ne_region));
2274     if (btest == BoolTest::ne) {
2275       {
2276         PreserveJVMState pjvms(this);
2277         int target_bci = iter().get_dest();
2278         merge(target_bci);
2279       }
2280       record_for_igvn(eq_region);
2281       set_control(_gvn.transform(eq_region));
2282     }
2283     return;
2284   }
2285 
2286   // Both operands are values types of the same class, we need to perform a
2287   // substitutability test. Delegate to ValueObjectMethods::isSubstitutable().
2288   Node* ne_io_phi = PhiNode::make(ne_region, i_o());
2289   Node* mem = reset_memory();
2290   Node* ne_mem_phi = PhiNode::make(ne_region, mem);
2291 
2292   Node* eq_io_phi = nullptr;
2293   Node* eq_mem_phi = nullptr;
2294   if (eq_region != nullptr) {
2295     eq_io_phi = PhiNode::make(eq_region, i_o());
2296     eq_mem_phi = PhiNode::make(eq_region, mem);
2297   }
2298 
2299   set_all_memory(mem);
2300 
2301   kill_dead_locals();
2302   ciMethod* subst_method = ciEnv::current()->ValueObjectMethods_klass()->find_method(ciSymbols::isSubstitutable_name(), ciSymbols::object_object_boolean_signature());
2303   CallStaticJavaNode *call = new CallStaticJavaNode(C, TypeFunc::make(subst_method), SharedRuntime::get_resolve_static_call_stub(), subst_method);
2304   call->set_override_symbolic_info(true);
2305   call->init_req(TypeFunc::Parms, not_null_left);
2306   call->init_req(TypeFunc::Parms+1, not_null_right);
2307   inc_sp(2);
2308   set_edges_for_java_call(call, false, false);
2309   Node* ret = set_results_for_java_call(call, false, true);
2310   dec_sp(2);
2311 
2312   // Test the return value of ValueObjectMethods::isSubstitutable()
2313   // This is the last check, do_if can emit traps now.
2314   Node* subst_cmp = _gvn.transform(new CmpINode(ret, intcon(1)));
2315   Node* ctl = C->top();
2316   if (btest == BoolTest::eq) {
2317     PreserveJVMState pjvms(this);
2318     do_if(btest, subst_cmp, can_trap);
2319     if (!stopped()) {
2320       ctl = control();
2321     }
2322   } else {
2323     assert(btest == BoolTest::ne, "only eq or ne");
2324     PreserveJVMState pjvms(this);
2325     do_if(btest, subst_cmp, can_trap, false, &ctl);
2326     if (!stopped()) {
2327       eq_region->init_req(2, control());
2328       eq_io_phi->init_req(2, i_o());
2329       eq_mem_phi->init_req(2, reset_memory());
2330     }
2331   }
2332   ne_region->init_req(5, ctl);
2333   ne_io_phi->init_req(5, i_o());
2334   ne_mem_phi->init_req(5, reset_memory());
2335 
2336   record_for_igvn(ne_region);
2337   set_control(_gvn.transform(ne_region));
2338   set_i_o(_gvn.transform(ne_io_phi));
2339   set_all_memory(_gvn.transform(ne_mem_phi));
2340 
2341   if (btest == BoolTest::ne) {
2342     {
2343       PreserveJVMState pjvms(this);
2344       int target_bci = iter().get_dest();
2345       merge(target_bci);
2346     }
2347 
2348     record_for_igvn(eq_region);
2349     set_control(_gvn.transform(eq_region));
2350     set_i_o(_gvn.transform(eq_io_phi));
2351     set_all_memory(_gvn.transform(eq_mem_phi));
2352   }
2353 }
2354 
2355 // Force unstable if traps to be taken randomly to trigger intermittent bugs such as incorrect debug information.
2356 // Add another if before the unstable if that checks a "random" condition at runtime (a simple shared counter) and
2357 // then either takes the trap or executes the original, unstable if.
2358 void Parse::stress_trap(IfNode* orig_iff, Node* counter, Node* incr_store) {
2359   // Search for an unstable if trap
2360   CallStaticJavaNode* trap = nullptr;
2361   assert(orig_iff->Opcode() == Op_If && orig_iff->outcnt() == 2, "malformed if");
2362   ProjNode* trap_proj = orig_iff->uncommon_trap_proj(trap, Deoptimization::Reason_unstable_if);
2363   if (trap == nullptr || !trap->jvms()->should_reexecute()) {
2364     // No suitable trap found. Remove unused counter load and increment.
2365     C->gvn_replace_by(incr_store, incr_store->in(MemNode::Memory));
2366     return;
2367   }
2368 
2369   // Remove trap from optimization list since we add another path to the trap.
2370   bool success = C->remove_unstable_if_trap(trap, true);
2371   assert(success, "Trap already modified");
2372 
2373   // Add a check before the original if that will trap with a certain frequency and execute the original if otherwise
2374   int freq_log = (C->random() % 31) + 1; // Random logarithmic frequency in [1, 31]
2375   Node* mask = intcon(right_n_bits(freq_log));
2376   counter = _gvn.transform(new AndINode(counter, mask));
2377   Node* cmp = _gvn.transform(new CmpINode(counter, intcon(0)));
2378   Node* bol = _gvn.transform(new BoolNode(cmp, BoolTest::mask::eq));
2379   IfNode* iff = _gvn.transform(new IfNode(orig_iff->in(0), bol, orig_iff->_prob, orig_iff->_fcnt))->as_If();
2380   Node* if_true = _gvn.transform(new IfTrueNode(iff));
2381   Node* if_false = _gvn.transform(new IfFalseNode(iff));
2382   assert(!if_true->is_top() && !if_false->is_top(), "trap always / never taken");
2383 
2384   // Trap
2385   assert(trap_proj->outcnt() == 1, "some other nodes are dependent on the trap projection");
2386 
2387   Node* trap_region = new RegionNode(3);
2388   trap_region->set_req(1, trap_proj);
2389   trap_region->set_req(2, if_true);
2390   trap->set_req(0, _gvn.transform(trap_region));
2391 
2392   // Don't trap, execute original if
2393   orig_iff->set_req(0, if_false);
2394 }
2395 
2396 bool Parse::path_is_suitable_for_uncommon_trap(float prob) const {
2397   // Randomly skip emitting an uncommon trap
2398   if (StressUnstableIfTraps && ((C->random() % 2) == 0)) {
2399     return false;
2400   }
2401   // Don't want to speculate on uncommon traps when running with -Xcomp
2402   if (!UseInterpreter) {
2403     return false;
2404   }
2405   return seems_never_taken(prob) &&
2406          !C->too_many_traps(method(), bci(), Deoptimization::Reason_unstable_if);
2407 }
2408 
2409 void Parse::maybe_add_predicate_after_if(Block* path) {
2410   if (path->is_SEL_head() && path->preds_parsed() == 0) {
2411     // Add predicates at bci of if dominating the loop so traps can be
2412     // recorded on the if's profile data
2413     int bc_depth = repush_if_args();
2414     add_parse_predicates();
2415     dec_sp(bc_depth);
2416     path->set_has_predicates();
2417   }
2418 }
2419 
2420 
2421 //----------------------------adjust_map_after_if------------------------------
2422 // Adjust the JVM state to reflect the result of taking this path.
2423 // Basically, it means inspecting the CmpNode controlling this
2424 // branch, seeing how it constrains a tested value, and then
2425 // deciding if it's worth our while to encode this constraint
2426 // as graph nodes in the current abstract interpretation map.
2427 void Parse::adjust_map_after_if(BoolTest::mask btest, Node* c, float prob, Block* path, bool can_trap) {
2428   if (!c->is_Cmp()) {
2429     maybe_add_predicate_after_if(path);
2430     return;
2431   }
2432 
2433   if (stopped() || btest == BoolTest::illegal) {
2434     return;                             // nothing to do
2435   }
2436 
2437   bool is_fallthrough = (path == successor_for_bci(iter().next_bci()));
2438 
2439   if (can_trap && path_is_suitable_for_uncommon_trap(prob)) {
2440     repush_if_args();
2441     Node* call = uncommon_trap(Deoptimization::Reason_unstable_if,
2442                   Deoptimization::Action_reinterpret,
2443                   nullptr,
2444                   (is_fallthrough ? "taken always" : "taken never"));
2445 
2446     if (call != nullptr) {
2447       C->record_unstable_if_trap(new UnstableIfTrap(call->as_CallStaticJava(), path));
2448     }
2449     return;
2450   }
2451 
2452   Node* val = c->in(1);
2453   Node* con = c->in(2);
2454   const Type* tcon = _gvn.type(con);
2455   const Type* tval = _gvn.type(val);
2456   bool have_con = tcon->singleton();
2457   if (tval->singleton()) {
2458     if (!have_con) {
2459       // Swap, so constant is in con.
2460       con  = val;
2461       tcon = tval;
2462       val  = c->in(2);
2463       tval = _gvn.type(val);
2464       btest = BoolTest(btest).commute();
2465       have_con = true;
2466     } else {
2467       // Do we have two constants?  Then leave well enough alone.
2468       have_con = false;
2469     }
2470   }
2471   if (!have_con) {                        // remaining adjustments need a con
2472     maybe_add_predicate_after_if(path);
2473     return;
2474   }
2475 
2476   sharpen_type_after_if(btest, con, tcon, val, tval);
2477   maybe_add_predicate_after_if(path);
2478 }
2479 
2480 
2481 static Node* extract_obj_from_klass_load(PhaseGVN* gvn, Node* n) {
2482   Node* ldk;
2483   if (n->is_DecodeNKlass()) {
2484     if (n->in(1)->Opcode() != Op_LoadNKlass) {
2485       return nullptr;
2486     } else {
2487       ldk = n->in(1);
2488     }
2489   } else if (n->Opcode() != Op_LoadKlass) {
2490     return nullptr;
2491   } else {
2492     ldk = n;
2493   }
2494   assert(ldk != nullptr && ldk->is_Load(), "should have found a LoadKlass or LoadNKlass node");
2495 
2496   Node* adr = ldk->in(MemNode::Address);
2497   intptr_t off = 0;
2498   Node* obj = AddPNode::Ideal_base_and_offset(adr, gvn, off);
2499   if (obj == nullptr || off != oopDesc::klass_offset_in_bytes()) // loading oopDesc::_klass?
2500     return nullptr;
2501   const TypePtr* tp = gvn->type(obj)->is_ptr();
2502   if (tp == nullptr || !(tp->isa_instptr() || tp->isa_aryptr())) // is obj a Java object ptr?
2503     return nullptr;
2504 
2505   return obj;
2506 }
2507 
2508 void Parse::sharpen_type_after_if(BoolTest::mask btest,
2509                                   Node* con, const Type* tcon,
2510                                   Node* val, const Type* tval) {
2511   // Look for opportunities to sharpen the type of a node
2512   // whose klass is compared with a constant klass.
2513   if (btest == BoolTest::eq && tcon->isa_klassptr()) {
2514     Node* obj = extract_obj_from_klass_load(&_gvn, val);
2515     const TypeOopPtr* con_type = tcon->isa_klassptr()->as_instance_type();
2516     if (obj != nullptr && (con_type->isa_instptr() || con_type->isa_aryptr())) {
2517        // Found:
2518        //   Bool(CmpP(LoadKlass(obj._klass), ConP(Foo.klass)), [eq])
2519        // or the narrowOop equivalent.
2520        const Type* obj_type = _gvn.type(obj);
2521        const TypeOopPtr* tboth = obj_type->join_speculative(con_type)->isa_oopptr();
2522        if (tboth != nullptr && tboth->klass_is_exact() && tboth != obj_type &&
2523            tboth->higher_equal(obj_type)) {
2524           // obj has to be of the exact type Foo if the CmpP succeeds.
2525           int obj_in_map = map()->find_edge(obj);
2526           JVMState* jvms = this->jvms();
2527           if (obj_in_map >= 0 &&
2528               (jvms->is_loc(obj_in_map) || jvms->is_stk(obj_in_map))) {
2529             TypeNode* ccast = new CheckCastPPNode(control(), obj, tboth);
2530             const Type* tcc = ccast->as_Type()->type();
2531             assert(tcc != obj_type && tcc->higher_equal(obj_type), "must improve");
2532             // Delay transform() call to allow recovery of pre-cast value
2533             // at the control merge.
2534             _gvn.set_type_bottom(ccast);
2535             record_for_igvn(ccast);
2536             if (tboth->is_inlinetypeptr()) {
2537               ccast = InlineTypeNode::make_from_oop(this, ccast, tboth->exact_klass(true)->as_inline_klass());
2538             }
2539             // Here's the payoff.
2540             replace_in_map(obj, ccast);
2541           }
2542        }
2543     }
2544   }
2545 
2546   int val_in_map = map()->find_edge(val);
2547   if (val_in_map < 0)  return;          // replace_in_map would be useless
2548   {
2549     JVMState* jvms = this->jvms();
2550     if (!(jvms->is_loc(val_in_map) ||
2551           jvms->is_stk(val_in_map)))
2552       return;                           // again, it would be useless
2553   }
2554 
2555   // Check for a comparison to a constant, and "know" that the compared
2556   // value is constrained on this path.
2557   assert(tcon->singleton(), "");
2558   ConstraintCastNode* ccast = nullptr;
2559   Node* cast = nullptr;
2560 
2561   switch (btest) {
2562   case BoolTest::eq:                    // Constant test?
2563     {
2564       const Type* tboth = tcon->join_speculative(tval);
2565       if (tboth == tval)  break;        // Nothing to gain.
2566       if (tcon->isa_int()) {
2567         ccast = new CastIINode(control(), val, tboth);
2568       } else if (tcon == TypePtr::NULL_PTR) {
2569         // Cast to null, but keep the pointer identity temporarily live.
2570         ccast = new CastPPNode(control(), val, tboth);
2571       } else {
2572         const TypeF* tf = tcon->isa_float_constant();
2573         const TypeD* td = tcon->isa_double_constant();
2574         // Exclude tests vs float/double 0 as these could be
2575         // either +0 or -0.  Just because you are equal to +0
2576         // doesn't mean you ARE +0!
2577         // Note, following code also replaces Long and Oop values.
2578         if ((!tf || tf->_f != 0.0) &&
2579             (!td || td->_d != 0.0))
2580           cast = con;                   // Replace non-constant val by con.
2581       }
2582     }
2583     break;
2584 
2585   case BoolTest::ne:
2586     if (tcon == TypePtr::NULL_PTR) {
2587       cast = cast_not_null(val, false);
2588     }
2589     break;
2590 
2591   default:
2592     // (At this point we could record int range types with CastII.)
2593     break;
2594   }
2595 
2596   if (ccast != nullptr) {
2597     const Type* tcc = ccast->as_Type()->type();
2598     assert(tcc != tval && tcc->higher_equal(tval), "must improve");
2599     // Delay transform() call to allow recovery of pre-cast value
2600     // at the control merge.
2601     _gvn.set_type_bottom(ccast);
2602     record_for_igvn(ccast);
2603     cast = ccast;
2604   }
2605 
2606   if (cast != nullptr) {                   // Here's the payoff.
2607     replace_in_map(val, cast);
2608   }
2609 }
2610 
2611 /**
2612  * Use speculative type to optimize CmpP node: if comparison is
2613  * against the low level class, cast the object to the speculative
2614  * type if any. CmpP should then go away.
2615  *
2616  * @param c  expected CmpP node
2617  * @return   result of CmpP on object casted to speculative type
2618  *
2619  */
2620 Node* Parse::optimize_cmp_with_klass(Node* c) {
2621   // If this is transformed by the _gvn to a comparison with the low
2622   // level klass then we may be able to use speculation
2623   if (c->Opcode() == Op_CmpP &&
2624       (c->in(1)->Opcode() == Op_LoadKlass || c->in(1)->Opcode() == Op_DecodeNKlass) &&
2625       c->in(2)->is_Con()) {
2626     Node* load_klass = nullptr;
2627     Node* decode = nullptr;
2628     if (c->in(1)->Opcode() == Op_DecodeNKlass) {
2629       decode = c->in(1);
2630       load_klass = c->in(1)->in(1);
2631     } else {
2632       load_klass = c->in(1);
2633     }
2634     if (load_klass->in(2)->is_AddP()) {
2635       Node* addp = load_klass->in(2);
2636       Node* obj = addp->in(AddPNode::Address);
2637       const TypeOopPtr* obj_type = _gvn.type(obj)->is_oopptr();
2638       if (obj_type->speculative_type_not_null() != nullptr) {
2639         ciKlass* k = obj_type->speculative_type();
2640         inc_sp(2);
2641         obj = maybe_cast_profiled_obj(obj, k);
2642         dec_sp(2);
2643         if (obj->is_InlineType()) {
2644           assert(obj->as_InlineType()->is_allocated(&_gvn), "must be allocated");
2645           obj = obj->as_InlineType()->get_oop();
2646         }
2647         // Make the CmpP use the casted obj
2648         addp = basic_plus_adr(obj, addp->in(AddPNode::Offset));
2649         load_klass = load_klass->clone();
2650         load_klass->set_req(2, addp);
2651         load_klass = _gvn.transform(load_klass);
2652         if (decode != nullptr) {
2653           decode = decode->clone();
2654           decode->set_req(1, load_klass);
2655           load_klass = _gvn.transform(decode);
2656         }
2657         c = c->clone();
2658         c->set_req(1, load_klass);
2659         c = _gvn.transform(c);
2660       }
2661     }
2662   }
2663   return c;
2664 }
2665 
2666 //------------------------------do_one_bytecode--------------------------------
2667 // Parse this bytecode, and alter the Parsers JVM->Node mapping
2668 void Parse::do_one_bytecode() {
2669   Node *a, *b, *c, *d;          // Handy temps
2670   BoolTest::mask btest;
2671   int i;
2672 
2673   assert(!has_exceptions(), "bytecode entry state must be clear of throws");
2674 
2675   if (C->check_node_count(NodeLimitFudgeFactor * 5,
2676                           "out of nodes parsing method")) {
2677     return;
2678   }
2679 
2680 #ifdef ASSERT
2681   // for setting breakpoints
2682   if (TraceOptoParse) {
2683     tty->print(" @");
2684     dump_bci(bci());
2685     tty->print(" %s", Bytecodes::name(bc()));
2686     tty->cr();
2687   }
2688 #endif
2689 
2690   switch (bc()) {
2691   case Bytecodes::_nop:
2692     // do nothing
2693     break;
2694   case Bytecodes::_lconst_0:
2695     push_pair(longcon(0));
2696     break;
2697 
2698   case Bytecodes::_lconst_1:
2699     push_pair(longcon(1));
2700     break;
2701 
2702   case Bytecodes::_fconst_0:
2703     push(zerocon(T_FLOAT));
2704     break;
2705 
2706   case Bytecodes::_fconst_1:
2707     push(makecon(TypeF::ONE));
2708     break;
2709 
2710   case Bytecodes::_fconst_2:
2711     push(makecon(TypeF::make(2.0f)));
2712     break;
2713 
2714   case Bytecodes::_dconst_0:
2715     push_pair(zerocon(T_DOUBLE));
2716     break;
2717 
2718   case Bytecodes::_dconst_1:
2719     push_pair(makecon(TypeD::ONE));
2720     break;
2721 
2722   case Bytecodes::_iconst_m1:push(intcon(-1)); break;
2723   case Bytecodes::_iconst_0: push(intcon( 0)); break;
2724   case Bytecodes::_iconst_1: push(intcon( 1)); break;
2725   case Bytecodes::_iconst_2: push(intcon( 2)); break;
2726   case Bytecodes::_iconst_3: push(intcon( 3)); break;
2727   case Bytecodes::_iconst_4: push(intcon( 4)); break;
2728   case Bytecodes::_iconst_5: push(intcon( 5)); break;
2729   case Bytecodes::_bipush:   push(intcon(iter().get_constant_u1())); break;
2730   case Bytecodes::_sipush:   push(intcon(iter().get_constant_u2())); break;
2731   case Bytecodes::_aconst_null: push(null());  break;
2732 
2733   case Bytecodes::_ldc:
2734   case Bytecodes::_ldc_w:
2735   case Bytecodes::_ldc2_w: {
2736     // ciTypeFlow should trap if the ldc is in error state or if the constant is not loaded
2737     assert(!iter().is_in_error(), "ldc is in error state");
2738     ciConstant constant = iter().get_constant();
2739     assert(constant.is_loaded(), "constant is not loaded");
2740     const Type* con_type = Type::make_from_constant(constant);
2741     if (con_type != nullptr) {
2742       push_node(con_type->basic_type(), makecon(con_type));
2743     }
2744     break;
2745   }
2746 
2747   case Bytecodes::_aload_0:
2748     push( local(0) );
2749     break;
2750   case Bytecodes::_aload_1:
2751     push( local(1) );
2752     break;
2753   case Bytecodes::_aload_2:
2754     push( local(2) );
2755     break;
2756   case Bytecodes::_aload_3:
2757     push( local(3) );
2758     break;
2759   case Bytecodes::_aload:
2760     push( local(iter().get_index()) );
2761     break;
2762 
2763   case Bytecodes::_fload_0:
2764   case Bytecodes::_iload_0:
2765     push( local(0) );
2766     break;
2767   case Bytecodes::_fload_1:
2768   case Bytecodes::_iload_1:
2769     push( local(1) );
2770     break;
2771   case Bytecodes::_fload_2:
2772   case Bytecodes::_iload_2:
2773     push( local(2) );
2774     break;
2775   case Bytecodes::_fload_3:
2776   case Bytecodes::_iload_3:
2777     push( local(3) );
2778     break;
2779   case Bytecodes::_fload:
2780   case Bytecodes::_iload:
2781     push( local(iter().get_index()) );
2782     break;
2783   case Bytecodes::_lload_0:
2784     push_pair_local( 0 );
2785     break;
2786   case Bytecodes::_lload_1:
2787     push_pair_local( 1 );
2788     break;
2789   case Bytecodes::_lload_2:
2790     push_pair_local( 2 );
2791     break;
2792   case Bytecodes::_lload_3:
2793     push_pair_local( 3 );
2794     break;
2795   case Bytecodes::_lload:
2796     push_pair_local( iter().get_index() );
2797     break;
2798 
2799   case Bytecodes::_dload_0:
2800     push_pair_local(0);
2801     break;
2802   case Bytecodes::_dload_1:
2803     push_pair_local(1);
2804     break;
2805   case Bytecodes::_dload_2:
2806     push_pair_local(2);
2807     break;
2808   case Bytecodes::_dload_3:
2809     push_pair_local(3);
2810     break;
2811   case Bytecodes::_dload:
2812     push_pair_local(iter().get_index());
2813     break;
2814   case Bytecodes::_fstore_0:
2815   case Bytecodes::_istore_0:
2816   case Bytecodes::_astore_0:
2817     set_local( 0, pop() );
2818     break;
2819   case Bytecodes::_fstore_1:
2820   case Bytecodes::_istore_1:
2821   case Bytecodes::_astore_1:
2822     set_local( 1, pop() );
2823     break;
2824   case Bytecodes::_fstore_2:
2825   case Bytecodes::_istore_2:
2826   case Bytecodes::_astore_2:
2827     set_local( 2, pop() );
2828     break;
2829   case Bytecodes::_fstore_3:
2830   case Bytecodes::_istore_3:
2831   case Bytecodes::_astore_3:
2832     set_local( 3, pop() );
2833     break;
2834   case Bytecodes::_fstore:
2835   case Bytecodes::_istore:
2836   case Bytecodes::_astore:
2837     set_local( iter().get_index(), pop() );
2838     break;
2839   // long stores
2840   case Bytecodes::_lstore_0:
2841     set_pair_local( 0, pop_pair() );
2842     break;
2843   case Bytecodes::_lstore_1:
2844     set_pair_local( 1, pop_pair() );
2845     break;
2846   case Bytecodes::_lstore_2:
2847     set_pair_local( 2, pop_pair() );
2848     break;
2849   case Bytecodes::_lstore_3:
2850     set_pair_local( 3, pop_pair() );
2851     break;
2852   case Bytecodes::_lstore:
2853     set_pair_local( iter().get_index(), pop_pair() );
2854     break;
2855 
2856   // double stores
2857   case Bytecodes::_dstore_0:
2858     set_pair_local( 0, pop_pair() );
2859     break;
2860   case Bytecodes::_dstore_1:
2861     set_pair_local( 1, pop_pair() );
2862     break;
2863   case Bytecodes::_dstore_2:
2864     set_pair_local( 2, pop_pair() );
2865     break;
2866   case Bytecodes::_dstore_3:
2867     set_pair_local( 3, pop_pair() );
2868     break;
2869   case Bytecodes::_dstore:
2870     set_pair_local( iter().get_index(), pop_pair() );
2871     break;
2872 
2873   case Bytecodes::_pop:  dec_sp(1);   break;
2874   case Bytecodes::_pop2: dec_sp(2);   break;
2875   case Bytecodes::_swap:
2876     a = pop();
2877     b = pop();
2878     push(a);
2879     push(b);
2880     break;
2881   case Bytecodes::_dup:
2882     a = pop();
2883     push(a);
2884     push(a);
2885     break;
2886   case Bytecodes::_dup_x1:
2887     a = pop();
2888     b = pop();
2889     push( a );
2890     push( b );
2891     push( a );
2892     break;
2893   case Bytecodes::_dup_x2:
2894     a = pop();
2895     b = pop();
2896     c = pop();
2897     push( a );
2898     push( c );
2899     push( b );
2900     push( a );
2901     break;
2902   case Bytecodes::_dup2:
2903     a = pop();
2904     b = pop();
2905     push( b );
2906     push( a );
2907     push( b );
2908     push( a );
2909     break;
2910 
2911   case Bytecodes::_dup2_x1:
2912     // before: .. c, b, a
2913     // after:  .. b, a, c, b, a
2914     // not tested
2915     a = pop();
2916     b = pop();
2917     c = pop();
2918     push( b );
2919     push( a );
2920     push( c );
2921     push( b );
2922     push( a );
2923     break;
2924   case Bytecodes::_dup2_x2:
2925     // before: .. d, c, b, a
2926     // after:  .. b, a, d, c, b, a
2927     // not tested
2928     a = pop();
2929     b = pop();
2930     c = pop();
2931     d = pop();
2932     push( b );
2933     push( a );
2934     push( d );
2935     push( c );
2936     push( b );
2937     push( a );
2938     break;
2939 
2940   case Bytecodes::_arraylength: {
2941     // Must do null-check with value on expression stack
2942     Node *ary = null_check(peek(), T_ARRAY);
2943     // Compile-time detect of null-exception?
2944     if (stopped())  return;
2945     a = pop();
2946     push(load_array_length(a));
2947     break;
2948   }
2949 
2950   case Bytecodes::_baload:  array_load(T_BYTE);    break;
2951   case Bytecodes::_caload:  array_load(T_CHAR);    break;
2952   case Bytecodes::_iaload:  array_load(T_INT);     break;
2953   case Bytecodes::_saload:  array_load(T_SHORT);   break;
2954   case Bytecodes::_faload:  array_load(T_FLOAT);   break;
2955   case Bytecodes::_aaload:  array_load(T_OBJECT);  break;
2956   case Bytecodes::_laload:  array_load(T_LONG);    break;
2957   case Bytecodes::_daload:  array_load(T_DOUBLE);  break;
2958   case Bytecodes::_bastore: array_store(T_BYTE);   break;
2959   case Bytecodes::_castore: array_store(T_CHAR);   break;
2960   case Bytecodes::_iastore: array_store(T_INT);    break;
2961   case Bytecodes::_sastore: array_store(T_SHORT);  break;
2962   case Bytecodes::_fastore: array_store(T_FLOAT);  break;
2963   case Bytecodes::_aastore: array_store(T_OBJECT); break;
2964   case Bytecodes::_lastore: array_store(T_LONG);   break;
2965   case Bytecodes::_dastore: array_store(T_DOUBLE); break;
2966 
2967   case Bytecodes::_getfield:
2968     do_getfield();
2969     break;
2970 
2971   case Bytecodes::_getstatic:
2972     do_getstatic();
2973     break;
2974 
2975   case Bytecodes::_putfield:
2976     do_putfield();
2977     break;
2978 
2979   case Bytecodes::_putstatic:
2980     do_putstatic();
2981     break;
2982 
2983   case Bytecodes::_irem:
2984     // Must keep both values on the expression-stack during null-check
2985     zero_check_int(peek());
2986     // Compile-time detect of null-exception?
2987     if (stopped())  return;
2988     b = pop();
2989     a = pop();
2990     push(_gvn.transform(new ModINode(control(), a, b)));
2991     break;
2992   case Bytecodes::_idiv:
2993     // Must keep both values on the expression-stack during null-check
2994     zero_check_int(peek());
2995     // Compile-time detect of null-exception?
2996     if (stopped())  return;
2997     b = pop();
2998     a = pop();
2999     push( _gvn.transform( new DivINode(control(),a,b) ) );
3000     break;
3001   case Bytecodes::_imul:
3002     b = pop(); a = pop();
3003     push( _gvn.transform( new MulINode(a,b) ) );
3004     break;
3005   case Bytecodes::_iadd:
3006     b = pop(); a = pop();
3007     push( _gvn.transform( new AddINode(a,b) ) );
3008     break;
3009   case Bytecodes::_ineg:
3010     a = pop();
3011     push( _gvn.transform( new SubINode(_gvn.intcon(0),a)) );
3012     break;
3013   case Bytecodes::_isub:
3014     b = pop(); a = pop();
3015     push( _gvn.transform( new SubINode(a,b) ) );
3016     break;
3017   case Bytecodes::_iand:
3018     b = pop(); a = pop();
3019     push( _gvn.transform( new AndINode(a,b) ) );
3020     break;
3021   case Bytecodes::_ior:
3022     b = pop(); a = pop();
3023     push( _gvn.transform( new OrINode(a,b) ) );
3024     break;
3025   case Bytecodes::_ixor:
3026     b = pop(); a = pop();
3027     push( _gvn.transform( new XorINode(a,b) ) );
3028     break;
3029   case Bytecodes::_ishl:
3030     b = pop(); a = pop();
3031     push( _gvn.transform( new LShiftINode(a,b) ) );
3032     break;
3033   case Bytecodes::_ishr:
3034     b = pop(); a = pop();
3035     push( _gvn.transform( new RShiftINode(a,b) ) );
3036     break;
3037   case Bytecodes::_iushr:
3038     b = pop(); a = pop();
3039     push( _gvn.transform( new URShiftINode(a,b) ) );
3040     break;
3041 
3042   case Bytecodes::_fneg:
3043     a = pop();
3044     b = _gvn.transform(new NegFNode (a));
3045     push(b);
3046     break;
3047 
3048   case Bytecodes::_fsub:
3049     b = pop();
3050     a = pop();
3051     c = _gvn.transform( new SubFNode(a,b) );
3052     push(c);
3053     break;
3054 
3055   case Bytecodes::_fadd:
3056     b = pop();
3057     a = pop();
3058     c = _gvn.transform( new AddFNode(a,b) );
3059     push(c);
3060     break;
3061 
3062   case Bytecodes::_fmul:
3063     b = pop();
3064     a = pop();
3065     c = _gvn.transform( new MulFNode(a,b) );
3066     push(c);
3067     break;
3068 
3069   case Bytecodes::_fdiv:
3070     b = pop();
3071     a = pop();
3072     c = _gvn.transform( new DivFNode(nullptr,a,b) );
3073     push(c);
3074     break;
3075 
3076   case Bytecodes::_frem:
3077     // Generate a ModF node.
3078     b = pop();
3079     a = pop();
3080     push(floating_point_mod(a, b, BasicType::T_FLOAT));
3081     break;
3082 
3083   case Bytecodes::_fcmpl:
3084     b = pop();
3085     a = pop();
3086     c = _gvn.transform( new CmpF3Node( a, b));
3087     push(c);
3088     break;
3089   case Bytecodes::_fcmpg:
3090     b = pop();
3091     a = pop();
3092 
3093     // Same as fcmpl but need to flip the unordered case.  Swap the inputs,
3094     // which negates the result sign except for unordered.  Flip the unordered
3095     // as well by using CmpF3 which implements unordered-lesser instead of
3096     // unordered-greater semantics.  Finally, commute the result bits.  Result
3097     // is same as using a CmpF3Greater except we did it with CmpF3 alone.
3098     c = _gvn.transform( new CmpF3Node( b, a));
3099     c = _gvn.transform( new SubINode(_gvn.intcon(0),c) );
3100     push(c);
3101     break;
3102 
3103   case Bytecodes::_f2i:
3104     a = pop();
3105     push(_gvn.transform(new ConvF2INode(a)));
3106     break;
3107 
3108   case Bytecodes::_d2i:
3109     a = pop_pair();
3110     b = _gvn.transform(new ConvD2INode(a));
3111     push( b );
3112     break;
3113 
3114   case Bytecodes::_f2d:
3115     a = pop();
3116     b = _gvn.transform( new ConvF2DNode(a));
3117     push_pair( b );
3118     break;
3119 
3120   case Bytecodes::_d2f:
3121     a = pop_pair();
3122     b = _gvn.transform( new ConvD2FNode(a));
3123     push( b );
3124     break;
3125 
3126   case Bytecodes::_l2f:
3127     if (Matcher::convL2FSupported()) {
3128       a = pop_pair();
3129       b = _gvn.transform( new ConvL2FNode(a));
3130       push(b);
3131     } else {
3132       l2f();
3133     }
3134     break;
3135 
3136   case Bytecodes::_l2d:
3137     a = pop_pair();
3138     b = _gvn.transform( new ConvL2DNode(a));
3139     push_pair(b);
3140     break;
3141 
3142   case Bytecodes::_f2l:
3143     a = pop();
3144     b = _gvn.transform( new ConvF2LNode(a));
3145     push_pair(b);
3146     break;
3147 
3148   case Bytecodes::_d2l:
3149     a = pop_pair();
3150     b = _gvn.transform( new ConvD2LNode(a));
3151     push_pair(b);
3152     break;
3153 
3154   case Bytecodes::_dsub:
3155     b = pop_pair();
3156     a = pop_pair();
3157     c = _gvn.transform( new SubDNode(a,b) );
3158     push_pair(c);
3159     break;
3160 
3161   case Bytecodes::_dadd:
3162     b = pop_pair();
3163     a = pop_pair();
3164     c = _gvn.transform( new AddDNode(a,b) );
3165     push_pair(c);
3166     break;
3167 
3168   case Bytecodes::_dmul:
3169     b = pop_pair();
3170     a = pop_pair();
3171     c = _gvn.transform( new MulDNode(a,b) );
3172     push_pair(c);
3173     break;
3174 
3175   case Bytecodes::_ddiv:
3176     b = pop_pair();
3177     a = pop_pair();
3178     c = _gvn.transform( new DivDNode(nullptr,a,b) );
3179     push_pair(c);
3180     break;
3181 
3182   case Bytecodes::_dneg:
3183     a = pop_pair();
3184     b = _gvn.transform(new NegDNode (a));
3185     push_pair(b);
3186     break;
3187 
3188   case Bytecodes::_drem:
3189     // Generate a ModD node.
3190     b = pop_pair();
3191     a = pop_pair();
3192     push_pair(floating_point_mod(a, b, BasicType::T_DOUBLE));
3193     break;
3194 
3195   case Bytecodes::_dcmpl:
3196     b = pop_pair();
3197     a = pop_pair();
3198     c = _gvn.transform( new CmpD3Node( a, b));
3199     push(c);
3200     break;
3201 
3202   case Bytecodes::_dcmpg:
3203     b = pop_pair();
3204     a = pop_pair();
3205     // Same as dcmpl but need to flip the unordered case.
3206     // Commute the inputs, which negates the result sign except for unordered.
3207     // Flip the unordered as well by using CmpD3 which implements
3208     // unordered-lesser instead of unordered-greater semantics.
3209     // Finally, negate the result bits.  Result is same as using a
3210     // CmpD3Greater except we did it with CmpD3 alone.
3211     c = _gvn.transform( new CmpD3Node( b, a));
3212     c = _gvn.transform( new SubINode(_gvn.intcon(0),c) );
3213     push(c);
3214     break;
3215 
3216 
3217     // Note for longs -> lo word is on TOS, hi word is on TOS - 1
3218   case Bytecodes::_land:
3219     b = pop_pair();
3220     a = pop_pair();
3221     c = _gvn.transform( new AndLNode(a,b) );
3222     push_pair(c);
3223     break;
3224   case Bytecodes::_lor:
3225     b = pop_pair();
3226     a = pop_pair();
3227     c = _gvn.transform( new OrLNode(a,b) );
3228     push_pair(c);
3229     break;
3230   case Bytecodes::_lxor:
3231     b = pop_pair();
3232     a = pop_pair();
3233     c = _gvn.transform( new XorLNode(a,b) );
3234     push_pair(c);
3235     break;
3236 
3237   case Bytecodes::_lshl:
3238     b = pop();                  // the shift count
3239     a = pop_pair();             // value to be shifted
3240     c = _gvn.transform( new LShiftLNode(a,b) );
3241     push_pair(c);
3242     break;
3243   case Bytecodes::_lshr:
3244     b = pop();                  // the shift count
3245     a = pop_pair();             // value to be shifted
3246     c = _gvn.transform( new RShiftLNode(a,b) );
3247     push_pair(c);
3248     break;
3249   case Bytecodes::_lushr:
3250     b = pop();                  // the shift count
3251     a = pop_pair();             // value to be shifted
3252     c = _gvn.transform( new URShiftLNode(a,b) );
3253     push_pair(c);
3254     break;
3255   case Bytecodes::_lmul:
3256     b = pop_pair();
3257     a = pop_pair();
3258     c = _gvn.transform( new MulLNode(a,b) );
3259     push_pair(c);
3260     break;
3261 
3262   case Bytecodes::_lrem:
3263     // Must keep both values on the expression-stack during null-check
3264     assert(peek(0) == top(), "long word order");
3265     zero_check_long(peek(1));
3266     // Compile-time detect of null-exception?
3267     if (stopped())  return;
3268     b = pop_pair();
3269     a = pop_pair();
3270     c = _gvn.transform( new ModLNode(control(),a,b) );
3271     push_pair(c);
3272     break;
3273 
3274   case Bytecodes::_ldiv:
3275     // Must keep both values on the expression-stack during null-check
3276     assert(peek(0) == top(), "long word order");
3277     zero_check_long(peek(1));
3278     // Compile-time detect of null-exception?
3279     if (stopped())  return;
3280     b = pop_pair();
3281     a = pop_pair();
3282     c = _gvn.transform( new DivLNode(control(),a,b) );
3283     push_pair(c);
3284     break;
3285 
3286   case Bytecodes::_ladd:
3287     b = pop_pair();
3288     a = pop_pair();
3289     c = _gvn.transform( new AddLNode(a,b) );
3290     push_pair(c);
3291     break;
3292   case Bytecodes::_lsub:
3293     b = pop_pair();
3294     a = pop_pair();
3295     c = _gvn.transform( new SubLNode(a,b) );
3296     push_pair(c);
3297     break;
3298   case Bytecodes::_lcmp:
3299     // Safepoints are now inserted _before_ branches.  The long-compare
3300     // bytecode painfully produces a 3-way value (-1,0,+1) which requires a
3301     // slew of control flow.  These are usually followed by a CmpI vs zero and
3302     // a branch; this pattern then optimizes to the obvious long-compare and
3303     // branch.  However, if the branch is backwards there's a Safepoint
3304     // inserted.  The inserted Safepoint captures the JVM state at the
3305     // pre-branch point, i.e. it captures the 3-way value.  Thus if a
3306     // long-compare is used to control a loop the debug info will force
3307     // computation of the 3-way value, even though the generated code uses a
3308     // long-compare and branch.  We try to rectify the situation by inserting
3309     // a SafePoint here and have it dominate and kill the safepoint added at a
3310     // following backwards branch.  At this point the JVM state merely holds 2
3311     // longs but not the 3-way value.
3312     switch (iter().next_bc()) {
3313       case Bytecodes::_ifgt:
3314       case Bytecodes::_iflt:
3315       case Bytecodes::_ifge:
3316       case Bytecodes::_ifle:
3317       case Bytecodes::_ifne:
3318       case Bytecodes::_ifeq:
3319         // If this is a backwards branch in the bytecodes, add Safepoint
3320         maybe_add_safepoint(iter().next_get_dest());
3321       default:
3322         break;
3323     }
3324     b = pop_pair();
3325     a = pop_pair();
3326     c = _gvn.transform( new CmpL3Node( a, b ));
3327     push(c);
3328     break;
3329 
3330   case Bytecodes::_lneg:
3331     a = pop_pair();
3332     b = _gvn.transform( new SubLNode(longcon(0),a));
3333     push_pair(b);
3334     break;
3335   case Bytecodes::_l2i:
3336     a = pop_pair();
3337     push( _gvn.transform( new ConvL2INode(a)));
3338     break;
3339   case Bytecodes::_i2l:
3340     a = pop();
3341     b = _gvn.transform( new ConvI2LNode(a));
3342     push_pair(b);
3343     break;
3344   case Bytecodes::_i2b:
3345     // Sign extend
3346     a = pop();
3347     a = Compile::narrow_value(T_BYTE, a, nullptr, &_gvn, true);
3348     push(a);
3349     break;
3350   case Bytecodes::_i2s:
3351     a = pop();
3352     a = Compile::narrow_value(T_SHORT, a, nullptr, &_gvn, true);
3353     push(a);
3354     break;
3355   case Bytecodes::_i2c:
3356     a = pop();
3357     a = Compile::narrow_value(T_CHAR, a, nullptr, &_gvn, true);
3358     push(a);
3359     break;
3360 
3361   case Bytecodes::_i2f:
3362     a = pop();
3363     b = _gvn.transform( new ConvI2FNode(a) ) ;
3364     push(b);
3365     break;
3366 
3367   case Bytecodes::_i2d:
3368     a = pop();
3369     b = _gvn.transform( new ConvI2DNode(a));
3370     push_pair(b);
3371     break;
3372 
3373   case Bytecodes::_iinc:        // Increment local
3374     i = iter().get_index();     // Get local index
3375     set_local( i, _gvn.transform( new AddINode( _gvn.intcon(iter().get_iinc_con()), local(i) ) ) );
3376     break;
3377 
3378   // Exit points of synchronized methods must have an unlock node
3379   case Bytecodes::_return:
3380     return_current(nullptr);
3381     break;
3382 
3383   case Bytecodes::_ireturn:
3384   case Bytecodes::_areturn:
3385   case Bytecodes::_freturn:
3386     return_current(cast_to_non_larval(pop()));
3387     break;
3388   case Bytecodes::_lreturn:
3389   case Bytecodes::_dreturn:
3390     return_current(pop_pair());
3391     break;
3392 
3393   case Bytecodes::_athrow:
3394     // null exception oop throws null pointer exception
3395     null_check(peek());
3396     if (stopped())  return;
3397     // Hook the thrown exception directly to subsequent handlers.
3398     if (BailoutToInterpreterForThrows) {
3399       // Keep method interpreted from now on.
3400       uncommon_trap(Deoptimization::Reason_unhandled,
3401                     Deoptimization::Action_make_not_compilable);
3402       return;
3403     }
3404     if (env()->jvmti_can_post_on_exceptions()) {
3405       // check if we must post exception events, take uncommon trap if so (with must_throw = false)
3406       uncommon_trap_if_should_post_on_exceptions(Deoptimization::Reason_unhandled, false);
3407     }
3408     // Here if either can_post_on_exceptions or should_post_on_exceptions is false
3409     add_exception_state(make_exception_state(peek()));
3410     break;
3411 
3412   case Bytecodes::_goto:   // fall through
3413   case Bytecodes::_goto_w: {
3414     int target_bci = (bc() == Bytecodes::_goto) ? iter().get_dest() : iter().get_far_dest();
3415 
3416     // If this is a backwards branch in the bytecodes, add Safepoint
3417     maybe_add_safepoint(target_bci);
3418 
3419     // Merge the current control into the target basic block
3420     merge(target_bci);
3421 
3422     // See if we can get some profile data and hand it off to the next block
3423     Block *target_block = block()->successor_for_bci(target_bci);
3424     if (target_block->pred_count() != 1)  break;
3425     ciMethodData* methodData = method()->method_data();
3426     if (!methodData->is_mature())  break;
3427     ciProfileData* data = methodData->bci_to_data(bci());
3428     assert(data != nullptr && data->is_JumpData(), "need JumpData for taken branch");
3429     int taken = ((ciJumpData*)data)->taken();
3430     taken = method()->scale_count(taken);
3431     target_block->set_count(taken);
3432     break;
3433   }
3434 
3435   case Bytecodes::_ifnull:    btest = BoolTest::eq; goto handle_if_null;
3436   case Bytecodes::_ifnonnull: btest = BoolTest::ne; goto handle_if_null;
3437   handle_if_null:
3438     // If this is a backwards branch in the bytecodes, add Safepoint
3439     maybe_add_safepoint(iter().get_dest());
3440     a = null();
3441     b = cast_to_non_larval(pop());
3442     if (b->is_InlineType()) {
3443       // Null checking a scalarized but nullable inline type. Check the IsInit
3444       // input instead of the oop input to avoid keeping buffer allocations alive
3445       c = _gvn.transform(new CmpINode(b->as_InlineType()->get_is_init(), zerocon(T_INT)));
3446     } else {
3447       if (!_gvn.type(b)->speculative_maybe_null() &&
3448           !too_many_traps(Deoptimization::Reason_speculate_null_check)) {
3449         inc_sp(1);
3450         Node* null_ctl = top();
3451         b = null_check_oop(b, &null_ctl, true, true, true);
3452         assert(null_ctl->is_top(), "no null control here");
3453         dec_sp(1);
3454       } else if (_gvn.type(b)->speculative_always_null() &&
3455                  !too_many_traps(Deoptimization::Reason_speculate_null_assert)) {
3456         inc_sp(1);
3457         b = null_assert(b);
3458         dec_sp(1);
3459       }
3460       c = _gvn.transform( new CmpPNode(b, a) );
3461     }
3462     do_ifnull(btest, c);
3463     break;
3464 
3465   case Bytecodes::_if_acmpeq: btest = BoolTest::eq; goto handle_if_acmp;
3466   case Bytecodes::_if_acmpne: btest = BoolTest::ne; goto handle_if_acmp;
3467   handle_if_acmp:
3468     // If this is a backwards branch in the bytecodes, add Safepoint
3469     maybe_add_safepoint(iter().get_dest());
3470     a = cast_to_non_larval(pop());
3471     b = cast_to_non_larval(pop());
3472     do_acmp(btest, b, a);
3473     break;
3474 
3475   case Bytecodes::_ifeq: btest = BoolTest::eq; goto handle_ifxx;
3476   case Bytecodes::_ifne: btest = BoolTest::ne; goto handle_ifxx;
3477   case Bytecodes::_iflt: btest = BoolTest::lt; goto handle_ifxx;
3478   case Bytecodes::_ifle: btest = BoolTest::le; goto handle_ifxx;
3479   case Bytecodes::_ifgt: btest = BoolTest::gt; goto handle_ifxx;
3480   case Bytecodes::_ifge: btest = BoolTest::ge; goto handle_ifxx;
3481   handle_ifxx:
3482     // If this is a backwards branch in the bytecodes, add Safepoint
3483     maybe_add_safepoint(iter().get_dest());
3484     a = _gvn.intcon(0);
3485     b = pop();
3486     c = _gvn.transform( new CmpINode(b, a) );
3487     do_if(btest, c);
3488     break;
3489 
3490   case Bytecodes::_if_icmpeq: btest = BoolTest::eq; goto handle_if_icmp;
3491   case Bytecodes::_if_icmpne: btest = BoolTest::ne; goto handle_if_icmp;
3492   case Bytecodes::_if_icmplt: btest = BoolTest::lt; goto handle_if_icmp;
3493   case Bytecodes::_if_icmple: btest = BoolTest::le; goto handle_if_icmp;
3494   case Bytecodes::_if_icmpgt: btest = BoolTest::gt; goto handle_if_icmp;
3495   case Bytecodes::_if_icmpge: btest = BoolTest::ge; goto handle_if_icmp;
3496   handle_if_icmp:
3497     // If this is a backwards branch in the bytecodes, add Safepoint
3498     maybe_add_safepoint(iter().get_dest());
3499     a = pop();
3500     b = pop();
3501     c = _gvn.transform( new CmpINode( b, a ) );
3502     do_if(btest, c);
3503     break;
3504 
3505   case Bytecodes::_tableswitch:
3506     do_tableswitch();
3507     break;
3508 
3509   case Bytecodes::_lookupswitch:
3510     do_lookupswitch();
3511     break;
3512 
3513   case Bytecodes::_invokestatic:
3514   case Bytecodes::_invokedynamic:
3515   case Bytecodes::_invokespecial:
3516   case Bytecodes::_invokevirtual:
3517   case Bytecodes::_invokeinterface:
3518     do_call();
3519     break;
3520   case Bytecodes::_checkcast:
3521     do_checkcast();
3522     break;
3523   case Bytecodes::_instanceof:
3524     do_instanceof();
3525     break;
3526   case Bytecodes::_anewarray:
3527     do_newarray();
3528     break;
3529   case Bytecodes::_newarray:
3530     do_newarray((BasicType)iter().get_index());
3531     break;
3532   case Bytecodes::_multianewarray:
3533     do_multianewarray();
3534     break;
3535   case Bytecodes::_new:
3536     do_new();
3537     break;
3538 
3539   case Bytecodes::_jsr:
3540   case Bytecodes::_jsr_w:
3541     do_jsr();
3542     break;
3543 
3544   case Bytecodes::_ret:
3545     do_ret();
3546     break;
3547 
3548 
3549   case Bytecodes::_monitorenter:
3550     do_monitor_enter();
3551     break;
3552 
3553   case Bytecodes::_monitorexit:
3554     do_monitor_exit();
3555     break;
3556 
3557   case Bytecodes::_breakpoint:
3558     // Breakpoint set concurrently to compile
3559     // %%% use an uncommon trap?
3560     C->record_failure("breakpoint in method");
3561     return;
3562 
3563   default:
3564 #ifndef PRODUCT
3565     map()->dump(99);
3566 #endif
3567     tty->print("\nUnhandled bytecode %s\n", Bytecodes::name(bc()) );
3568     ShouldNotReachHere();
3569   }
3570 
3571 #ifndef PRODUCT
3572   if (failing()) { return; }
3573   constexpr int perBytecode = 6;
3574   if (C->should_print_igv(perBytecode)) {
3575     IdealGraphPrinter* printer = C->igv_printer();
3576     char buffer[256];
3577     jio_snprintf(buffer, sizeof(buffer), "Bytecode %d: %s, map: %d", bci(), Bytecodes::name(bc()), map() == nullptr ? -1 : map()->_idx);
3578     bool old = printer->traverse_outs();
3579     printer->set_traverse_outs(true);
3580     printer->print_graph(buffer);
3581     printer->set_traverse_outs(old);
3582   }
3583 #endif
3584 }