1 /*
   2  * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "jvm.h"
  27 #include "classfile/javaClasses.inline.hpp"
  28 #include "classfile/symbolTable.hpp"
  29 #include "classfile/systemDictionary.hpp"
  30 #include "classfile/vmClasses.hpp"
  31 #include "code/codeCache.hpp"
  32 #include "code/debugInfoRec.hpp"
  33 #include "code/nmethod.hpp"
  34 #include "code/pcDesc.hpp"
  35 #include "code/scopeDesc.hpp"
  36 #include "compiler/compilationPolicy.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "interpreter/bytecode.hpp"
  39 #include "interpreter/interpreter.hpp"
  40 #include "interpreter/oopMapCache.hpp"
  41 #include "memory/allocation.inline.hpp"
  42 #include "memory/oopFactory.hpp"
  43 #include "memory/resourceArea.hpp"
  44 #include "memory/universe.hpp"
  45 #include "oops/constantPool.hpp"
  46 #include "oops/flatArrayKlass.hpp"
  47 #include "oops/flatArrayOop.hpp"
  48 #include "oops/method.hpp"
  49 #include "oops/objArrayKlass.hpp"
  50 #include "oops/objArrayOop.inline.hpp"
  51 #include "oops/oop.inline.hpp"
  52 #include "oops/fieldStreams.inline.hpp"
  53 #include "oops/inlineKlass.inline.hpp"
  54 #include "oops/typeArrayOop.inline.hpp"
  55 #include "oops/verifyOopClosure.hpp"
  56 #include "prims/jvmtiDeferredUpdates.hpp"
  57 #include "prims/jvmtiExport.hpp"
  58 #include "prims/jvmtiThreadState.hpp"
  59 #include "prims/vectorSupport.hpp"
  60 #include "prims/methodHandles.hpp"
  61 #include "runtime/atomic.hpp"
  62 #include "runtime/deoptimization.hpp"
  63 #include "runtime/escapeBarrier.hpp"
  64 #include "runtime/fieldDescriptor.hpp"
  65 #include "runtime/fieldDescriptor.inline.hpp"
  66 #include "runtime/frame.inline.hpp"
  67 #include "runtime/handles.inline.hpp"
  68 #include "runtime/interfaceSupport.inline.hpp"
  69 #include "runtime/jniHandles.inline.hpp"
  70 #include "runtime/keepStackGCProcessed.hpp"
  71 #include "runtime/objectMonitor.inline.hpp"
  72 #include "runtime/osThread.hpp"
  73 #include "runtime/safepointVerifiers.hpp"
  74 #include "runtime/sharedRuntime.hpp"
  75 #include "runtime/signature.hpp"
  76 #include "runtime/stackFrameStream.inline.hpp"
  77 #include "runtime/stackWatermarkSet.hpp"
  78 #include "runtime/stubRoutines.hpp"
  79 #include "runtime/thread.hpp"
  80 #include "runtime/threadSMR.hpp"
  81 #include "runtime/threadWXSetters.inline.hpp"
  82 #include "runtime/vframe.hpp"
  83 #include "runtime/vframeArray.hpp"
  84 #include "runtime/vframe_hp.hpp"
  85 #include "runtime/vmOperations.hpp"
  86 #include "utilities/events.hpp"
  87 #include "utilities/macros.hpp"
  88 #include "utilities/preserveException.hpp"
  89 #include "utilities/xmlstream.hpp"
  90 #if INCLUDE_JFR
  91 #include "jfr/jfrEvents.hpp"
  92 #include "jfr/metadata/jfrSerializer.hpp"
  93 #endif
  94 
  95 bool DeoptimizationMarker::_is_active = false;
  96 
  97 Deoptimization::UnrollBlock::UnrollBlock(int  size_of_deoptimized_frame,
  98                                          int  caller_adjustment,
  99                                          int  caller_actual_parameters,
 100                                          int  number_of_frames,
 101                                          intptr_t* frame_sizes,
 102                                          address* frame_pcs,
 103                                          BasicType return_type,
 104                                          int exec_mode) {
 105   _size_of_deoptimized_frame = size_of_deoptimized_frame;
 106   _caller_adjustment         = caller_adjustment;
 107   _caller_actual_parameters  = caller_actual_parameters;
 108   _number_of_frames          = number_of_frames;
 109   _frame_sizes               = frame_sizes;
 110   _frame_pcs                 = frame_pcs;
 111   _register_block            = NEW_C_HEAP_ARRAY(intptr_t, RegisterMap::reg_count * 2, mtCompiler);
 112   _return_type               = return_type;
 113   _initial_info              = 0;
 114   // PD (x86 only)
 115   _counter_temp              = 0;
 116   _unpack_kind               = exec_mode;
 117   _sender_sp_temp            = 0;
 118 
 119   _total_frame_sizes         = size_of_frames();
 120   assert(exec_mode >= 0 && exec_mode < Unpack_LIMIT, "Unexpected exec_mode");
 121 }
 122 
 123 
 124 Deoptimization::UnrollBlock::~UnrollBlock() {
 125   FREE_C_HEAP_ARRAY(intptr_t, _frame_sizes);
 126   FREE_C_HEAP_ARRAY(intptr_t, _frame_pcs);
 127   FREE_C_HEAP_ARRAY(intptr_t, _register_block);
 128 }
 129 
 130 
 131 intptr_t* Deoptimization::UnrollBlock::value_addr_at(int register_number) const {
 132   assert(register_number < RegisterMap::reg_count, "checking register number");
 133   return &_register_block[register_number * 2];
 134 }
 135 
 136 
 137 
 138 int Deoptimization::UnrollBlock::size_of_frames() const {
 139   // Acount first for the adjustment of the initial frame
 140   int result = _caller_adjustment;
 141   for (int index = 0; index < number_of_frames(); index++) {
 142     result += frame_sizes()[index];
 143   }
 144   return result;
 145 }
 146 
 147 
 148 void Deoptimization::UnrollBlock::print() {
 149   ResourceMark rm;
 150   stringStream st;
 151   st.print_cr("UnrollBlock");
 152   st.print_cr("  size_of_deoptimized_frame = %d", _size_of_deoptimized_frame);
 153   st.print(   "  frame_sizes: ");
 154   for (int index = 0; index < number_of_frames(); index++) {
 155     st.print(INTX_FORMAT " ", frame_sizes()[index]);
 156   }
 157   st.cr();
 158   tty->print_raw(st.as_string());
 159 }
 160 
 161 
 162 // In order to make fetch_unroll_info work properly with escape
 163 // analysis, the method was changed from JRT_LEAF to JRT_BLOCK_ENTRY.
 164 // The actual reallocation of previously eliminated objects occurs in realloc_objects,
 165 // which is called from the method fetch_unroll_info_helper below.
 166 JRT_BLOCK_ENTRY(Deoptimization::UnrollBlock*, Deoptimization::fetch_unroll_info(JavaThread* current, int exec_mode))
 167   // fetch_unroll_info() is called at the beginning of the deoptimization
 168   // handler. Note this fact before we start generating temporary frames
 169   // that can confuse an asynchronous stack walker. This counter is
 170   // decremented at the end of unpack_frames().
 171   if (TraceDeoptimization) {
 172     tty->print_cr("Deoptimizing thread " INTPTR_FORMAT, p2i(current));
 173   }
 174   current->inc_in_deopt_handler();
 175 
 176   if (exec_mode == Unpack_exception) {
 177     // When we get here, a callee has thrown an exception into a deoptimized
 178     // frame. That throw might have deferred stack watermark checking until
 179     // after unwinding. So we deal with such deferred requests here.
 180     StackWatermarkSet::after_unwind(current);
 181   }
 182 
 183   return fetch_unroll_info_helper(current, exec_mode);
 184 JRT_END
 185 
 186 #if COMPILER2_OR_JVMCI
 187 #ifndef PRODUCT
 188 // print information about reallocated objects
 189 static void print_objects(JavaThread* deoptee_thread,
 190                           GrowableArray<ScopeValue*>* objects, bool realloc_failures) {
 191   ResourceMark rm;
 192   stringStream st;  // change to logStream with logging
 193   st.print_cr("REALLOC OBJECTS in thread " INTPTR_FORMAT, p2i(deoptee_thread));
 194   fieldDescriptor fd;
 195 
 196   for (int i = 0; i < objects->length(); i++) {
 197     ObjectValue* sv = (ObjectValue*) objects->at(i);
 198     Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
 199     Handle obj = sv->value();
 200 
 201     st.print("     object <" INTPTR_FORMAT "> of type ", p2i(sv->value()()));
 202     k->print_value_on(&st);
 203     assert(obj.not_null() || k->is_inline_klass() || realloc_failures, "reallocation was missed");
 204     if (obj.is_null()) {
 205       if (k->is_inline_klass()) {
 206         st.print(" is null");
 207       } else {
 208         st.print(" allocation failed");
 209       }
 210     } else {
 211       st.print(" allocated (%d bytes)", obj->size() * HeapWordSize);
 212     }
 213     st.cr();
 214 
 215     if (Verbose && !obj.is_null()) {
 216       k->oop_print_on(obj(), &st);
 217     }
 218   }
 219   tty->print_raw(st.as_string());
 220 }
 221 #endif
 222 
 223 static bool rematerialize_objects(JavaThread* thread, int exec_mode, CompiledMethod* compiled_method,
 224                                   frame& deoptee, RegisterMap& map, GrowableArray<compiledVFrame*>* chunk,
 225                                   bool& deoptimized_objects) {
 226   bool realloc_failures = false;
 227   assert (chunk->at(0)->scope() != NULL,"expect only compiled java frames");
 228 
 229   JavaThread* deoptee_thread = chunk->at(0)->thread();
 230   assert(exec_mode == Deoptimization::Unpack_none || (deoptee_thread == thread),
 231          "a frame can only be deoptimized by the owner thread");
 232 
 233   GrowableArray<ScopeValue*>* objects = chunk->at(0)->scope()->objects();
 234 
 235   // The flag return_oop() indicates call sites which return oop
 236   // in compiled code. Such sites include java method calls,
 237   // runtime calls (for example, used to allocate new objects/arrays
 238   // on slow code path) and any other calls generated in compiled code.
 239   // It is not guaranteed that we can get such information here only
 240   // by analyzing bytecode in deoptimized frames. This is why this flag
 241   // is set during method compilation (see Compile::Process_OopMap_Node()).
 242   // If the previous frame was popped or if we are dispatching an exception,
 243   // we don't have an oop result.
 244   ScopeDesc* scope = chunk->at(0)->scope();
 245   bool save_oop_result = scope->return_oop() && !thread->popframe_forcing_deopt_reexecution() && (exec_mode == Deoptimization::Unpack_deopt);
 246   // In case of the return of multiple values, we must take care
 247   // of all oop return values.
 248   GrowableArray<Handle> return_oops;
 249   InlineKlass* vk = NULL;
 250   if (save_oop_result && scope->return_scalarized()) {
 251     vk = InlineKlass::returned_inline_klass(map);
 252     if (vk != NULL) {
 253       vk->save_oop_fields(map, return_oops);
 254       save_oop_result = false;
 255     }
 256   }
 257   if (save_oop_result) {
 258     // Reallocation may trigger GC. If deoptimization happened on return from
 259     // call which returns oop we need to save it since it is not in oopmap.
 260     oop result = deoptee.saved_oop_result(&map);
 261     assert(oopDesc::is_oop_or_null(result), "must be oop");
 262     return_oops.push(Handle(thread, result));
 263     assert(Universe::heap()->is_in_or_null(result), "must be heap pointer");
 264     if (TraceDeoptimization) {
 265       tty->print_cr("SAVED OOP RESULT " INTPTR_FORMAT " in thread " INTPTR_FORMAT, p2i(result), p2i(thread));
 266     }
 267   }
 268   if (objects != NULL || vk != NULL) {
 269     if (exec_mode == Deoptimization::Unpack_none) {
 270       assert(thread->thread_state() == _thread_in_vm, "assumption");
 271       JavaThread* THREAD = thread; // For exception macros.
 272       // Clear pending OOM if reallocation fails and return true indicating allocation failure
 273       if (vk != NULL) {
 274         realloc_failures = Deoptimization::realloc_inline_type_result(vk, map, return_oops, CHECK_AND_CLEAR_(true));
 275       }
 276       if (objects != NULL) {
 277         realloc_failures = realloc_failures || Deoptimization::realloc_objects(thread, &deoptee, &map, objects, CHECK_AND_CLEAR_(true));
 278         bool skip_internal = (compiled_method != NULL) && !compiled_method->is_compiled_by_jvmci();
 279         Deoptimization::reassign_fields(&deoptee, &map, objects, realloc_failures, skip_internal, CHECK_AND_CLEAR_(true));
 280       }
 281       deoptimized_objects = true;
 282     } else {
 283       JavaThread* current = thread; // For JRT_BLOCK
 284       JRT_BLOCK
 285       if (vk != NULL) {
 286         realloc_failures = Deoptimization::realloc_inline_type_result(vk, map, return_oops, THREAD);
 287       }
 288       if (objects != NULL) {
 289         realloc_failures = realloc_failures || Deoptimization::realloc_objects(thread, &deoptee, &map, objects, THREAD);
 290         bool skip_internal = (compiled_method != NULL) && !compiled_method->is_compiled_by_jvmci();
 291         Deoptimization::reassign_fields(&deoptee, &map, objects, realloc_failures, skip_internal, THREAD);
 292       }
 293       JRT_END
 294     }


 295 #ifndef PRODUCT
 296     if (TraceDeoptimization) {
 297       print_objects(deoptee_thread, objects, realloc_failures);
 298     }
 299 #endif
 300   }
 301   if (save_oop_result || vk != NULL) {
 302     // Restore result.
 303     assert(return_oops.length() == 1, "no inline type");
 304     deoptee.set_saved_oop_result(&map, return_oops.pop()());
 305   }
 306   return realloc_failures;
 307 }
 308 
 309 static void restore_eliminated_locks(JavaThread* thread, GrowableArray<compiledVFrame*>* chunk, bool realloc_failures,
 310                                      frame& deoptee, int exec_mode, bool& deoptimized_objects) {
 311   JavaThread* deoptee_thread = chunk->at(0)->thread();
 312   assert(!EscapeBarrier::objs_are_deoptimized(deoptee_thread, deoptee.id()), "must relock just once");
 313   assert(thread == Thread::current(), "should be");
 314   HandleMark hm(thread);
 315 #ifndef PRODUCT
 316   bool first = true;
 317 #endif
 318   for (int i = 0; i < chunk->length(); i++) {
 319     compiledVFrame* cvf = chunk->at(i);
 320     assert (cvf->scope() != NULL,"expect only compiled java frames");
 321     GrowableArray<MonitorInfo*>* monitors = cvf->monitors();
 322     if (monitors->is_nonempty()) {
 323       bool relocked = Deoptimization::relock_objects(thread, monitors, deoptee_thread, deoptee,
 324                                                      exec_mode, realloc_failures);
 325       deoptimized_objects = deoptimized_objects || relocked;
 326 #ifndef PRODUCT
 327       if (PrintDeoptimizationDetails) {
 328         ResourceMark rm;
 329         stringStream st;
 330         for (int j = 0; j < monitors->length(); j++) {
 331           MonitorInfo* mi = monitors->at(j);
 332           if (mi->eliminated()) {
 333             if (first) {
 334               first = false;
 335               st.print_cr("RELOCK OBJECTS in thread " INTPTR_FORMAT, p2i(thread));
 336             }
 337             if (exec_mode == Deoptimization::Unpack_none) {
 338               ObjectMonitor* monitor = deoptee_thread->current_waiting_monitor();
 339               if (monitor != NULL && monitor->object() == mi->owner()) {
 340                 st.print_cr("     object <" INTPTR_FORMAT "> DEFERRED relocking after wait", p2i(mi->owner()));
 341                 continue;
 342               }
 343             }
 344             if (mi->owner_is_scalar_replaced()) {
 345               Klass* k = java_lang_Class::as_Klass(mi->owner_klass());
 346               st.print_cr("     failed reallocation for klass %s", k->external_name());
 347             } else {
 348               st.print_cr("     object <" INTPTR_FORMAT "> locked", p2i(mi->owner()));
 349             }
 350           }
 351         }
 352         tty->print_raw(st.as_string());
 353       }
 354 #endif // !PRODUCT
 355     }
 356   }
 357 }
 358 
 359 // Deoptimize objects, that is reallocate and relock them, just before they escape through JVMTI.
 360 // The given vframes cover one physical frame.
 361 bool Deoptimization::deoptimize_objects_internal(JavaThread* thread, GrowableArray<compiledVFrame*>* chunk,
 362                                                  bool& realloc_failures) {
 363   frame deoptee = chunk->at(0)->fr();
 364   JavaThread* deoptee_thread = chunk->at(0)->thread();
 365   CompiledMethod* cm = deoptee.cb()->as_compiled_method_or_null();
 366   RegisterMap map(chunk->at(0)->register_map());
 367   bool deoptimized_objects = false;
 368 
 369   bool const jvmci_enabled = JVMCI_ONLY(UseJVMCICompiler) NOT_JVMCI(false);
 370 
 371   // Reallocate the non-escaping objects and restore their fields.
 372   if (jvmci_enabled COMPILER2_PRESENT(|| (DoEscapeAnalysis && EliminateAllocations)
 373                                       || EliminateAutoBox || EnableVectorAggressiveReboxing)) {
 374     realloc_failures = rematerialize_objects(thread, Unpack_none, cm, deoptee, map, chunk, deoptimized_objects);
 375   }
 376 
 377   // MonitorInfo structures used in eliminate_locks are not GC safe.
 378   NoSafepointVerifier no_safepoint;
 379 
 380   // Now relock objects if synchronization on them was eliminated.
 381   if (jvmci_enabled COMPILER2_PRESENT(|| ((DoEscapeAnalysis || EliminateNestedLocks) && EliminateLocks))) {
 382     restore_eliminated_locks(thread, chunk, realloc_failures, deoptee, Unpack_none, deoptimized_objects);
 383   }
 384   return deoptimized_objects;
 385 }
 386 #endif // COMPILER2_OR_JVMCI
 387 
 388 // This is factored, since it is both called from a JRT_LEAF (deoptimization) and a JRT_ENTRY (uncommon_trap)
 389 Deoptimization::UnrollBlock* Deoptimization::fetch_unroll_info_helper(JavaThread* current, int exec_mode) {
 390   // When we get here we are about to unwind the deoptee frame. In order to
 391   // catch not yet safe to use frames, the following stack watermark barrier
 392   // poll will make such frames safe to use.
 393   StackWatermarkSet::before_unwind(current);
 394 
 395   // Note: there is a safepoint safety issue here. No matter whether we enter
 396   // via vanilla deopt or uncommon trap we MUST NOT stop at a safepoint once
 397   // the vframeArray is created.
 398   //
 399 
 400   // Allocate our special deoptimization ResourceMark
 401   DeoptResourceMark* dmark = new DeoptResourceMark(current);
 402   assert(current->deopt_mark() == NULL, "Pending deopt!");
 403   current->set_deopt_mark(dmark);
 404 
 405   frame stub_frame = current->last_frame(); // Makes stack walkable as side effect
 406   RegisterMap map(current, true);
 407   RegisterMap dummy_map(current, false);
 408   // Now get the deoptee with a valid map
 409   frame deoptee = stub_frame.sender(&map);
 410   // Set the deoptee nmethod
 411   assert(current->deopt_compiled_method() == NULL, "Pending deopt!");
 412   CompiledMethod* cm = deoptee.cb()->as_compiled_method_or_null();
 413   current->set_deopt_compiled_method(cm);
 414 
 415   if (VerifyStack) {
 416     current->validate_frame_layout();
 417   }
 418 
 419   // Create a growable array of VFrames where each VFrame represents an inlined
 420   // Java frame.  This storage is allocated with the usual system arena.
 421   assert(deoptee.is_compiled_frame(), "Wrong frame type");
 422   GrowableArray<compiledVFrame*>* chunk = new GrowableArray<compiledVFrame*>(10);
 423   vframe* vf = vframe::new_vframe(&deoptee, &map, current);
 424   while (!vf->is_top()) {
 425     assert(vf->is_compiled_frame(), "Wrong frame type");
 426     chunk->push(compiledVFrame::cast(vf));
 427     vf = vf->sender();
 428   }
 429   assert(vf->is_compiled_frame(), "Wrong frame type");
 430   chunk->push(compiledVFrame::cast(vf));
 431 
 432   bool realloc_failures = false;
 433 
 434 #if COMPILER2_OR_JVMCI
 435   bool const jvmci_enabled = JVMCI_ONLY(EnableJVMCI) NOT_JVMCI(false);
 436 
 437   // Reallocate the non-escaping objects and restore their fields. Then
 438   // relock objects if synchronization on them was eliminated.
 439   if (jvmci_enabled COMPILER2_PRESENT( || (DoEscapeAnalysis && EliminateAllocations)
 440                                        || EliminateAutoBox || EnableVectorAggressiveReboxing )) {
 441     bool unused;
 442     realloc_failures = rematerialize_objects(current, exec_mode, cm, deoptee, map, chunk, unused);
 443   }
 444 #endif // COMPILER2_OR_JVMCI
 445 
 446   // Ensure that no safepoint is taken after pointers have been stored
 447   // in fields of rematerialized objects.  If a safepoint occurs from here on
 448   // out the java state residing in the vframeArray will be missed.
 449   // Locks may be rebaised in a safepoint.
 450   NoSafepointVerifier no_safepoint;
 451 
 452 #if COMPILER2_OR_JVMCI
 453   if ((jvmci_enabled COMPILER2_PRESENT( || ((DoEscapeAnalysis || EliminateNestedLocks) && EliminateLocks) ))
 454       && !EscapeBarrier::objs_are_deoptimized(current, deoptee.id())) {
 455     bool unused;
 456     restore_eliminated_locks(current, chunk, realloc_failures, deoptee, exec_mode, unused);
 457   }
 458 #endif // COMPILER2_OR_JVMCI
 459 
 460   ScopeDesc* trap_scope = chunk->at(0)->scope();
 461   Handle exceptionObject;
 462   if (trap_scope->rethrow_exception()) {
 463     if (PrintDeoptimizationDetails) {
 464       tty->print_cr("Exception to be rethrown in the interpreter for method %s::%s at bci %d", trap_scope->method()->method_holder()->name()->as_C_string(), trap_scope->method()->name()->as_C_string(), trap_scope->bci());
 465     }
 466     GrowableArray<ScopeValue*>* expressions = trap_scope->expressions();
 467     guarantee(expressions != NULL && expressions->length() > 0, "must have exception to throw");
 468     ScopeValue* topOfStack = expressions->top();
 469     exceptionObject = StackValue::create_stack_value(&deoptee, &map, topOfStack)->get_obj();
 470     guarantee(exceptionObject() != NULL, "exception oop can not be null");
 471   }
 472 
 473   vframeArray* array = create_vframeArray(current, deoptee, &map, chunk, realloc_failures);
 474 #if COMPILER2_OR_JVMCI
 475   if (realloc_failures) {
 476     pop_frames_failed_reallocs(current, array);
 477   }
 478 #endif
 479 
 480   assert(current->vframe_array_head() == NULL, "Pending deopt!");
 481   current->set_vframe_array_head(array);
 482 
 483   // Now that the vframeArray has been created if we have any deferred local writes
 484   // added by jvmti then we can free up that structure as the data is now in the
 485   // vframeArray
 486 
 487   JvmtiDeferredUpdates::delete_updates_for_frame(current, array->original().id());
 488 
 489   // Compute the caller frame based on the sender sp of stub_frame and stored frame sizes info.
 490   CodeBlob* cb = stub_frame.cb();
 491   // Verify we have the right vframeArray
 492   assert(cb->frame_size() >= 0, "Unexpected frame size");
 493   intptr_t* unpack_sp = stub_frame.sp() + cb->frame_size();
 494 
 495   // If the deopt call site is a MethodHandle invoke call site we have
 496   // to adjust the unpack_sp.
 497   nmethod* deoptee_nm = deoptee.cb()->as_nmethod_or_null();
 498   if (deoptee_nm != NULL && deoptee_nm->is_method_handle_return(deoptee.pc()))
 499     unpack_sp = deoptee.unextended_sp();
 500 
 501 #ifdef ASSERT
 502   assert(cb->is_deoptimization_stub() ||
 503          cb->is_uncommon_trap_stub() ||
 504          strcmp("Stub<DeoptimizationStub.deoptimizationHandler>", cb->name()) == 0 ||
 505          strcmp("Stub<UncommonTrapStub.uncommonTrapHandler>", cb->name()) == 0,
 506          "unexpected code blob: %s", cb->name());
 507 #endif
 508 
 509   // This is a guarantee instead of an assert because if vframe doesn't match
 510   // we will unpack the wrong deoptimized frame and wind up in strange places
 511   // where it will be very difficult to figure out what went wrong. Better
 512   // to die an early death here than some very obscure death later when the
 513   // trail is cold.
 514   // Note: on ia64 this guarantee can be fooled by frames with no memory stack
 515   // in that it will fail to detect a problem when there is one. This needs
 516   // more work in tiger timeframe.
 517   guarantee(array->unextended_sp() == unpack_sp, "vframe_array_head must contain the vframeArray to unpack");
 518 
 519   int number_of_frames = array->frames();
 520 
 521   // Compute the vframes' sizes.  Note that frame_sizes[] entries are ordered from outermost to innermost
 522   // virtual activation, which is the reverse of the elements in the vframes array.
 523   intptr_t* frame_sizes = NEW_C_HEAP_ARRAY(intptr_t, number_of_frames, mtCompiler);
 524   // +1 because we always have an interpreter return address for the final slot.
 525   address* frame_pcs = NEW_C_HEAP_ARRAY(address, number_of_frames + 1, mtCompiler);
 526   int popframe_extra_args = 0;
 527   // Create an interpreter return address for the stub to use as its return
 528   // address so the skeletal frames are perfectly walkable
 529   frame_pcs[number_of_frames] = Interpreter::deopt_entry(vtos, 0);
 530 
 531   // PopFrame requires that the preserved incoming arguments from the recently-popped topmost
 532   // activation be put back on the expression stack of the caller for reexecution
 533   if (JvmtiExport::can_pop_frame() && current->popframe_forcing_deopt_reexecution()) {
 534     popframe_extra_args = in_words(current->popframe_preserved_args_size_in_words());
 535   }
 536 
 537   // Find the current pc for sender of the deoptee. Since the sender may have been deoptimized
 538   // itself since the deoptee vframeArray was created we must get a fresh value of the pc rather
 539   // than simply use array->sender.pc(). This requires us to walk the current set of frames
 540   //
 541   frame deopt_sender = stub_frame.sender(&dummy_map); // First is the deoptee frame
 542   deopt_sender = deopt_sender.sender(&dummy_map);     // Now deoptee caller
 543 
 544   // It's possible that the number of parameters at the call site is
 545   // different than number of arguments in the callee when method
 546   // handles are used.  If the caller is interpreted get the real
 547   // value so that the proper amount of space can be added to it's
 548   // frame.
 549   bool caller_was_method_handle = false;
 550   if (deopt_sender.is_interpreted_frame()) {
 551     methodHandle method(current, deopt_sender.interpreter_frame_method());
 552     Bytecode_invoke cur = Bytecode_invoke_check(method, deopt_sender.interpreter_frame_bci());
 553     if (cur.is_invokedynamic() || cur.is_invokehandle()) {
 554       // Method handle invokes may involve fairly arbitrary chains of
 555       // calls so it's impossible to know how much actual space the
 556       // caller has for locals.
 557       caller_was_method_handle = true;
 558     }
 559   }
 560 
 561   //
 562   // frame_sizes/frame_pcs[0] oldest frame (int or c2i)
 563   // frame_sizes/frame_pcs[1] next oldest frame (int)
 564   // frame_sizes/frame_pcs[n] youngest frame (int)
 565   //
 566   // Now a pc in frame_pcs is actually the return address to the frame's caller (a frame
 567   // owns the space for the return address to it's caller).  Confusing ain't it.
 568   //
 569   // The vframe array can address vframes with indices running from
 570   // 0.._frames-1. Index  0 is the youngest frame and _frame - 1 is the oldest (root) frame.
 571   // When we create the skeletal frames we need the oldest frame to be in the zero slot
 572   // in the frame_sizes/frame_pcs so the assembly code can do a trivial walk.
 573   // so things look a little strange in this loop.
 574   //
 575   int callee_parameters = 0;
 576   int callee_locals = 0;
 577   for (int index = 0; index < array->frames(); index++ ) {
 578     // frame[number_of_frames - 1 ] = on_stack_size(youngest)
 579     // frame[number_of_frames - 2 ] = on_stack_size(sender(youngest))
 580     // frame[number_of_frames - 3 ] = on_stack_size(sender(sender(youngest)))
 581     frame_sizes[number_of_frames - 1 - index] = BytesPerWord * array->element(index)->on_stack_size(callee_parameters,
 582                                                                                                     callee_locals,
 583                                                                                                     index == 0,
 584                                                                                                     popframe_extra_args);
 585     // This pc doesn't have to be perfect just good enough to identify the frame
 586     // as interpreted so the skeleton frame will be walkable
 587     // The correct pc will be set when the skeleton frame is completely filled out
 588     // The final pc we store in the loop is wrong and will be overwritten below
 589     frame_pcs[number_of_frames - 1 - index ] = Interpreter::deopt_entry(vtos, 0) - frame::pc_return_offset;
 590 
 591     callee_parameters = array->element(index)->method()->size_of_parameters();
 592     callee_locals = array->element(index)->method()->max_locals();
 593     popframe_extra_args = 0;
 594   }
 595 
 596   // Compute whether the root vframe returns a float or double value.
 597   BasicType return_type;
 598   {
 599     methodHandle method(current, array->element(0)->method());
 600     Bytecode_invoke invoke = Bytecode_invoke_check(method, array->element(0)->bci());
 601     return_type = invoke.is_valid() ? invoke.result_type() : T_ILLEGAL;
 602   }
 603 
 604   // Compute information for handling adapters and adjusting the frame size of the caller.
 605   int caller_adjustment = 0;
 606 
 607   // Compute the amount the oldest interpreter frame will have to adjust
 608   // its caller's stack by. If the caller is a compiled frame then
 609   // we pretend that the callee has no parameters so that the
 610   // extension counts for the full amount of locals and not just
 611   // locals-parms. This is because without a c2i adapter the parm
 612   // area as created by the compiled frame will not be usable by
 613   // the interpreter. (Depending on the calling convention there
 614   // may not even be enough space).
 615 
 616   // QQQ I'd rather see this pushed down into last_frame_adjust
 617   // and have it take the sender (aka caller).
 618 
 619   if (deopt_sender.is_compiled_frame() || caller_was_method_handle) {
 620     caller_adjustment = last_frame_adjust(0, callee_locals);
 621   } else if (callee_locals > callee_parameters) {
 622     // The caller frame may need extending to accommodate
 623     // non-parameter locals of the first unpacked interpreted frame.
 624     // Compute that adjustment.
 625     caller_adjustment = last_frame_adjust(callee_parameters, callee_locals);
 626   }
 627 
 628   // If the sender is deoptimized we must retrieve the address of the handler
 629   // since the frame will "magically" show the original pc before the deopt
 630   // and we'd undo the deopt.
 631 
 632   frame_pcs[0] = deopt_sender.raw_pc();
 633 
 634   assert(CodeCache::find_blob_unsafe(frame_pcs[0]) != NULL, "bad pc");
 635 
 636 #if INCLUDE_JVMCI
 637   if (exceptionObject() != NULL) {
 638     current->set_exception_oop(exceptionObject());
 639     exec_mode = Unpack_exception;
 640   }
 641 #endif
 642 
 643   if (current->frames_to_pop_failed_realloc() > 0 && exec_mode != Unpack_uncommon_trap) {
 644     assert(current->has_pending_exception(), "should have thrown OOME");
 645     current->set_exception_oop(current->pending_exception());
 646     current->clear_pending_exception();
 647     exec_mode = Unpack_exception;
 648   }
 649 
 650 #if INCLUDE_JVMCI
 651   if (current->frames_to_pop_failed_realloc() > 0) {
 652     current->set_pending_monitorenter(false);
 653   }
 654 #endif
 655 
 656   UnrollBlock* info = new UnrollBlock(array->frame_size() * BytesPerWord,
 657                                       caller_adjustment * BytesPerWord,
 658                                       caller_was_method_handle ? 0 : callee_parameters,
 659                                       number_of_frames,
 660                                       frame_sizes,
 661                                       frame_pcs,
 662                                       return_type,
 663                                       exec_mode);
 664   // On some platforms, we need a way to pass some platform dependent
 665   // information to the unpacking code so the skeletal frames come out
 666   // correct (initial fp value, unextended sp, ...)
 667   info->set_initial_info((intptr_t) array->sender().initial_deoptimization_info());
 668 
 669   if (array->frames() > 1) {
 670     if (VerifyStack && TraceDeoptimization) {
 671       tty->print_cr("Deoptimizing method containing inlining");
 672     }
 673   }
 674 
 675   array->set_unroll_block(info);
 676   return info;
 677 }
 678 
 679 // Called to cleanup deoptimization data structures in normal case
 680 // after unpacking to stack and when stack overflow error occurs
 681 void Deoptimization::cleanup_deopt_info(JavaThread *thread,
 682                                         vframeArray *array) {
 683 
 684   // Get array if coming from exception
 685   if (array == NULL) {
 686     array = thread->vframe_array_head();
 687   }
 688   thread->set_vframe_array_head(NULL);
 689 
 690   // Free the previous UnrollBlock
 691   vframeArray* old_array = thread->vframe_array_last();
 692   thread->set_vframe_array_last(array);
 693 
 694   if (old_array != NULL) {
 695     UnrollBlock* old_info = old_array->unroll_block();
 696     old_array->set_unroll_block(NULL);
 697     delete old_info;
 698     delete old_array;
 699   }
 700 
 701   // Deallocate any resource creating in this routine and any ResourceObjs allocated
 702   // inside the vframeArray (StackValueCollections)
 703 
 704   delete thread->deopt_mark();
 705   thread->set_deopt_mark(NULL);
 706   thread->set_deopt_compiled_method(NULL);
 707 
 708 
 709   if (JvmtiExport::can_pop_frame()) {
 710     // Regardless of whether we entered this routine with the pending
 711     // popframe condition bit set, we should always clear it now
 712     thread->clear_popframe_condition();
 713   }
 714 
 715   // unpack_frames() is called at the end of the deoptimization handler
 716   // and (in C2) at the end of the uncommon trap handler. Note this fact
 717   // so that an asynchronous stack walker can work again. This counter is
 718   // incremented at the beginning of fetch_unroll_info() and (in C2) at
 719   // the beginning of uncommon_trap().
 720   thread->dec_in_deopt_handler();
 721 }
 722 
 723 // Moved from cpu directories because none of the cpus has callee save values.
 724 // If a cpu implements callee save values, move this to deoptimization_<cpu>.cpp.
 725 void Deoptimization::unwind_callee_save_values(frame* f, vframeArray* vframe_array) {
 726 
 727   // This code is sort of the equivalent of C2IAdapter::setup_stack_frame back in
 728   // the days we had adapter frames. When we deoptimize a situation where a
 729   // compiled caller calls a compiled caller will have registers it expects
 730   // to survive the call to the callee. If we deoptimize the callee the only
 731   // way we can restore these registers is to have the oldest interpreter
 732   // frame that we create restore these values. That is what this routine
 733   // will accomplish.
 734 
 735   // At the moment we have modified c2 to not have any callee save registers
 736   // so this problem does not exist and this routine is just a place holder.
 737 
 738   assert(f->is_interpreted_frame(), "must be interpreted");
 739 }
 740 
 741 // Return BasicType of value being returned
 742 JRT_LEAF(BasicType, Deoptimization::unpack_frames(JavaThread* thread, int exec_mode))
 743 
 744   // We are already active in the special DeoptResourceMark any ResourceObj's we
 745   // allocate will be freed at the end of the routine.
 746 
 747   // JRT_LEAF methods don't normally allocate handles and there is a
 748   // NoHandleMark to enforce that. It is actually safe to use Handles
 749   // in a JRT_LEAF method, and sometimes desirable, but to do so we
 750   // must use ResetNoHandleMark to bypass the NoHandleMark, and
 751   // then use a HandleMark to ensure any Handles we do create are
 752   // cleaned up in this scope.
 753   ResetNoHandleMark rnhm;
 754   HandleMark hm(thread);
 755 
 756   frame stub_frame = thread->last_frame();
 757 
 758   // Since the frame to unpack is the top frame of this thread, the vframe_array_head
 759   // must point to the vframeArray for the unpack frame.
 760   vframeArray* array = thread->vframe_array_head();
 761 
 762 #ifndef PRODUCT
 763   if (TraceDeoptimization) {
 764     tty->print_cr("DEOPT UNPACKING thread " INTPTR_FORMAT " vframeArray " INTPTR_FORMAT " mode %d",
 765                   p2i(thread), p2i(array), exec_mode);
 766   }
 767 #endif
 768   Events::log_deopt_message(thread, "DEOPT UNPACKING pc=" INTPTR_FORMAT " sp=" INTPTR_FORMAT " mode %d",
 769               p2i(stub_frame.pc()), p2i(stub_frame.sp()), exec_mode);
 770 
 771   UnrollBlock* info = array->unroll_block();
 772 
 773   // We set the last_Java frame. But the stack isn't really parsable here. So we
 774   // clear it to make sure JFR understands not to try and walk stacks from events
 775   // in here.
 776   intptr_t* sp = thread->frame_anchor()->last_Java_sp();
 777   thread->frame_anchor()->set_last_Java_sp(NULL);
 778 
 779   // Unpack the interpreter frames and any adapter frame (c2 only) we might create.
 780   array->unpack_to_stack(stub_frame, exec_mode, info->caller_actual_parameters());
 781 
 782   thread->frame_anchor()->set_last_Java_sp(sp);
 783 
 784   BasicType bt = info->return_type();
 785 
 786   // If we have an exception pending, claim that the return type is an oop
 787   // so the deopt_blob does not overwrite the exception_oop.
 788 
 789   if (exec_mode == Unpack_exception)
 790     bt = T_OBJECT;
 791 
 792   // Cleanup thread deopt data
 793   cleanup_deopt_info(thread, array);
 794 
 795 #ifndef PRODUCT
 796   if (VerifyStack) {
 797     ResourceMark res_mark;
 798     // Clear pending exception to not break verification code (restored afterwards)
 799     PreserveExceptionMark pm(thread);
 800 
 801     thread->validate_frame_layout();
 802 
 803     // Verify that the just-unpacked frames match the interpreter's
 804     // notions of expression stack and locals
 805     vframeArray* cur_array = thread->vframe_array_last();
 806     RegisterMap rm(thread, false);
 807     rm.set_include_argument_oops(false);
 808     bool is_top_frame = true;
 809     int callee_size_of_parameters = 0;
 810     int callee_max_locals = 0;
 811     for (int i = 0; i < cur_array->frames(); i++) {
 812       vframeArrayElement* el = cur_array->element(i);
 813       frame* iframe = el->iframe();
 814       guarantee(iframe->is_interpreted_frame(), "Wrong frame type");
 815 
 816       // Get the oop map for this bci
 817       InterpreterOopMap mask;
 818       int cur_invoke_parameter_size = 0;
 819       bool try_next_mask = false;
 820       int next_mask_expression_stack_size = -1;
 821       int top_frame_expression_stack_adjustment = 0;
 822       methodHandle mh(thread, iframe->interpreter_frame_method());
 823       OopMapCache::compute_one_oop_map(mh, iframe->interpreter_frame_bci(), &mask);
 824       BytecodeStream str(mh, iframe->interpreter_frame_bci());
 825       int max_bci = mh->code_size();
 826       // Get to the next bytecode if possible
 827       assert(str.bci() < max_bci, "bci in interpreter frame out of bounds");
 828       // Check to see if we can grab the number of outgoing arguments
 829       // at an uncommon trap for an invoke (where the compiler
 830       // generates debug info before the invoke has executed)
 831       Bytecodes::Code cur_code = str.next();
 832       if (Bytecodes::is_invoke(cur_code)) {
 833         Bytecode_invoke invoke(mh, iframe->interpreter_frame_bci());
 834         cur_invoke_parameter_size = invoke.size_of_parameters();
 835         if (i != 0 && !invoke.is_invokedynamic() && MethodHandles::has_member_arg(invoke.klass(), invoke.name())) {
 836           callee_size_of_parameters++;
 837         }
 838       }
 839       if (str.bci() < max_bci) {
 840         Bytecodes::Code next_code = str.next();
 841         if (next_code >= 0) {
 842           // The interpreter oop map generator reports results before
 843           // the current bytecode has executed except in the case of
 844           // calls. It seems to be hard to tell whether the compiler
 845           // has emitted debug information matching the "state before"
 846           // a given bytecode or the state after, so we try both
 847           if (!Bytecodes::is_invoke(cur_code) && cur_code != Bytecodes::_athrow) {
 848             // Get expression stack size for the next bytecode
 849             InterpreterOopMap next_mask;
 850             OopMapCache::compute_one_oop_map(mh, str.bci(), &next_mask);
 851             next_mask_expression_stack_size = next_mask.expression_stack_size();
 852             if (Bytecodes::is_invoke(next_code)) {
 853               Bytecode_invoke invoke(mh, str.bci());
 854               next_mask_expression_stack_size += invoke.size_of_parameters();
 855             }
 856             // Need to subtract off the size of the result type of
 857             // the bytecode because this is not described in the
 858             // debug info but returned to the interpreter in the TOS
 859             // caching register
 860             BasicType bytecode_result_type = Bytecodes::result_type(cur_code);
 861             if (bytecode_result_type != T_ILLEGAL) {
 862               top_frame_expression_stack_adjustment = type2size[bytecode_result_type];
 863             }
 864             assert(top_frame_expression_stack_adjustment >= 0, "stack adjustment must be positive");
 865             try_next_mask = true;
 866           }
 867         }
 868       }
 869 
 870       // Verify stack depth and oops in frame
 871       // This assertion may be dependent on the platform we're running on and may need modification (tested on x86 and sparc)
 872       if (!(
 873             /* SPARC */
 874             (iframe->interpreter_frame_expression_stack_size() == mask.expression_stack_size() + callee_size_of_parameters) ||
 875             /* x86 */
 876             (iframe->interpreter_frame_expression_stack_size() == mask.expression_stack_size() + callee_max_locals) ||
 877             (try_next_mask &&
 878              (iframe->interpreter_frame_expression_stack_size() == (next_mask_expression_stack_size -
 879                                                                     top_frame_expression_stack_adjustment))) ||
 880             (is_top_frame && (exec_mode == Unpack_exception) && iframe->interpreter_frame_expression_stack_size() == 0) ||
 881             (is_top_frame && (exec_mode == Unpack_uncommon_trap || exec_mode == Unpack_reexecute || el->should_reexecute()) &&
 882              (iframe->interpreter_frame_expression_stack_size() == mask.expression_stack_size() + cur_invoke_parameter_size))
 883             )) {
 884         {
 885           // Print out some information that will help us debug the problem
 886           tty->print_cr("Wrong number of expression stack elements during deoptimization");
 887           tty->print_cr("  Error occurred while verifying frame %d (0..%d, 0 is topmost)", i, cur_array->frames() - 1);
 888           tty->print_cr("  Fabricated interpreter frame had %d expression stack elements",
 889                         iframe->interpreter_frame_expression_stack_size());
 890           tty->print_cr("  Interpreter oop map had %d expression stack elements", mask.expression_stack_size());
 891           tty->print_cr("  try_next_mask = %d", try_next_mask);
 892           tty->print_cr("  next_mask_expression_stack_size = %d", next_mask_expression_stack_size);
 893           tty->print_cr("  callee_size_of_parameters = %d", callee_size_of_parameters);
 894           tty->print_cr("  callee_max_locals = %d", callee_max_locals);
 895           tty->print_cr("  top_frame_expression_stack_adjustment = %d", top_frame_expression_stack_adjustment);
 896           tty->print_cr("  exec_mode = %d", exec_mode);
 897           tty->print_cr("  cur_invoke_parameter_size = %d", cur_invoke_parameter_size);
 898           tty->print_cr("  Thread = " INTPTR_FORMAT ", thread ID = %d", p2i(thread), thread->osthread()->thread_id());
 899           tty->print_cr("  Interpreted frames:");
 900           for (int k = 0; k < cur_array->frames(); k++) {
 901             vframeArrayElement* el = cur_array->element(k);
 902             tty->print_cr("    %s (bci %d)", el->method()->name_and_sig_as_C_string(), el->bci());
 903           }
 904           cur_array->print_on_2(tty);
 905         }
 906         guarantee(false, "wrong number of expression stack elements during deopt");
 907       }
 908       VerifyOopClosure verify;
 909       iframe->oops_interpreted_do(&verify, &rm, false);
 910       callee_size_of_parameters = mh->size_of_parameters();
 911       callee_max_locals = mh->max_locals();
 912       is_top_frame = false;
 913     }
 914   }
 915 #endif /* !PRODUCT */
 916 
 917   return bt;
 918 JRT_END
 919 
 920 class DeoptimizeMarkedClosure : public HandshakeClosure {
 921  public:
 922   DeoptimizeMarkedClosure() : HandshakeClosure("Deoptimize") {}
 923   void do_thread(Thread* thread) {
 924     JavaThread* jt = JavaThread::cast(thread);
 925     jt->deoptimize_marked_methods();
 926   }
 927 };
 928 
 929 void Deoptimization::deoptimize_all_marked(nmethod* nmethod_only) {
 930   ResourceMark rm;
 931   DeoptimizationMarker dm;
 932 
 933   // Make the dependent methods not entrant
 934   if (nmethod_only != NULL) {
 935     nmethod_only->mark_for_deoptimization();
 936     nmethod_only->make_not_entrant();
 937   } else {
 938     MutexLocker mu(SafepointSynchronize::is_at_safepoint() ? NULL : CodeCache_lock, Mutex::_no_safepoint_check_flag);
 939     CodeCache::make_marked_nmethods_not_entrant();
 940   }
 941 
 942   DeoptimizeMarkedClosure deopt;
 943   if (SafepointSynchronize::is_at_safepoint()) {
 944     Threads::java_threads_do(&deopt);
 945   } else {
 946     Handshake::execute(&deopt);
 947   }
 948 }
 949 
 950 Deoptimization::DeoptAction Deoptimization::_unloaded_action
 951   = Deoptimization::Action_reinterpret;
 952 
 953 #if COMPILER2_OR_JVMCI
 954 template<typename CacheType>
 955 class BoxCacheBase : public CHeapObj<mtCompiler> {
 956 protected:
 957   static InstanceKlass* find_cache_klass(Symbol* klass_name) {
 958     ResourceMark rm;
 959     char* klass_name_str = klass_name->as_C_string();
 960     InstanceKlass* ik = SystemDictionary::find_instance_klass(klass_name, Handle(), Handle());
 961     guarantee(ik != NULL, "%s must be loaded", klass_name_str);
 962     guarantee(ik->is_initialized(), "%s must be initialized", klass_name_str);
 963     CacheType::compute_offsets(ik);
 964     return ik;
 965   }
 966 };
 967 
 968 template<typename PrimitiveType, typename CacheType, typename BoxType> class BoxCache  : public BoxCacheBase<CacheType> {
 969   PrimitiveType _low;
 970   PrimitiveType _high;
 971   jobject _cache;
 972 protected:
 973   static BoxCache<PrimitiveType, CacheType, BoxType> *_singleton;
 974   BoxCache(Thread* thread) {
 975     InstanceKlass* ik = BoxCacheBase<CacheType>::find_cache_klass(CacheType::symbol());
 976     objArrayOop cache = CacheType::cache(ik);
 977     assert(cache->length() > 0, "Empty cache");
 978     _low = BoxType::value(cache->obj_at(0));
 979     _high = _low + cache->length() - 1;
 980     _cache = JNIHandles::make_global(Handle(thread, cache));
 981   }
 982   ~BoxCache() {
 983     JNIHandles::destroy_global(_cache);
 984   }
 985 public:
 986   static BoxCache<PrimitiveType, CacheType, BoxType>* singleton(Thread* thread) {
 987     if (_singleton == NULL) {
 988       BoxCache<PrimitiveType, CacheType, BoxType>* s = new BoxCache<PrimitiveType, CacheType, BoxType>(thread);
 989       if (!Atomic::replace_if_null(&_singleton, s)) {
 990         delete s;
 991       }
 992     }
 993     return _singleton;
 994   }
 995   oop lookup(PrimitiveType value) {
 996     if (_low <= value && value <= _high) {
 997       int offset = value - _low;
 998       return objArrayOop(JNIHandles::resolve_non_null(_cache))->obj_at(offset);
 999     }
1000     return NULL;
1001   }
1002   oop lookup_raw(intptr_t raw_value) {
1003     // Have to cast to avoid little/big-endian problems.
1004     if (sizeof(PrimitiveType) > sizeof(jint)) {
1005       jlong value = (jlong)raw_value;
1006       return lookup(value);
1007     }
1008     PrimitiveType value = (PrimitiveType)*((jint*)&raw_value);
1009     return lookup(value);
1010   }
1011 };
1012 
1013 typedef BoxCache<jint, java_lang_Integer_IntegerCache, java_lang_Integer> IntegerBoxCache;
1014 typedef BoxCache<jlong, java_lang_Long_LongCache, java_lang_Long> LongBoxCache;
1015 typedef BoxCache<jchar, java_lang_Character_CharacterCache, java_lang_Character> CharacterBoxCache;
1016 typedef BoxCache<jshort, java_lang_Short_ShortCache, java_lang_Short> ShortBoxCache;
1017 typedef BoxCache<jbyte, java_lang_Byte_ByteCache, java_lang_Byte> ByteBoxCache;
1018 
1019 template<> BoxCache<jint, java_lang_Integer_IntegerCache, java_lang_Integer>* BoxCache<jint, java_lang_Integer_IntegerCache, java_lang_Integer>::_singleton = NULL;
1020 template<> BoxCache<jlong, java_lang_Long_LongCache, java_lang_Long>* BoxCache<jlong, java_lang_Long_LongCache, java_lang_Long>::_singleton = NULL;
1021 template<> BoxCache<jchar, java_lang_Character_CharacterCache, java_lang_Character>* BoxCache<jchar, java_lang_Character_CharacterCache, java_lang_Character>::_singleton = NULL;
1022 template<> BoxCache<jshort, java_lang_Short_ShortCache, java_lang_Short>* BoxCache<jshort, java_lang_Short_ShortCache, java_lang_Short>::_singleton = NULL;
1023 template<> BoxCache<jbyte, java_lang_Byte_ByteCache, java_lang_Byte>* BoxCache<jbyte, java_lang_Byte_ByteCache, java_lang_Byte>::_singleton = NULL;
1024 
1025 class BooleanBoxCache : public BoxCacheBase<java_lang_Boolean> {
1026   jobject _true_cache;
1027   jobject _false_cache;
1028 protected:
1029   static BooleanBoxCache *_singleton;
1030   BooleanBoxCache(Thread *thread) {
1031     InstanceKlass* ik = find_cache_klass(java_lang_Boolean::symbol());
1032     _true_cache = JNIHandles::make_global(Handle(thread, java_lang_Boolean::get_TRUE(ik)));
1033     _false_cache = JNIHandles::make_global(Handle(thread, java_lang_Boolean::get_FALSE(ik)));
1034   }
1035   ~BooleanBoxCache() {
1036     JNIHandles::destroy_global(_true_cache);
1037     JNIHandles::destroy_global(_false_cache);
1038   }
1039 public:
1040   static BooleanBoxCache* singleton(Thread* thread) {
1041     if (_singleton == NULL) {
1042       BooleanBoxCache* s = new BooleanBoxCache(thread);
1043       if (!Atomic::replace_if_null(&_singleton, s)) {
1044         delete s;
1045       }
1046     }
1047     return _singleton;
1048   }
1049   oop lookup_raw(intptr_t raw_value) {
1050     // Have to cast to avoid little/big-endian problems.
1051     jboolean value = (jboolean)*((jint*)&raw_value);
1052     return lookup(value);
1053   }
1054   oop lookup(jboolean value) {
1055     if (value != 0) {
1056       return JNIHandles::resolve_non_null(_true_cache);
1057     }
1058     return JNIHandles::resolve_non_null(_false_cache);
1059   }
1060 };
1061 
1062 BooleanBoxCache* BooleanBoxCache::_singleton = NULL;
1063 
1064 oop Deoptimization::get_cached_box(AutoBoxObjectValue* bv, frame* fr, RegisterMap* reg_map, TRAPS) {
1065    Klass* k = java_lang_Class::as_Klass(bv->klass()->as_ConstantOopReadValue()->value()());
1066    BasicType box_type = vmClasses::box_klass_type(k);
1067    if (box_type != T_OBJECT) {
1068      StackValue* value = StackValue::create_stack_value(fr, reg_map, bv->field_at(box_type == T_LONG ? 1 : 0));
1069      switch(box_type) {
1070        case T_INT:     return IntegerBoxCache::singleton(THREAD)->lookup_raw(value->get_int());
1071        case T_CHAR:    return CharacterBoxCache::singleton(THREAD)->lookup_raw(value->get_int());
1072        case T_SHORT:   return ShortBoxCache::singleton(THREAD)->lookup_raw(value->get_int());
1073        case T_BYTE:    return ByteBoxCache::singleton(THREAD)->lookup_raw(value->get_int());
1074        case T_BOOLEAN: return BooleanBoxCache::singleton(THREAD)->lookup_raw(value->get_int());
1075        case T_LONG:    return LongBoxCache::singleton(THREAD)->lookup_raw(value->get_int());
1076        default:;
1077      }
1078    }
1079    return NULL;
1080 }
1081 
1082 bool Deoptimization::realloc_objects(JavaThread* thread, frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects, TRAPS) {
1083   Handle pending_exception(THREAD, thread->pending_exception());
1084   const char* exception_file = thread->exception_file();
1085   int exception_line = thread->exception_line();
1086   thread->clear_pending_exception();
1087 
1088   bool failures = false;
1089 
1090   for (int i = 0; i < objects->length(); i++) {
1091     assert(objects->at(i)->is_object(), "invalid debug information");
1092     ObjectValue* sv = (ObjectValue*) objects->at(i);

1093     Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());

1094 
1095     // Check if the object may be null and has an additional is_init input that needs
1096     // to be checked before using the field values. Skip re-allocation if it is null.
1097     if (sv->maybe_null()) {
1098       assert(k->is_inline_klass(), "must be an inline klass");
1099       intptr_t init_value = StackValue::create_stack_value(fr, reg_map, sv->is_init())->get_int();
1100       jint is_init = (jint)*((jint*)&init_value);
1101       if (is_init == 0) {
1102         continue;
1103       }
1104     }
1105 
1106     oop obj = NULL;
1107     if (k->is_instance_klass()) {
1108       if (sv->is_auto_box()) {
1109         AutoBoxObjectValue* abv = (AutoBoxObjectValue*) sv;
1110         obj = get_cached_box(abv, fr, reg_map, THREAD);
1111         if (obj != NULL) {
1112           // Set the flag to indicate the box came from a cache, so that we can skip the field reassignment for it.
1113           abv->set_cached(true);
1114         }
1115       }
1116 
1117       InstanceKlass* ik = InstanceKlass::cast(k);
1118       if (obj == NULL) {
1119 #ifdef COMPILER2
1120         if (EnableVectorSupport && VectorSupport::is_vector(ik)) {
1121           obj = VectorSupport::allocate_vector(ik, fr, reg_map, sv, THREAD);
1122         } else {
1123           obj = ik->allocate_instance(THREAD);
1124         }
1125 #else
1126         obj = ik->allocate_instance(THREAD);
1127 #endif // COMPILER2
1128       }
1129     } else if (k->is_flatArray_klass()) {
1130       FlatArrayKlass* ak = FlatArrayKlass::cast(k);
1131       // Inline type array must be zeroed because not all memory is reassigned
1132       obj = ak->allocate(sv->field_size(), THREAD);
1133     } else if (k->is_typeArray_klass()) {
1134       TypeArrayKlass* ak = TypeArrayKlass::cast(k);
1135       assert(sv->field_size() % type2size[ak->element_type()] == 0, "non-integral array length");
1136       int len = sv->field_size() / type2size[ak->element_type()];
1137       obj = ak->allocate(len, THREAD);
1138     } else if (k->is_objArray_klass()) {
1139       ObjArrayKlass* ak = ObjArrayKlass::cast(k);
1140       obj = ak->allocate(sv->field_size(), THREAD);
1141     }
1142 
1143     if (obj == NULL) {
1144       failures = true;
1145     }
1146 
1147     assert(sv->value().is_null(), "redundant reallocation");
1148     assert(obj != NULL || HAS_PENDING_EXCEPTION, "allocation should succeed or we should get an exception");
1149     CLEAR_PENDING_EXCEPTION;
1150     sv->set_value(obj);
1151   }
1152 
1153   if (failures) {
1154     THROW_OOP_(Universe::out_of_memory_error_realloc_objects(), failures);
1155   } else if (pending_exception.not_null()) {
1156     thread->set_pending_exception(pending_exception(), exception_file, exception_line);
1157   }
1158 
1159   return failures;
1160 }
1161 
1162 // We're deoptimizing at the return of a call, inline type fields are
1163 // in registers. When we go back to the interpreter, it will expect a
1164 // reference to an inline type instance. Allocate and initialize it from
1165 // the register values here.
1166 bool Deoptimization::realloc_inline_type_result(InlineKlass* vk, const RegisterMap& map, GrowableArray<Handle>& return_oops, TRAPS) {
1167   oop new_vt = vk->realloc_result(map, return_oops, THREAD);
1168   if (new_vt == NULL) {
1169     CLEAR_PENDING_EXCEPTION;
1170     THROW_OOP_(Universe::out_of_memory_error_realloc_objects(), true);
1171   }
1172   return_oops.clear();
1173   return_oops.push(Handle(THREAD, new_vt));
1174   return false;
1175 }
1176 
1177 #if INCLUDE_JVMCI
1178 /**
1179  * For primitive types whose kind gets "erased" at runtime (shorts become stack ints),
1180  * we need to somehow be able to recover the actual kind to be able to write the correct
1181  * amount of bytes.
1182  * For that purpose, this method assumes that, for an entry spanning n bytes at index i,
1183  * the entries at index n + 1 to n + i are 'markers'.
1184  * For example, if we were writing a short at index 4 of a byte array of size 8, the
1185  * expected form of the array would be:
1186  *
1187  * {b0, b1, b2, b3, INT, marker, b6, b7}
1188  *
1189  * Thus, in order to get back the size of the entry, we simply need to count the number
1190  * of marked entries
1191  *
1192  * @param virtualArray the virtualized byte array
1193  * @param i index of the virtual entry we are recovering
1194  * @return The number of bytes the entry spans
1195  */
1196 static int count_number_of_bytes_for_entry(ObjectValue *virtualArray, int i) {
1197   int index = i;
1198   while (++index < virtualArray->field_size() &&
1199            virtualArray->field_at(index)->is_marker()) {}
1200   return index - i;
1201 }
1202 
1203 /**
1204  * If there was a guarantee for byte array to always start aligned to a long, we could
1205  * do a simple check on the parity of the index. Unfortunately, that is not always the
1206  * case. Thus, we check alignment of the actual address we are writing to.
1207  * In the unlikely case index 0 is 5-aligned for example, it would then be possible to
1208  * write a long to index 3.
1209  */
1210 static jbyte* check_alignment_get_addr(typeArrayOop obj, int index, int expected_alignment) {
1211     jbyte* res = obj->byte_at_addr(index);
1212     assert((((intptr_t) res) % expected_alignment) == 0, "Non-aligned write");
1213     return res;
1214 }
1215 
1216 static void byte_array_put(typeArrayOop obj, intptr_t val, int index, int byte_count) {
1217   switch (byte_count) {
1218     case 1:
1219       obj->byte_at_put(index, (jbyte) *((jint *) &val));
1220       break;
1221     case 2:
1222       *((jshort *) check_alignment_get_addr(obj, index, 2)) = (jshort) *((jint *) &val);
1223       break;
1224     case 4:
1225       *((jint *) check_alignment_get_addr(obj, index, 4)) = (jint) *((jint *) &val);
1226       break;
1227     case 8:
1228       *((jlong *) check_alignment_get_addr(obj, index, 8)) = (jlong) *((jlong *) &val);
1229       break;
1230     default:
1231       ShouldNotReachHere();
1232   }
1233 }
1234 #endif // INCLUDE_JVMCI
1235 
1236 
1237 // restore elements of an eliminated type array
1238 void Deoptimization::reassign_type_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, typeArrayOop obj, BasicType type) {
1239   int index = 0;
1240   intptr_t val;
1241 
1242   for (int i = 0; i < sv->field_size(); i++) {
1243     StackValue* value = StackValue::create_stack_value(fr, reg_map, sv->field_at(i));
1244     switch(type) {
1245     case T_LONG: case T_DOUBLE: {
1246       assert(value->type() == T_INT, "Agreement.");
1247       StackValue* low =
1248         StackValue::create_stack_value(fr, reg_map, sv->field_at(++i));
1249 #ifdef _LP64
1250       jlong res = (jlong)low->get_int();
1251 #else
1252       jlong res = jlong_from((jint)value->get_int(), (jint)low->get_int());
1253 #endif
1254       obj->long_at_put(index, res);
1255       break;
1256     }
1257 
1258     // Have to cast to INT (32 bits) pointer to avoid little/big-endian problem.
1259     case T_INT: case T_FLOAT: { // 4 bytes.
1260       assert(value->type() == T_INT, "Agreement.");
1261       bool big_value = false;
1262       if (i + 1 < sv->field_size() && type == T_INT) {
1263         if (sv->field_at(i)->is_location()) {
1264           Location::Type type = ((LocationValue*) sv->field_at(i))->location().type();
1265           if (type == Location::dbl || type == Location::lng) {
1266             big_value = true;
1267           }
1268         } else if (sv->field_at(i)->is_constant_int()) {
1269           ScopeValue* next_scope_field = sv->field_at(i + 1);
1270           if (next_scope_field->is_constant_long() || next_scope_field->is_constant_double()) {
1271             big_value = true;
1272           }
1273         }
1274       }
1275 
1276       if (big_value) {
1277         StackValue* low = StackValue::create_stack_value(fr, reg_map, sv->field_at(++i));
1278   #ifdef _LP64
1279         jlong res = (jlong)low->get_int();
1280   #else
1281         jlong res = jlong_from((jint)value->get_int(), (jint)low->get_int());
1282   #endif
1283         obj->int_at_put(index, (jint)*((jint*)&res));
1284         obj->int_at_put(++index, (jint)*(((jint*)&res) + 1));
1285       } else {
1286         val = value->get_int();
1287         obj->int_at_put(index, (jint)*((jint*)&val));
1288       }
1289       break;
1290     }
1291 
1292     case T_SHORT:
1293       assert(value->type() == T_INT, "Agreement.");
1294       val = value->get_int();
1295       obj->short_at_put(index, (jshort)*((jint*)&val));
1296       break;
1297 
1298     case T_CHAR:
1299       assert(value->type() == T_INT, "Agreement.");
1300       val = value->get_int();
1301       obj->char_at_put(index, (jchar)*((jint*)&val));
1302       break;
1303 
1304     case T_BYTE: {
1305       assert(value->type() == T_INT, "Agreement.");
1306       // The value we get is erased as a regular int. We will need to find its actual byte count 'by hand'.
1307       val = value->get_int();
1308 #if INCLUDE_JVMCI
1309       int byte_count = count_number_of_bytes_for_entry(sv, i);
1310       byte_array_put(obj, val, index, byte_count);
1311       // According to byte_count contract, the values from i + 1 to i + byte_count are illegal values. Skip.
1312       i += byte_count - 1; // Balance the loop counter.
1313       index += byte_count;
1314       // index has been updated so continue at top of loop
1315       continue;
1316 #else
1317       obj->byte_at_put(index, (jbyte)*((jint*)&val));
1318       break;
1319 #endif // INCLUDE_JVMCI
1320     }
1321 
1322     case T_BOOLEAN: {
1323       assert(value->type() == T_INT, "Agreement.");
1324       val = value->get_int();
1325       obj->bool_at_put(index, (jboolean)*((jint*)&val));
1326       break;
1327     }
1328 
1329       default:
1330         ShouldNotReachHere();
1331     }
1332     index++;
1333   }
1334 }
1335 
1336 // restore fields of an eliminated object array
1337 void Deoptimization::reassign_object_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, objArrayOop obj) {
1338   for (int i = 0; i < sv->field_size(); i++) {
1339     StackValue* value = StackValue::create_stack_value(fr, reg_map, sv->field_at(i));
1340     assert(value->type() == T_OBJECT, "object element expected");
1341     obj->obj_at_put(i, value->get_obj()());
1342   }
1343 }
1344 
1345 class ReassignedField {
1346 public:
1347   int _offset;
1348   BasicType _type;
1349   InstanceKlass* _klass;
1350 public:
1351   ReassignedField() {
1352     _offset = 0;
1353     _type = T_ILLEGAL;
1354     _klass = NULL;
1355   }
1356 };
1357 
1358 int compare(ReassignedField* left, ReassignedField* right) {
1359   return left->_offset - right->_offset;
1360 }
1361 
1362 // Restore fields of an eliminated instance object using the same field order
1363 // returned by HotSpotResolvedObjectTypeImpl.getInstanceFields(true)
1364 static int reassign_fields_by_klass(InstanceKlass* klass, frame* fr, RegisterMap* reg_map, ObjectValue* sv, int svIndex, oop obj, bool skip_internal, int base_offset, TRAPS) {
1365   GrowableArray<ReassignedField>* fields = new GrowableArray<ReassignedField>();
1366   InstanceKlass* ik = klass;
1367   while (ik != NULL) {
1368     for (AllFieldStream fs(ik); !fs.done(); fs.next()) {
1369       if (!fs.access_flags().is_static() && (!skip_internal || !fs.access_flags().is_internal())) {
1370         ReassignedField field;
1371         field._offset = fs.offset();
1372         field._type = Signature::basic_type(fs.signature());
1373         if (fs.signature()->is_Q_signature()) {
1374           if (fs.is_inlined()) {
1375             // Resolve klass of flattened inline type field
1376             field._klass = InlineKlass::cast(klass->get_inline_type_field_klass(fs.index()));
1377           } else {
1378             field._type = T_OBJECT;
1379           }
1380         }
1381         fields->append(field);
1382       }
1383     }
1384     ik = ik->superklass();
1385   }
1386   fields->sort(compare);
1387   for (int i = 0; i < fields->length(); i++) {
1388     BasicType type = fields->at(i)._type;
1389     int offset = base_offset + fields->at(i)._offset;
1390     // Check for flattened inline type field before accessing the ScopeValue because it might not have any fields
1391     if (type == T_INLINE_TYPE) {
1392       // Recursively re-assign flattened inline type fields
1393       InstanceKlass* vk = fields->at(i)._klass;
1394       assert(vk != NULL, "must be resolved");
1395       offset -= InlineKlass::cast(vk)->first_field_offset(); // Adjust offset to omit oop header
1396       svIndex = reassign_fields_by_klass(vk, fr, reg_map, sv, svIndex, obj, skip_internal, offset, CHECK_0);
1397       continue; // Continue because we don't need to increment svIndex
1398     }
1399     intptr_t val;
1400     ScopeValue* scope_field = sv->field_at(svIndex);
1401     StackValue* value = StackValue::create_stack_value(fr, reg_map, scope_field);


1402     switch (type) {
1403       case T_OBJECT:
1404       case T_ARRAY:
1405         assert(value->type() == T_OBJECT, "Agreement.");
1406         obj->obj_field_put(offset, value->get_obj()());
1407         break;
1408 
1409       // Have to cast to INT (32 bits) pointer to avoid little/big-endian problem.
1410       case T_INT: case T_FLOAT: { // 4 bytes.
1411         assert(value->type() == T_INT, "Agreement.");
1412         bool big_value = false;
1413         if (i+1 < fields->length() && fields->at(i+1)._type == T_INT) {
1414           if (scope_field->is_location()) {
1415             Location::Type type = ((LocationValue*) scope_field)->location().type();
1416             if (type == Location::dbl || type == Location::lng) {
1417               big_value = true;
1418             }
1419           }
1420           if (scope_field->is_constant_int()) {
1421             ScopeValue* next_scope_field = sv->field_at(svIndex + 1);
1422             if (next_scope_field->is_constant_long() || next_scope_field->is_constant_double()) {
1423               big_value = true;
1424             }
1425           }
1426         }
1427 
1428         if (big_value) {
1429           i++;
1430           assert(i < fields->length(), "second T_INT field needed");
1431           assert(fields->at(i)._type == T_INT, "T_INT field needed");
1432         } else {
1433           val = value->get_int();
1434           obj->int_field_put(offset, (jint)*((jint*)&val));
1435           break;
1436         }
1437       }
1438         /* no break */
1439 
1440       case T_LONG: case T_DOUBLE: {
1441         assert(value->type() == T_INT, "Agreement.");
1442         StackValue* low = StackValue::create_stack_value(fr, reg_map, sv->field_at(++svIndex));
1443 #ifdef _LP64
1444         jlong res = (jlong)low->get_int();
1445 #else
1446         jlong res = jlong_from((jint)value->get_int(), (jint)low->get_int());
1447 #endif
1448         obj->long_field_put(offset, res);
1449         break;
1450       }
1451 
1452       case T_SHORT:
1453         assert(value->type() == T_INT, "Agreement.");
1454         val = value->get_int();
1455         obj->short_field_put(offset, (jshort)*((jint*)&val));
1456         break;
1457 
1458       case T_CHAR:
1459         assert(value->type() == T_INT, "Agreement.");
1460         val = value->get_int();
1461         obj->char_field_put(offset, (jchar)*((jint*)&val));
1462         break;
1463 
1464       case T_BYTE:
1465         assert(value->type() == T_INT, "Agreement.");
1466         val = value->get_int();
1467         obj->byte_field_put(offset, (jbyte)*((jint*)&val));
1468         break;
1469 
1470       case T_BOOLEAN:
1471         assert(value->type() == T_INT, "Agreement.");
1472         val = value->get_int();
1473         obj->bool_field_put(offset, (jboolean)*((jint*)&val));
1474         break;
1475 
1476       default:
1477         ShouldNotReachHere();
1478     }
1479     svIndex++;
1480   }
1481   return svIndex;
1482 }
1483 
1484 // restore fields of an eliminated inline type array
1485 void Deoptimization::reassign_flat_array_elements(frame* fr, RegisterMap* reg_map, ObjectValue* sv, flatArrayOop obj, FlatArrayKlass* vak, bool skip_internal, TRAPS) {
1486   InlineKlass* vk = vak->element_klass();
1487   assert(vk->flatten_array(), "should only be used for flattened inline type arrays");
1488   // Adjust offset to omit oop header
1489   int base_offset = arrayOopDesc::base_offset_in_bytes(T_INLINE_TYPE) - InlineKlass::cast(vk)->first_field_offset();
1490   // Initialize all elements of the flattened inline type array
1491   for (int i = 0; i < sv->field_size(); i++) {
1492     ScopeValue* val = sv->field_at(i);
1493     int offset = base_offset + (i << Klass::layout_helper_log2_element_size(vak->layout_helper()));
1494     reassign_fields_by_klass(vk, fr, reg_map, val->as_ObjectValue(), 0, (oop)obj, skip_internal, offset, CHECK);
1495   }
1496 }
1497 
1498 // restore fields of all eliminated objects and arrays
1499 void Deoptimization::reassign_fields(frame* fr, RegisterMap* reg_map, GrowableArray<ScopeValue*>* objects, bool realloc_failures, bool skip_internal, TRAPS) {
1500   for (int i = 0; i < objects->length(); i++) {
1501     ObjectValue* sv = (ObjectValue*) objects->at(i);
1502     Klass* k = java_lang_Class::as_Klass(sv->klass()->as_ConstantOopReadValue()->value()());
1503     Handle obj = sv->value();
1504     assert(obj.not_null() || realloc_failures || sv->maybe_null(), "reallocation was missed");
1505     if (PrintDeoptimizationDetails) {
1506       tty->print_cr("reassign fields for object of type %s!", k->name()->as_C_string());
1507     }
1508     if (obj.is_null()) {
1509       continue;
1510     }
1511 
1512     // Don't reassign fields of boxes that came from a cache. Caches may be in CDS.
1513     if (sv->is_auto_box() && ((AutoBoxObjectValue*) sv)->is_cached()) {
1514       continue;
1515     }
1516 #ifdef COMPILER2
1517     if (EnableVectorSupport && VectorSupport::is_vector(k)) {
1518       assert(sv->field_size() == 1, "%s not a vector", k->name()->as_C_string());
1519       ScopeValue* payload = sv->field_at(0);
1520       if (payload->is_location() &&
1521           payload->as_LocationValue()->location().type() == Location::vector) {
1522         if (PrintDeoptimizationDetails) {
1523           tty->print_cr("skip field reassignment for this vector - it should be assigned already");
1524           if (Verbose) {
1525             Handle obj = sv->value();
1526             k->oop_print_on(obj(), tty);
1527           }
1528         }
1529         continue; // Such vector's value was already restored in VectorSupport::allocate_vector().
1530       }
1531       // Else fall-through to do assignment for scalar-replaced boxed vector representation
1532       // which could be restored after vector object allocation.
1533     }
1534 #endif
1535     if (k->is_instance_klass()) {
1536       InstanceKlass* ik = InstanceKlass::cast(k);
1537       reassign_fields_by_klass(ik, fr, reg_map, sv, 0, obj(), skip_internal, 0, CHECK);
1538     } else if (k->is_flatArray_klass()) {
1539       FlatArrayKlass* vak = FlatArrayKlass::cast(k);
1540       reassign_flat_array_elements(fr, reg_map, sv, (flatArrayOop) obj(), vak, skip_internal, CHECK);
1541     } else if (k->is_typeArray_klass()) {
1542       TypeArrayKlass* ak = TypeArrayKlass::cast(k);
1543       reassign_type_array_elements(fr, reg_map, sv, (typeArrayOop) obj(), ak->element_type());
1544     } else if (k->is_objArray_klass()) {
1545       reassign_object_array_elements(fr, reg_map, sv, (objArrayOop) obj());
1546     }
1547   }
1548 }
1549 
1550 
1551 // relock objects for which synchronization was eliminated
1552 bool Deoptimization::relock_objects(JavaThread* thread, GrowableArray<MonitorInfo*>* monitors,
1553                                     JavaThread* deoptee_thread, frame& fr, int exec_mode, bool realloc_failures) {
1554   bool relocked_objects = false;
1555   for (int i = 0; i < monitors->length(); i++) {
1556     MonitorInfo* mon_info = monitors->at(i);
1557     if (mon_info->eliminated()) {
1558       assert(!mon_info->owner_is_scalar_replaced() || realloc_failures, "reallocation was missed");
1559       relocked_objects = true;
1560       if (!mon_info->owner_is_scalar_replaced()) {
1561         Handle obj(thread, mon_info->owner());
1562         markWord mark = obj->mark();
1563         if (exec_mode == Unpack_none) {
1564           if (mark.has_locker() && fr.sp() > (intptr_t*)mark.locker()) {
1565             // With exec_mode == Unpack_none obj may be thread local and locked in
1566             // a callee frame. Make the lock in the callee a recursive lock and restore the displaced header.
1567             markWord dmw = mark.displaced_mark_helper();
1568             mark.locker()->set_displaced_header(markWord::encode((BasicLock*) NULL));
1569             obj->set_mark(dmw);
1570           }
1571           if (mark.has_monitor()) {
1572             // defer relocking if the deoptee thread is currently waiting for obj
1573             ObjectMonitor* waiting_monitor = deoptee_thread->current_waiting_monitor();
1574             if (waiting_monitor != NULL && waiting_monitor->object() == obj()) {
1575               assert(fr.is_deoptimized_frame(), "frame must be scheduled for deoptimization");
1576               mon_info->lock()->set_displaced_header(markWord::unused_mark());
1577               JvmtiDeferredUpdates::inc_relock_count_after_wait(deoptee_thread);
1578               continue;
1579             }
1580           }
1581         }
1582         BasicLock* lock = mon_info->lock();
1583         ObjectSynchronizer::enter(obj, lock, deoptee_thread);
1584         assert(mon_info->owner()->is_locked(), "object must be locked now");
1585       }
1586     }
1587   }
1588   return relocked_objects;
1589 }
1590 #endif // COMPILER2_OR_JVMCI
1591 
1592 vframeArray* Deoptimization::create_vframeArray(JavaThread* thread, frame fr, RegisterMap *reg_map, GrowableArray<compiledVFrame*>* chunk, bool realloc_failures) {
1593   Events::log_deopt_message(thread, "DEOPT PACKING pc=" INTPTR_FORMAT " sp=" INTPTR_FORMAT, p2i(fr.pc()), p2i(fr.sp()));
1594 
1595 #ifndef PRODUCT
1596   if (PrintDeoptimizationDetails) {
1597     ResourceMark rm;
1598     stringStream st;
1599     st.print("DEOPT PACKING thread " INTPTR_FORMAT " ", p2i(thread));
1600     fr.print_on(&st);
1601     st.print_cr("     Virtual frames (innermost first):");
1602     for (int index = 0; index < chunk->length(); index++) {
1603       compiledVFrame* vf = chunk->at(index);
1604       st.print("       %2d - ", index);
1605       vf->print_value_on(&st);
1606       int bci = chunk->at(index)->raw_bci();
1607       const char* code_name;
1608       if (bci == SynchronizationEntryBCI) {
1609         code_name = "sync entry";
1610       } else {
1611         Bytecodes::Code code = vf->method()->code_at(bci);
1612         code_name = Bytecodes::name(code);
1613       }
1614       st.print(" - %s", code_name);
1615       st.print_cr(" @ bci %d ", bci);
1616       if (Verbose) {
1617         vf->print_on(&st);
1618         st.cr();
1619       }
1620     }
1621     tty->print_raw(st.as_string());
1622   }
1623 #endif
1624 
1625   // Register map for next frame (used for stack crawl).  We capture
1626   // the state of the deopt'ing frame's caller.  Thus if we need to
1627   // stuff a C2I adapter we can properly fill in the callee-save
1628   // register locations.
1629   frame caller = fr.sender(reg_map);
1630   int frame_size = caller.sp() - fr.sp();
1631 
1632   frame sender = caller;
1633 
1634   // Since the Java thread being deoptimized will eventually adjust it's own stack,
1635   // the vframeArray containing the unpacking information is allocated in the C heap.
1636   // For Compiler1, the caller of the deoptimized frame is saved for use by unpack_frames().
1637   vframeArray* array = vframeArray::allocate(thread, frame_size, chunk, reg_map, sender, caller, fr, realloc_failures);
1638 
1639   // Compare the vframeArray to the collected vframes
1640   assert(array->structural_compare(thread, chunk), "just checking");
1641 
1642 #ifndef PRODUCT
1643   if (PrintDeoptimizationDetails) {
1644     tty->print_cr("     Created vframeArray " INTPTR_FORMAT, p2i(array));
1645   }
1646 #endif // PRODUCT
1647 
1648   return array;
1649 }
1650 
1651 #if COMPILER2_OR_JVMCI
1652 void Deoptimization::pop_frames_failed_reallocs(JavaThread* thread, vframeArray* array) {
1653   // Reallocation of some scalar replaced objects failed. Record
1654   // that we need to pop all the interpreter frames for the
1655   // deoptimized compiled frame.
1656   assert(thread->frames_to_pop_failed_realloc() == 0, "missed frames to pop?");
1657   thread->set_frames_to_pop_failed_realloc(array->frames());
1658   // Unlock all monitors here otherwise the interpreter will see a
1659   // mix of locked and unlocked monitors (because of failed
1660   // reallocations of synchronized objects) and be confused.
1661   for (int i = 0; i < array->frames(); i++) {
1662     MonitorChunk* monitors = array->element(i)->monitors();
1663     if (monitors != NULL) {
1664       for (int j = 0; j < monitors->number_of_monitors(); j++) {
1665         BasicObjectLock* src = monitors->at(j);
1666         if (src->obj() != NULL) {
1667           ObjectSynchronizer::exit(src->obj(), src->lock(), thread);
1668         }
1669       }
1670       array->element(i)->free_monitors(thread);
1671 #ifdef ASSERT
1672       array->element(i)->set_removed_monitors();
1673 #endif
1674     }
1675   }
1676 }
1677 #endif
1678 
1679 void Deoptimization::deoptimize_single_frame(JavaThread* thread, frame fr, Deoptimization::DeoptReason reason) {
1680   assert(fr.can_be_deoptimized(), "checking frame type");
1681 
1682   gather_statistics(reason, Action_none, Bytecodes::_illegal);
1683 
1684   if (LogCompilation && xtty != NULL) {
1685     CompiledMethod* cm = fr.cb()->as_compiled_method_or_null();
1686     assert(cm != NULL, "only compiled methods can deopt");
1687 
1688     ttyLocker ttyl;
1689     xtty->begin_head("deoptimized thread='" UINTX_FORMAT "' reason='%s' pc='" INTPTR_FORMAT "'",(uintx)thread->osthread()->thread_id(), trap_reason_name(reason), p2i(fr.pc()));
1690     cm->log_identity(xtty);
1691     xtty->end_head();
1692     for (ScopeDesc* sd = cm->scope_desc_at(fr.pc()); ; sd = sd->sender()) {
1693       xtty->begin_elem("jvms bci='%d'", sd->bci());
1694       xtty->method(sd->method());
1695       xtty->end_elem();
1696       if (sd->is_top())  break;
1697     }
1698     xtty->tail("deoptimized");
1699   }
1700 
1701   // Patch the compiled method so that when execution returns to it we will
1702   // deopt the execution state and return to the interpreter.
1703   fr.deoptimize(thread);
1704 }
1705 
1706 void Deoptimization::deoptimize(JavaThread* thread, frame fr, DeoptReason reason) {
1707   // Deoptimize only if the frame comes from compiled code.
1708   // Do not deoptimize the frame which is already patched
1709   // during the execution of the loops below.
1710   if (!fr.is_compiled_frame() || fr.is_deoptimized_frame()) {
1711     return;
1712   }
1713   ResourceMark rm;
1714   DeoptimizationMarker dm;
1715   deoptimize_single_frame(thread, fr, reason);
1716 }
1717 
1718 #if INCLUDE_JVMCI
1719 address Deoptimization::deoptimize_for_missing_exception_handler(CompiledMethod* cm) {
1720   // there is no exception handler for this pc => deoptimize
1721   cm->make_not_entrant();
1722 
1723   // Use Deoptimization::deoptimize for all of its side-effects:
1724   // gathering traps statistics, logging...
1725   // it also patches the return pc but we do not care about that
1726   // since we return a continuation to the deopt_blob below.
1727   JavaThread* thread = JavaThread::current();
1728   RegisterMap reg_map(thread, false);
1729   frame runtime_frame = thread->last_frame();
1730   frame caller_frame = runtime_frame.sender(&reg_map);
1731   assert(caller_frame.cb()->as_compiled_method_or_null() == cm, "expect top frame compiled method");
1732   vframe* vf = vframe::new_vframe(&caller_frame, &reg_map, thread);
1733   compiledVFrame* cvf = compiledVFrame::cast(vf);
1734   ScopeDesc* imm_scope = cvf->scope();
1735   MethodData* imm_mdo = get_method_data(thread, methodHandle(thread, imm_scope->method()), true);
1736   if (imm_mdo != NULL) {
1737     ProfileData* pdata = imm_mdo->allocate_bci_to_data(imm_scope->bci(), NULL);
1738     if (pdata != NULL && pdata->is_BitData()) {
1739       BitData* bit_data = (BitData*) pdata;
1740       bit_data->set_exception_seen();
1741     }
1742   }
1743 
1744   Deoptimization::deoptimize(thread, caller_frame, Deoptimization::Reason_not_compiled_exception_handler);
1745 
1746   MethodData* trap_mdo = get_method_data(thread, methodHandle(thread, cm->method()), true);
1747   if (trap_mdo != NULL) {
1748     trap_mdo->inc_trap_count(Deoptimization::Reason_not_compiled_exception_handler);
1749   }
1750 
1751   return SharedRuntime::deopt_blob()->unpack_with_exception_in_tls();
1752 }
1753 #endif
1754 
1755 void Deoptimization::deoptimize_frame_internal(JavaThread* thread, intptr_t* id, DeoptReason reason) {
1756   assert(thread == Thread::current() ||
1757          thread->is_handshake_safe_for(Thread::current()) ||
1758          SafepointSynchronize::is_at_safepoint(),
1759          "can only deoptimize other thread at a safepoint/handshake");
1760   // Compute frame and register map based on thread and sp.
1761   RegisterMap reg_map(thread, false);
1762   frame fr = thread->last_frame();
1763   while (fr.id() != id) {
1764     fr = fr.sender(&reg_map);
1765   }
1766   deoptimize(thread, fr, reason);
1767 }
1768 
1769 
1770 void Deoptimization::deoptimize_frame(JavaThread* thread, intptr_t* id, DeoptReason reason) {
1771   Thread* current = Thread::current();
1772   if (thread == current || thread->is_handshake_safe_for(current)) {
1773     Deoptimization::deoptimize_frame_internal(thread, id, reason);
1774   } else {
1775     VM_DeoptimizeFrame deopt(thread, id, reason);
1776     VMThread::execute(&deopt);
1777   }
1778 }
1779 
1780 void Deoptimization::deoptimize_frame(JavaThread* thread, intptr_t* id) {
1781   deoptimize_frame(thread, id, Reason_constraint);
1782 }
1783 
1784 // JVMTI PopFrame support
1785 JRT_LEAF(void, Deoptimization::popframe_preserve_args(JavaThread* thread, int bytes_to_save, void* start_address))
1786 {
1787   thread->popframe_preserve_args(in_ByteSize(bytes_to_save), start_address);
1788 }
1789 JRT_END
1790 
1791 MethodData*
1792 Deoptimization::get_method_data(JavaThread* thread, const methodHandle& m,
1793                                 bool create_if_missing) {
1794   JavaThread* THREAD = thread; // For exception macros.
1795   MethodData* mdo = m()->method_data();
1796   if (mdo == NULL && create_if_missing && !HAS_PENDING_EXCEPTION) {
1797     // Build an MDO.  Ignore errors like OutOfMemory;
1798     // that simply means we won't have an MDO to update.
1799     Method::build_interpreter_method_data(m, THREAD);
1800     if (HAS_PENDING_EXCEPTION) {
1801       // Only metaspace OOM is expected. No Java code executed.
1802       assert((PENDING_EXCEPTION->is_a(vmClasses::OutOfMemoryError_klass())), "we expect only an OOM error here");
1803       CLEAR_PENDING_EXCEPTION;
1804     }
1805     mdo = m()->method_data();
1806   }
1807   return mdo;
1808 }
1809 
1810 #if COMPILER2_OR_JVMCI
1811 void Deoptimization::load_class_by_index(const constantPoolHandle& constant_pool, int index, TRAPS) {
1812   // In case of an unresolved klass entry, load the class.
1813   // This path is exercised from case _ldc in Parse::do_one_bytecode,
1814   // and probably nowhere else.
1815   // Even that case would benefit from simply re-interpreting the
1816   // bytecode, without paying special attention to the class index.
1817   // So this whole "class index" feature should probably be removed.
1818 
1819   if (constant_pool->tag_at(index).is_unresolved_klass()) {
1820     Klass* tk = constant_pool->klass_at(index, THREAD);
1821     if (HAS_PENDING_EXCEPTION) {
1822       // Exception happened during classloading. We ignore the exception here, since it
1823       // is going to be rethrown since the current activation is going to be deoptimized and
1824       // the interpreter will re-execute the bytecode.
1825       // Do not clear probable Async Exceptions.
1826       CLEAR_PENDING_NONASYNC_EXCEPTION;
1827       // Class loading called java code which may have caused a stack
1828       // overflow. If the exception was thrown right before the return
1829       // to the runtime the stack is no longer guarded. Reguard the
1830       // stack otherwise if we return to the uncommon trap blob and the
1831       // stack bang causes a stack overflow we crash.
1832       JavaThread* jt = THREAD;
1833       bool guard_pages_enabled = jt->stack_overflow_state()->reguard_stack_if_needed();
1834       assert(guard_pages_enabled, "stack banging in uncommon trap blob may cause crash");
1835     }
1836     return;
1837   }
1838 
1839   assert(!constant_pool->tag_at(index).is_symbol(),
1840          "no symbolic names here, please");
1841 }
1842 
1843 #if INCLUDE_JFR
1844 
1845 class DeoptReasonSerializer : public JfrSerializer {
1846  public:
1847   void serialize(JfrCheckpointWriter& writer) {
1848     writer.write_count((u4)(Deoptimization::Reason_LIMIT + 1)); // + Reason::many (-1)
1849     for (int i = -1; i < Deoptimization::Reason_LIMIT; ++i) {
1850       writer.write_key((u8)i);
1851       writer.write(Deoptimization::trap_reason_name(i));
1852     }
1853   }
1854 };
1855 
1856 class DeoptActionSerializer : public JfrSerializer {
1857  public:
1858   void serialize(JfrCheckpointWriter& writer) {
1859     static const u4 nof_actions = Deoptimization::Action_LIMIT;
1860     writer.write_count(nof_actions);
1861     for (u4 i = 0; i < Deoptimization::Action_LIMIT; ++i) {
1862       writer.write_key(i);
1863       writer.write(Deoptimization::trap_action_name((int)i));
1864     }
1865   }
1866 };
1867 
1868 static void register_serializers() {
1869   static int critical_section = 0;
1870   if (1 == critical_section || Atomic::cmpxchg(&critical_section, 0, 1) == 1) {
1871     return;
1872   }
1873   JfrSerializer::register_serializer(TYPE_DEOPTIMIZATIONREASON, true, new DeoptReasonSerializer());
1874   JfrSerializer::register_serializer(TYPE_DEOPTIMIZATIONACTION, true, new DeoptActionSerializer());
1875 }
1876 
1877 static void post_deoptimization_event(CompiledMethod* nm,
1878                                       const Method* method,
1879                                       int trap_bci,
1880                                       int instruction,
1881                                       Deoptimization::DeoptReason reason,
1882                                       Deoptimization::DeoptAction action) {
1883   assert(nm != NULL, "invariant");
1884   assert(method != NULL, "invariant");
1885   if (EventDeoptimization::is_enabled()) {
1886     static bool serializers_registered = false;
1887     if (!serializers_registered) {
1888       register_serializers();
1889       serializers_registered = true;
1890     }
1891     EventDeoptimization event;
1892     event.set_compileId(nm->compile_id());
1893     event.set_compiler(nm->compiler_type());
1894     event.set_method(method);
1895     event.set_lineNumber(method->line_number_from_bci(trap_bci));
1896     event.set_bci(trap_bci);
1897     event.set_instruction(instruction);
1898     event.set_reason(reason);
1899     event.set_action(action);
1900     event.commit();
1901   }
1902 }
1903 
1904 #endif // INCLUDE_JFR
1905 
1906 JRT_ENTRY(void, Deoptimization::uncommon_trap_inner(JavaThread* current, jint trap_request)) {
1907   HandleMark hm(current);
1908 
1909   // uncommon_trap() is called at the beginning of the uncommon trap
1910   // handler. Note this fact before we start generating temporary frames
1911   // that can confuse an asynchronous stack walker. This counter is
1912   // decremented at the end of unpack_frames().
1913   current->inc_in_deopt_handler();
1914 
1915 #if INCLUDE_JVMCI
1916   // JVMCI might need to get an exception from the stack, which in turn requires the register map to be valid
1917   RegisterMap reg_map(current, true);
1918 #else
1919   RegisterMap reg_map(current, false);
1920 #endif
1921   frame stub_frame = current->last_frame();
1922   frame fr = stub_frame.sender(&reg_map);
1923   // Make sure the calling nmethod is not getting deoptimized and removed
1924   // before we are done with it.
1925   nmethodLocker nl(fr.pc());
1926 
1927   // Log a message
1928   Events::log_deopt_message(current, "Uncommon trap: trap_request=" PTR32_FORMAT " fr.pc=" INTPTR_FORMAT " relative=" INTPTR_FORMAT,
1929               trap_request, p2i(fr.pc()), fr.pc() - fr.cb()->code_begin());
1930 
1931   {
1932     ResourceMark rm;
1933 
1934     DeoptReason reason = trap_request_reason(trap_request);
1935     DeoptAction action = trap_request_action(trap_request);
1936 #if INCLUDE_JVMCI
1937     int debug_id = trap_request_debug_id(trap_request);
1938 #endif
1939     jint unloaded_class_index = trap_request_index(trap_request); // CP idx or -1
1940 
1941     vframe*  vf  = vframe::new_vframe(&fr, &reg_map, current);
1942     compiledVFrame* cvf = compiledVFrame::cast(vf);
1943 
1944     CompiledMethod* nm = cvf->code();
1945 
1946     ScopeDesc*      trap_scope  = cvf->scope();
1947 
1948     bool is_receiver_constraint_failure = COMPILER2_PRESENT(VerifyReceiverTypes &&) (reason == Deoptimization::Reason_receiver_constraint);
1949 
1950     if (TraceDeoptimization || is_receiver_constraint_failure) {
1951       tty->print_cr("  bci=%d pc=" INTPTR_FORMAT ", relative_pc=" INTPTR_FORMAT ", method=%s" JVMCI_ONLY(", debug_id=%d"), trap_scope->bci(), p2i(fr.pc()), fr.pc() - nm->code_begin(), trap_scope->method()->name_and_sig_as_C_string()
1952 #if INCLUDE_JVMCI
1953           , debug_id
1954 #endif
1955           );
1956     }
1957 
1958     methodHandle    trap_method(current, trap_scope->method());
1959     int             trap_bci    = trap_scope->bci();
1960 #if INCLUDE_JVMCI
1961     jlong           speculation = current->pending_failed_speculation();
1962     if (nm->is_compiled_by_jvmci()) {
1963       nm->as_nmethod()->update_speculation(current);
1964     } else {
1965       assert(speculation == 0, "There should not be a speculation for methods compiled by non-JVMCI compilers");
1966     }
1967 
1968     if (trap_bci == SynchronizationEntryBCI) {
1969       trap_bci = 0;
1970       current->set_pending_monitorenter(true);
1971     }
1972 
1973     if (reason == Deoptimization::Reason_transfer_to_interpreter) {
1974       current->set_pending_transfer_to_interpreter(true);
1975     }
1976 #endif
1977 
1978     Bytecodes::Code trap_bc     = trap_method->java_code_at(trap_bci);
1979     // Record this event in the histogram.
1980     gather_statistics(reason, action, trap_bc);
1981 
1982     // Ensure that we can record deopt. history:
1983     // Need MDO to record RTM code generation state.
1984     bool create_if_missing = ProfileTraps || UseCodeAging RTM_OPT_ONLY( || UseRTMLocking );
1985 
1986     methodHandle profiled_method;
1987 #if INCLUDE_JVMCI
1988     if (nm->is_compiled_by_jvmci()) {
1989       profiled_method = methodHandle(current, nm->method());
1990     } else {
1991       profiled_method = trap_method;
1992     }
1993 #else
1994     profiled_method = trap_method;
1995 #endif
1996 
1997     MethodData* trap_mdo =
1998       get_method_data(current, profiled_method, create_if_missing);
1999 
2000     JFR_ONLY(post_deoptimization_event(nm, trap_method(), trap_bci, trap_bc, reason, action);)
2001 
2002     // Log a message
2003     Events::log_deopt_message(current, "Uncommon trap: reason=%s action=%s pc=" INTPTR_FORMAT " method=%s @ %d %s",
2004                               trap_reason_name(reason), trap_action_name(action), p2i(fr.pc()),
2005                               trap_method->name_and_sig_as_C_string(), trap_bci, nm->compiler_name());
2006 
2007     // Print a bunch of diagnostics, if requested.
2008     if (TraceDeoptimization || LogCompilation || is_receiver_constraint_failure) {
2009       ResourceMark rm;
2010       ttyLocker ttyl;
2011       char buf[100];
2012       if (xtty != NULL) {
2013         xtty->begin_head("uncommon_trap thread='" UINTX_FORMAT "' %s",
2014                          os::current_thread_id(),
2015                          format_trap_request(buf, sizeof(buf), trap_request));
2016 #if INCLUDE_JVMCI
2017         if (speculation != 0) {
2018           xtty->print(" speculation='" JLONG_FORMAT "'", speculation);
2019         }
2020 #endif
2021         nm->log_identity(xtty);
2022       }
2023       Symbol* class_name = NULL;
2024       bool unresolved = false;
2025       if (unloaded_class_index >= 0) {
2026         constantPoolHandle constants (current, trap_method->constants());
2027         if (constants->tag_at(unloaded_class_index).is_unresolved_klass()) {
2028           class_name = constants->klass_name_at(unloaded_class_index);
2029           unresolved = true;
2030           if (xtty != NULL)
2031             xtty->print(" unresolved='1'");
2032         } else if (constants->tag_at(unloaded_class_index).is_symbol()) {
2033           class_name = constants->symbol_at(unloaded_class_index);
2034         }
2035         if (xtty != NULL)
2036           xtty->name(class_name);
2037       }
2038       if (xtty != NULL && trap_mdo != NULL && (int)reason < (int)MethodData::_trap_hist_limit) {
2039         // Dump the relevant MDO state.
2040         // This is the deopt count for the current reason, any previous
2041         // reasons or recompiles seen at this point.
2042         int dcnt = trap_mdo->trap_count(reason);
2043         if (dcnt != 0)
2044           xtty->print(" count='%d'", dcnt);
2045         ProfileData* pdata = trap_mdo->bci_to_data(trap_bci);
2046         int dos = (pdata == NULL)? 0: pdata->trap_state();
2047         if (dos != 0) {
2048           xtty->print(" state='%s'", format_trap_state(buf, sizeof(buf), dos));
2049           if (trap_state_is_recompiled(dos)) {
2050             int recnt2 = trap_mdo->overflow_recompile_count();
2051             if (recnt2 != 0)
2052               xtty->print(" recompiles2='%d'", recnt2);
2053           }
2054         }
2055       }
2056       if (xtty != NULL) {
2057         xtty->stamp();
2058         xtty->end_head();
2059       }
2060       if (TraceDeoptimization) {  // make noise on the tty
2061         tty->print("Uncommon trap occurred in");
2062         nm->method()->print_short_name(tty);
2063         tty->print(" compiler=%s compile_id=%d", nm->compiler_name(), nm->compile_id());
2064 #if INCLUDE_JVMCI
2065         if (nm->is_nmethod()) {
2066           const char* installed_code_name = nm->as_nmethod()->jvmci_name();
2067           if (installed_code_name != NULL) {
2068             tty->print(" (JVMCI: installed code name=%s) ", installed_code_name);
2069           }
2070         }
2071 #endif
2072         tty->print(" (@" INTPTR_FORMAT ") thread=" UINTX_FORMAT " reason=%s action=%s unloaded_class_index=%d" JVMCI_ONLY(" debug_id=%d"),
2073                    p2i(fr.pc()),
2074                    os::current_thread_id(),
2075                    trap_reason_name(reason),
2076                    trap_action_name(action),
2077                    unloaded_class_index
2078 #if INCLUDE_JVMCI
2079                    , debug_id
2080 #endif
2081                    );
2082         if (class_name != NULL) {
2083           tty->print(unresolved ? " unresolved class: " : " symbol: ");
2084           class_name->print_symbol_on(tty);
2085         }
2086         tty->cr();
2087       }
2088       if (xtty != NULL) {
2089         // Log the precise location of the trap.
2090         for (ScopeDesc* sd = trap_scope; ; sd = sd->sender()) {
2091           xtty->begin_elem("jvms bci='%d'", sd->bci());
2092           xtty->method(sd->method());
2093           xtty->end_elem();
2094           if (sd->is_top())  break;
2095         }
2096         xtty->tail("uncommon_trap");
2097       }
2098     }
2099     // (End diagnostic printout.)
2100 
2101     if (is_receiver_constraint_failure) {
2102       fatal("missing receiver type check");
2103     }
2104 
2105     // Load class if necessary
2106     if (unloaded_class_index >= 0) {
2107       constantPoolHandle constants(current, trap_method->constants());
2108       load_class_by_index(constants, unloaded_class_index, THREAD);
2109     }
2110 
2111     // Flush the nmethod if necessary and desirable.
2112     //
2113     // We need to avoid situations where we are re-flushing the nmethod
2114     // because of a hot deoptimization site.  Repeated flushes at the same
2115     // point need to be detected by the compiler and avoided.  If the compiler
2116     // cannot avoid them (or has a bug and "refuses" to avoid them), this
2117     // module must take measures to avoid an infinite cycle of recompilation
2118     // and deoptimization.  There are several such measures:
2119     //
2120     //   1. If a recompilation is ordered a second time at some site X
2121     //   and for the same reason R, the action is adjusted to 'reinterpret',
2122     //   to give the interpreter time to exercise the method more thoroughly.
2123     //   If this happens, the method's overflow_recompile_count is incremented.
2124     //
2125     //   2. If the compiler fails to reduce the deoptimization rate, then
2126     //   the method's overflow_recompile_count will begin to exceed the set
2127     //   limit PerBytecodeRecompilationCutoff.  If this happens, the action
2128     //   is adjusted to 'make_not_compilable', and the method is abandoned
2129     //   to the interpreter.  This is a performance hit for hot methods,
2130     //   but is better than a disastrous infinite cycle of recompilations.
2131     //   (Actually, only the method containing the site X is abandoned.)
2132     //
2133     //   3. In parallel with the previous measures, if the total number of
2134     //   recompilations of a method exceeds the much larger set limit
2135     //   PerMethodRecompilationCutoff, the method is abandoned.
2136     //   This should only happen if the method is very large and has
2137     //   many "lukewarm" deoptimizations.  The code which enforces this
2138     //   limit is elsewhere (class nmethod, class Method).
2139     //
2140     // Note that the per-BCI 'is_recompiled' bit gives the compiler one chance
2141     // to recompile at each bytecode independently of the per-BCI cutoff.
2142     //
2143     // The decision to update code is up to the compiler, and is encoded
2144     // in the Action_xxx code.  If the compiler requests Action_none
2145     // no trap state is changed, no compiled code is changed, and the
2146     // computation suffers along in the interpreter.
2147     //
2148     // The other action codes specify various tactics for decompilation
2149     // and recompilation.  Action_maybe_recompile is the loosest, and
2150     // allows the compiled code to stay around until enough traps are seen,
2151     // and until the compiler gets around to recompiling the trapping method.
2152     //
2153     // The other actions cause immediate removal of the present code.
2154 
2155     // Traps caused by injected profile shouldn't pollute trap counts.
2156     bool injected_profile_trap = trap_method->has_injected_profile() &&
2157                                  (reason == Reason_intrinsic || reason == Reason_unreached);
2158 
2159     bool update_trap_state = (reason != Reason_tenured) && !injected_profile_trap;
2160     bool make_not_entrant = false;
2161     bool make_not_compilable = false;
2162     bool reprofile = false;
2163     switch (action) {
2164     case Action_none:
2165       // Keep the old code.
2166       update_trap_state = false;
2167       break;
2168     case Action_maybe_recompile:
2169       // Do not need to invalidate the present code, but we can
2170       // initiate another
2171       // Start compiler without (necessarily) invalidating the nmethod.
2172       // The system will tolerate the old code, but new code should be
2173       // generated when possible.
2174       break;
2175     case Action_reinterpret:
2176       // Go back into the interpreter for a while, and then consider
2177       // recompiling form scratch.
2178       make_not_entrant = true;
2179       // Reset invocation counter for outer most method.
2180       // This will allow the interpreter to exercise the bytecodes
2181       // for a while before recompiling.
2182       // By contrast, Action_make_not_entrant is immediate.
2183       //
2184       // Note that the compiler will track null_check, null_assert,
2185       // range_check, and class_check events and log them as if they
2186       // had been traps taken from compiled code.  This will update
2187       // the MDO trap history so that the next compilation will
2188       // properly detect hot trap sites.
2189       reprofile = true;
2190       break;
2191     case Action_make_not_entrant:
2192       // Request immediate recompilation, and get rid of the old code.
2193       // Make them not entrant, so next time they are called they get
2194       // recompiled.  Unloaded classes are loaded now so recompile before next
2195       // time they are called.  Same for uninitialized.  The interpreter will
2196       // link the missing class, if any.
2197       make_not_entrant = true;
2198       break;
2199     case Action_make_not_compilable:
2200       // Give up on compiling this method at all.
2201       make_not_entrant = true;
2202       make_not_compilable = true;
2203       break;
2204     default:
2205       ShouldNotReachHere();
2206     }
2207 
2208     // Setting +ProfileTraps fixes the following, on all platforms:
2209     // 4852688: ProfileInterpreter is off by default for ia64.  The result is
2210     // infinite heroic-opt-uncommon-trap/deopt/recompile cycles, since the
2211     // recompile relies on a MethodData* to record heroic opt failures.
2212 
2213     // Whether the interpreter is producing MDO data or not, we also need
2214     // to use the MDO to detect hot deoptimization points and control
2215     // aggressive optimization.
2216     bool inc_recompile_count = false;
2217     ProfileData* pdata = NULL;
2218     if (ProfileTraps && CompilerConfig::is_c2_or_jvmci_compiler_enabled() && update_trap_state && trap_mdo != NULL) {
2219       assert(trap_mdo == get_method_data(current, profiled_method, false), "sanity");
2220       uint this_trap_count = 0;
2221       bool maybe_prior_trap = false;
2222       bool maybe_prior_recompile = false;
2223       pdata = query_update_method_data(trap_mdo, trap_bci, reason, true,
2224 #if INCLUDE_JVMCI
2225                                    nm->is_compiled_by_jvmci() && nm->is_osr_method(),
2226 #endif
2227                                    nm->method(),
2228                                    //outputs:
2229                                    this_trap_count,
2230                                    maybe_prior_trap,
2231                                    maybe_prior_recompile);
2232       // Because the interpreter also counts null, div0, range, and class
2233       // checks, these traps from compiled code are double-counted.
2234       // This is harmless; it just means that the PerXTrapLimit values
2235       // are in effect a little smaller than they look.
2236 
2237       DeoptReason per_bc_reason = reason_recorded_per_bytecode_if_any(reason);
2238       if (per_bc_reason != Reason_none) {
2239         // Now take action based on the partially known per-BCI history.
2240         if (maybe_prior_trap
2241             && this_trap_count >= (uint)PerBytecodeTrapLimit) {
2242           // If there are too many traps at this BCI, force a recompile.
2243           // This will allow the compiler to see the limit overflow, and
2244           // take corrective action, if possible.  The compiler generally
2245           // does not use the exact PerBytecodeTrapLimit value, but instead
2246           // changes its tactics if it sees any traps at all.  This provides
2247           // a little hysteresis, delaying a recompile until a trap happens
2248           // several times.
2249           //
2250           // Actually, since there is only one bit of counter per BCI,
2251           // the possible per-BCI counts are {0,1,(per-method count)}.
2252           // This produces accurate results if in fact there is only
2253           // one hot trap site, but begins to get fuzzy if there are
2254           // many sites.  For example, if there are ten sites each
2255           // trapping two or more times, they each get the blame for
2256           // all of their traps.
2257           make_not_entrant = true;
2258         }
2259 
2260         // Detect repeated recompilation at the same BCI, and enforce a limit.
2261         if (make_not_entrant && maybe_prior_recompile) {
2262           // More than one recompile at this point.
2263           inc_recompile_count = maybe_prior_trap;
2264         }
2265       } else {
2266         // For reasons which are not recorded per-bytecode, we simply
2267         // force recompiles unconditionally.
2268         // (Note that PerMethodRecompilationCutoff is enforced elsewhere.)
2269         make_not_entrant = true;
2270       }
2271 
2272       // Go back to the compiler if there are too many traps in this method.
2273       if (this_trap_count >= per_method_trap_limit(reason)) {
2274         // If there are too many traps in this method, force a recompile.
2275         // This will allow the compiler to see the limit overflow, and
2276         // take corrective action, if possible.
2277         // (This condition is an unlikely backstop only, because the
2278         // PerBytecodeTrapLimit is more likely to take effect first,
2279         // if it is applicable.)
2280         make_not_entrant = true;
2281       }
2282 
2283       // Here's more hysteresis:  If there has been a recompile at
2284       // this trap point already, run the method in the interpreter
2285       // for a while to exercise it more thoroughly.
2286       if (make_not_entrant && maybe_prior_recompile && maybe_prior_trap) {
2287         reprofile = true;
2288       }
2289     }
2290 
2291     // Take requested actions on the method:
2292 
2293     // Recompile
2294     if (make_not_entrant) {
2295       if (!nm->make_not_entrant()) {
2296         return; // the call did not change nmethod's state
2297       }
2298 
2299       if (pdata != NULL) {
2300         // Record the recompilation event, if any.
2301         int tstate0 = pdata->trap_state();
2302         int tstate1 = trap_state_set_recompiled(tstate0, true);
2303         if (tstate1 != tstate0)
2304           pdata->set_trap_state(tstate1);
2305       }
2306 
2307 #if INCLUDE_RTM_OPT
2308       // Restart collecting RTM locking abort statistic if the method
2309       // is recompiled for a reason other than RTM state change.
2310       // Assume that in new recompiled code the statistic could be different,
2311       // for example, due to different inlining.
2312       if ((reason != Reason_rtm_state_change) && (trap_mdo != NULL) &&
2313           UseRTMDeopt && (nm->as_nmethod()->rtm_state() != ProfileRTM)) {
2314         trap_mdo->atomic_set_rtm_state(ProfileRTM);
2315       }
2316 #endif
2317       // For code aging we count traps separately here, using make_not_entrant()
2318       // as a guard against simultaneous deopts in multiple threads.
2319       if (reason == Reason_tenured && trap_mdo != NULL) {
2320         trap_mdo->inc_tenure_traps();
2321       }
2322     }
2323 
2324     if (inc_recompile_count) {
2325       trap_mdo->inc_overflow_recompile_count();
2326       if ((uint)trap_mdo->overflow_recompile_count() >
2327           (uint)PerBytecodeRecompilationCutoff) {
2328         // Give up on the method containing the bad BCI.
2329         if (trap_method() == nm->method()) {
2330           make_not_compilable = true;
2331         } else {
2332           trap_method->set_not_compilable("overflow_recompile_count > PerBytecodeRecompilationCutoff", CompLevel_full_optimization);
2333           // But give grace to the enclosing nm->method().
2334         }
2335       }
2336     }
2337 
2338     // Reprofile
2339     if (reprofile) {
2340       CompilationPolicy::reprofile(trap_scope, nm->is_osr_method());
2341     }
2342 
2343     // Give up compiling
2344     if (make_not_compilable && !nm->method()->is_not_compilable(CompLevel_full_optimization)) {
2345       assert(make_not_entrant, "consistent");
2346       nm->method()->set_not_compilable("give up compiling", CompLevel_full_optimization);
2347     }
2348 
2349   } // Free marked resources
2350 
2351 }
2352 JRT_END
2353 
2354 ProfileData*
2355 Deoptimization::query_update_method_data(MethodData* trap_mdo,
2356                                          int trap_bci,
2357                                          Deoptimization::DeoptReason reason,
2358                                          bool update_total_trap_count,
2359 #if INCLUDE_JVMCI
2360                                          bool is_osr,
2361 #endif
2362                                          Method* compiled_method,
2363                                          //outputs:
2364                                          uint& ret_this_trap_count,
2365                                          bool& ret_maybe_prior_trap,
2366                                          bool& ret_maybe_prior_recompile) {
2367   bool maybe_prior_trap = false;
2368   bool maybe_prior_recompile = false;
2369   uint this_trap_count = 0;
2370   if (update_total_trap_count) {
2371     uint idx = reason;
2372 #if INCLUDE_JVMCI
2373     if (is_osr) {
2374       idx += Reason_LIMIT;
2375     }
2376 #endif
2377     uint prior_trap_count = trap_mdo->trap_count(idx);
2378     this_trap_count  = trap_mdo->inc_trap_count(idx);
2379 
2380     // If the runtime cannot find a place to store trap history,
2381     // it is estimated based on the general condition of the method.
2382     // If the method has ever been recompiled, or has ever incurred
2383     // a trap with the present reason , then this BCI is assumed
2384     // (pessimistically) to be the culprit.
2385     maybe_prior_trap      = (prior_trap_count != 0);
2386     maybe_prior_recompile = (trap_mdo->decompile_count() != 0);
2387   }
2388   ProfileData* pdata = NULL;
2389 
2390 
2391   // For reasons which are recorded per bytecode, we check per-BCI data.
2392   DeoptReason per_bc_reason = reason_recorded_per_bytecode_if_any(reason);
2393   assert(per_bc_reason != Reason_none || update_total_trap_count, "must be");
2394   if (per_bc_reason != Reason_none) {
2395     // Find the profile data for this BCI.  If there isn't one,
2396     // try to allocate one from the MDO's set of spares.
2397     // This will let us detect a repeated trap at this point.
2398     pdata = trap_mdo->allocate_bci_to_data(trap_bci, reason_is_speculate(reason) ? compiled_method : NULL);
2399 
2400     if (pdata != NULL) {
2401       if (reason_is_speculate(reason) && !pdata->is_SpeculativeTrapData()) {
2402         if (LogCompilation && xtty != NULL) {
2403           ttyLocker ttyl;
2404           // no more room for speculative traps in this MDO
2405           xtty->elem("speculative_traps_oom");
2406         }
2407       }
2408       // Query the trap state of this profile datum.
2409       int tstate0 = pdata->trap_state();
2410       if (!trap_state_has_reason(tstate0, per_bc_reason))
2411         maybe_prior_trap = false;
2412       if (!trap_state_is_recompiled(tstate0))
2413         maybe_prior_recompile = false;
2414 
2415       // Update the trap state of this profile datum.
2416       int tstate1 = tstate0;
2417       // Record the reason.
2418       tstate1 = trap_state_add_reason(tstate1, per_bc_reason);
2419       // Store the updated state on the MDO, for next time.
2420       if (tstate1 != tstate0)
2421         pdata->set_trap_state(tstate1);
2422     } else {
2423       if (LogCompilation && xtty != NULL) {
2424         ttyLocker ttyl;
2425         // Missing MDP?  Leave a small complaint in the log.
2426         xtty->elem("missing_mdp bci='%d'", trap_bci);
2427       }
2428     }
2429   }
2430 
2431   // Return results:
2432   ret_this_trap_count = this_trap_count;
2433   ret_maybe_prior_trap = maybe_prior_trap;
2434   ret_maybe_prior_recompile = maybe_prior_recompile;
2435   return pdata;
2436 }
2437 
2438 void
2439 Deoptimization::update_method_data_from_interpreter(MethodData* trap_mdo, int trap_bci, int reason) {
2440   ResourceMark rm;
2441   // Ignored outputs:
2442   uint ignore_this_trap_count;
2443   bool ignore_maybe_prior_trap;
2444   bool ignore_maybe_prior_recompile;
2445   assert(!reason_is_speculate(reason), "reason speculate only used by compiler");
2446   // JVMCI uses the total counts to determine if deoptimizations are happening too frequently -> do not adjust total counts
2447   bool update_total_counts = true JVMCI_ONLY( && !UseJVMCICompiler);
2448   query_update_method_data(trap_mdo, trap_bci,
2449                            (DeoptReason)reason,
2450                            update_total_counts,
2451 #if INCLUDE_JVMCI
2452                            false,
2453 #endif
2454                            NULL,
2455                            ignore_this_trap_count,
2456                            ignore_maybe_prior_trap,
2457                            ignore_maybe_prior_recompile);
2458 }
2459 
2460 Deoptimization::UnrollBlock* Deoptimization::uncommon_trap(JavaThread* current, jint trap_request, jint exec_mode) {
2461   // Enable WXWrite: current function is called from methods compiled by C2 directly
2462   MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, current));
2463 
2464   if (TraceDeoptimization) {
2465     tty->print("Uncommon trap ");
2466   }
2467   // Still in Java no safepoints
2468   {
2469     // This enters VM and may safepoint
2470     uncommon_trap_inner(current, trap_request);
2471   }
2472   HandleMark hm(current);
2473   return fetch_unroll_info_helper(current, exec_mode);
2474 }
2475 
2476 // Local derived constants.
2477 // Further breakdown of DataLayout::trap_state, as promised by DataLayout.
2478 const int DS_REASON_MASK   = ((uint)DataLayout::trap_mask) >> 1;
2479 const int DS_RECOMPILE_BIT = DataLayout::trap_mask - DS_REASON_MASK;
2480 
2481 //---------------------------trap_state_reason---------------------------------
2482 Deoptimization::DeoptReason
2483 Deoptimization::trap_state_reason(int trap_state) {
2484   // This assert provides the link between the width of DataLayout::trap_bits
2485   // and the encoding of "recorded" reasons.  It ensures there are enough
2486   // bits to store all needed reasons in the per-BCI MDO profile.
2487   assert(DS_REASON_MASK >= Reason_RECORDED_LIMIT, "enough bits");
2488   int recompile_bit = (trap_state & DS_RECOMPILE_BIT);
2489   trap_state -= recompile_bit;
2490   if (trap_state == DS_REASON_MASK) {
2491     return Reason_many;
2492   } else {
2493     assert((int)Reason_none == 0, "state=0 => Reason_none");
2494     return (DeoptReason)trap_state;
2495   }
2496 }
2497 //-------------------------trap_state_has_reason-------------------------------
2498 int Deoptimization::trap_state_has_reason(int trap_state, int reason) {
2499   assert(reason_is_recorded_per_bytecode((DeoptReason)reason), "valid reason");
2500   assert(DS_REASON_MASK >= Reason_RECORDED_LIMIT, "enough bits");
2501   int recompile_bit = (trap_state & DS_RECOMPILE_BIT);
2502   trap_state -= recompile_bit;
2503   if (trap_state == DS_REASON_MASK) {
2504     return -1;  // true, unspecifically (bottom of state lattice)
2505   } else if (trap_state == reason) {
2506     return 1;   // true, definitely
2507   } else if (trap_state == 0) {
2508     return 0;   // false, definitely (top of state lattice)
2509   } else {
2510     return 0;   // false, definitely
2511   }
2512 }
2513 //-------------------------trap_state_add_reason-------------------------------
2514 int Deoptimization::trap_state_add_reason(int trap_state, int reason) {
2515   assert(reason_is_recorded_per_bytecode((DeoptReason)reason) || reason == Reason_many, "valid reason");
2516   int recompile_bit = (trap_state & DS_RECOMPILE_BIT);
2517   trap_state -= recompile_bit;
2518   if (trap_state == DS_REASON_MASK) {
2519     return trap_state + recompile_bit;     // already at state lattice bottom
2520   } else if (trap_state == reason) {
2521     return trap_state + recompile_bit;     // the condition is already true
2522   } else if (trap_state == 0) {
2523     return reason + recompile_bit;          // no condition has yet been true
2524   } else {
2525     return DS_REASON_MASK + recompile_bit;  // fall to state lattice bottom
2526   }
2527 }
2528 //-----------------------trap_state_is_recompiled------------------------------
2529 bool Deoptimization::trap_state_is_recompiled(int trap_state) {
2530   return (trap_state & DS_RECOMPILE_BIT) != 0;
2531 }
2532 //-----------------------trap_state_set_recompiled-----------------------------
2533 int Deoptimization::trap_state_set_recompiled(int trap_state, bool z) {
2534   if (z)  return trap_state |  DS_RECOMPILE_BIT;
2535   else    return trap_state & ~DS_RECOMPILE_BIT;
2536 }
2537 //---------------------------format_trap_state---------------------------------
2538 // This is used for debugging and diagnostics, including LogFile output.
2539 const char* Deoptimization::format_trap_state(char* buf, size_t buflen,
2540                                               int trap_state) {
2541   assert(buflen > 0, "sanity");
2542   DeoptReason reason      = trap_state_reason(trap_state);
2543   bool        recomp_flag = trap_state_is_recompiled(trap_state);
2544   // Re-encode the state from its decoded components.
2545   int decoded_state = 0;
2546   if (reason_is_recorded_per_bytecode(reason) || reason == Reason_many)
2547     decoded_state = trap_state_add_reason(decoded_state, reason);
2548   if (recomp_flag)
2549     decoded_state = trap_state_set_recompiled(decoded_state, recomp_flag);
2550   // If the state re-encodes properly, format it symbolically.
2551   // Because this routine is used for debugging and diagnostics,
2552   // be robust even if the state is a strange value.
2553   size_t len;
2554   if (decoded_state != trap_state) {
2555     // Random buggy state that doesn't decode??
2556     len = jio_snprintf(buf, buflen, "#%d", trap_state);
2557   } else {
2558     len = jio_snprintf(buf, buflen, "%s%s",
2559                        trap_reason_name(reason),
2560                        recomp_flag ? " recompiled" : "");
2561   }
2562   return buf;
2563 }
2564 
2565 
2566 //--------------------------------statics--------------------------------------
2567 const char* Deoptimization::_trap_reason_name[] = {
2568   // Note:  Keep this in sync. with enum DeoptReason.
2569   "none",
2570   "null_check",
2571   "null_assert" JVMCI_ONLY("_or_unreached0"),
2572   "range_check",
2573   "class_check",
2574   "array_check",
2575   "intrinsic" JVMCI_ONLY("_or_type_checked_inlining"),
2576   "bimorphic" JVMCI_ONLY("_or_optimized_type_check"),
2577   "profile_predicate",
2578   "unloaded",
2579   "uninitialized",
2580   "initialized",
2581   "unreached",
2582   "unhandled",
2583   "constraint",
2584   "div0_check",
2585   "age",
2586   "predicate",
2587   "loop_limit_check",
2588   "speculate_class_check",
2589   "speculate_null_check",
2590   "speculate_null_assert",
2591   "rtm_state_change",
2592   "unstable_if",
2593   "unstable_fused_if",
2594   "receiver_constraint",
2595 #if INCLUDE_JVMCI
2596   "aliasing",
2597   "transfer_to_interpreter",
2598   "not_compiled_exception_handler",
2599   "unresolved",
2600   "jsr_mismatch",
2601 #endif
2602   "tenured"
2603 };
2604 const char* Deoptimization::_trap_action_name[] = {
2605   // Note:  Keep this in sync. with enum DeoptAction.
2606   "none",
2607   "maybe_recompile",
2608   "reinterpret",
2609   "make_not_entrant",
2610   "make_not_compilable"
2611 };
2612 
2613 const char* Deoptimization::trap_reason_name(int reason) {
2614   // Check that every reason has a name
2615   STATIC_ASSERT(sizeof(_trap_reason_name)/sizeof(const char*) == Reason_LIMIT);
2616 
2617   if (reason == Reason_many)  return "many";
2618   if ((uint)reason < Reason_LIMIT)
2619     return _trap_reason_name[reason];
2620   static char buf[20];
2621   sprintf(buf, "reason%d", reason);
2622   return buf;
2623 }
2624 const char* Deoptimization::trap_action_name(int action) {
2625   // Check that every action has a name
2626   STATIC_ASSERT(sizeof(_trap_action_name)/sizeof(const char*) == Action_LIMIT);
2627 
2628   if ((uint)action < Action_LIMIT)
2629     return _trap_action_name[action];
2630   static char buf[20];
2631   sprintf(buf, "action%d", action);
2632   return buf;
2633 }
2634 
2635 // This is used for debugging and diagnostics, including LogFile output.
2636 const char* Deoptimization::format_trap_request(char* buf, size_t buflen,
2637                                                 int trap_request) {
2638   jint unloaded_class_index = trap_request_index(trap_request);
2639   const char* reason = trap_reason_name(trap_request_reason(trap_request));
2640   const char* action = trap_action_name(trap_request_action(trap_request));
2641 #if INCLUDE_JVMCI
2642   int debug_id = trap_request_debug_id(trap_request);
2643 #endif
2644   size_t len;
2645   if (unloaded_class_index < 0) {
2646     len = jio_snprintf(buf, buflen, "reason='%s' action='%s'" JVMCI_ONLY(" debug_id='%d'"),
2647                        reason, action
2648 #if INCLUDE_JVMCI
2649                        ,debug_id
2650 #endif
2651                        );
2652   } else {
2653     len = jio_snprintf(buf, buflen, "reason='%s' action='%s' index='%d'" JVMCI_ONLY(" debug_id='%d'"),
2654                        reason, action, unloaded_class_index
2655 #if INCLUDE_JVMCI
2656                        ,debug_id
2657 #endif
2658                        );
2659   }
2660   return buf;
2661 }
2662 
2663 juint Deoptimization::_deoptimization_hist
2664         [Deoptimization::Reason_LIMIT]
2665     [1 + Deoptimization::Action_LIMIT]
2666         [Deoptimization::BC_CASE_LIMIT]
2667   = {0};
2668 
2669 enum {
2670   LSB_BITS = 8,
2671   LSB_MASK = right_n_bits(LSB_BITS)
2672 };
2673 
2674 void Deoptimization::gather_statistics(DeoptReason reason, DeoptAction action,
2675                                        Bytecodes::Code bc) {
2676   assert(reason >= 0 && reason < Reason_LIMIT, "oob");
2677   assert(action >= 0 && action < Action_LIMIT, "oob");
2678   _deoptimization_hist[Reason_none][0][0] += 1;  // total
2679   _deoptimization_hist[reason][0][0]      += 1;  // per-reason total
2680   juint* cases = _deoptimization_hist[reason][1+action];
2681   juint* bc_counter_addr = NULL;
2682   juint  bc_counter      = 0;
2683   // Look for an unused counter, or an exact match to this BC.
2684   if (bc != Bytecodes::_illegal) {
2685     for (int bc_case = 0; bc_case < BC_CASE_LIMIT; bc_case++) {
2686       juint* counter_addr = &cases[bc_case];
2687       juint  counter = *counter_addr;
2688       if ((counter == 0 && bc_counter_addr == NULL)
2689           || (Bytecodes::Code)(counter & LSB_MASK) == bc) {
2690         // this counter is either free or is already devoted to this BC
2691         bc_counter_addr = counter_addr;
2692         bc_counter = counter | bc;
2693       }
2694     }
2695   }
2696   if (bc_counter_addr == NULL) {
2697     // Overflow, or no given bytecode.
2698     bc_counter_addr = &cases[BC_CASE_LIMIT-1];
2699     bc_counter = (*bc_counter_addr & ~LSB_MASK);  // clear LSB
2700   }
2701   *bc_counter_addr = bc_counter + (1 << LSB_BITS);
2702 }
2703 
2704 jint Deoptimization::total_deoptimization_count() {
2705   return _deoptimization_hist[Reason_none][0][0];
2706 }
2707 
2708 void Deoptimization::print_statistics() {
2709   juint total = total_deoptimization_count();
2710   juint account = total;
2711   if (total != 0) {
2712     ttyLocker ttyl;
2713     if (xtty != NULL)  xtty->head("statistics type='deoptimization'");
2714     tty->print_cr("Deoptimization traps recorded:");
2715     #define PRINT_STAT_LINE(name, r) \
2716       tty->print_cr("  %4d (%4.1f%%) %s", (int)(r), ((r) * 100.0) / total, name);
2717     PRINT_STAT_LINE("total", total);
2718     // For each non-zero entry in the histogram, print the reason,
2719     // the action, and (if specifically known) the type of bytecode.
2720     for (int reason = 0; reason < Reason_LIMIT; reason++) {
2721       for (int action = 0; action < Action_LIMIT; action++) {
2722         juint* cases = _deoptimization_hist[reason][1+action];
2723         for (int bc_case = 0; bc_case < BC_CASE_LIMIT; bc_case++) {
2724           juint counter = cases[bc_case];
2725           if (counter != 0) {
2726             char name[1*K];
2727             Bytecodes::Code bc = (Bytecodes::Code)(counter & LSB_MASK);
2728             if (bc_case == BC_CASE_LIMIT && (int)bc == 0)
2729               bc = Bytecodes::_illegal;
2730             sprintf(name, "%s/%s/%s",
2731                     trap_reason_name(reason),
2732                     trap_action_name(action),
2733                     Bytecodes::is_defined(bc)? Bytecodes::name(bc): "other");
2734             juint r = counter >> LSB_BITS;
2735             tty->print_cr("  %40s: " UINT32_FORMAT " (%.1f%%)", name, r, (r * 100.0) / total);
2736             account -= r;
2737           }
2738         }
2739       }
2740     }
2741     if (account != 0) {
2742       PRINT_STAT_LINE("unaccounted", account);
2743     }
2744     #undef PRINT_STAT_LINE
2745     if (xtty != NULL)  xtty->tail("statistics");
2746   }
2747 }
2748 
2749 #else // COMPILER2_OR_JVMCI
2750 
2751 
2752 // Stubs for C1 only system.
2753 bool Deoptimization::trap_state_is_recompiled(int trap_state) {
2754   return false;
2755 }
2756 
2757 const char* Deoptimization::trap_reason_name(int reason) {
2758   return "unknown";
2759 }
2760 
2761 void Deoptimization::print_statistics() {
2762   // no output
2763 }
2764 
2765 void
2766 Deoptimization::update_method_data_from_interpreter(MethodData* trap_mdo, int trap_bci, int reason) {
2767   // no udpate
2768 }
2769 
2770 int Deoptimization::trap_state_has_reason(int trap_state, int reason) {
2771   return 0;
2772 }
2773 
2774 void Deoptimization::gather_statistics(DeoptReason reason, DeoptAction action,
2775                                        Bytecodes::Code bc) {
2776   // no update
2777 }
2778 
2779 const char* Deoptimization::format_trap_state(char* buf, size_t buflen,
2780                                               int trap_state) {
2781   jio_snprintf(buf, buflen, "#%d", trap_state);
2782   return buf;
2783 }
2784 
2785 #endif // COMPILER2_OR_JVMCI
--- EOF ---