1 /*
   2  * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.inline.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "code/compiledIC.hpp"
  29 #include "code/debugInfo.hpp"
  30 #include "code/debugInfoRec.hpp"
  31 #include "compiler/compileBroker.hpp"
  32 #include "compiler/compilerDirectives.hpp"
  33 #include "compiler/disassembler.hpp"
  34 #include "compiler/oopMap.hpp"
  35 #include "gc/shared/barrierSet.hpp"
  36 #include "gc/shared/c2/barrierSetC2.hpp"
  37 #include "memory/allocation.inline.hpp"
  38 #include "memory/allocation.hpp"
  39 #include "opto/ad.hpp"
  40 #include "opto/block.hpp"
  41 #include "opto/c2compiler.hpp"
  42 #include "opto/callnode.hpp"
  43 #include "opto/cfgnode.hpp"
  44 #include "opto/locknode.hpp"
  45 #include "opto/machnode.hpp"
  46 #include "opto/node.hpp"
  47 #include "opto/optoreg.hpp"
  48 #include "opto/output.hpp"
  49 #include "opto/regalloc.hpp"
  50 #include "opto/runtime.hpp"
  51 #include "opto/subnode.hpp"
  52 #include "opto/type.hpp"
  53 #include "runtime/handles.inline.hpp"
  54 #include "runtime/sharedRuntime.hpp"
  55 #include "utilities/macros.hpp"
  56 #include "utilities/powerOfTwo.hpp"
  57 #include "utilities/xmlstream.hpp"
  58 
  59 #ifndef PRODUCT
  60 #define DEBUG_ARG(x) , x
  61 #else
  62 #define DEBUG_ARG(x)
  63 #endif
  64 
  65 //------------------------------Scheduling----------------------------------
  66 // This class contains all the information necessary to implement instruction
  67 // scheduling and bundling.
  68 class Scheduling {
  69 
  70 private:
  71   // Arena to use
  72   Arena *_arena;
  73 
  74   // Control-Flow Graph info
  75   PhaseCFG *_cfg;
  76 
  77   // Register Allocation info
  78   PhaseRegAlloc *_regalloc;
  79 
  80   // Number of nodes in the method
  81   uint _node_bundling_limit;
  82 
  83   // List of scheduled nodes. Generated in reverse order
  84   Node_List _scheduled;
  85 
  86   // List of nodes currently available for choosing for scheduling
  87   Node_List _available;
  88 
  89   // For each instruction beginning a bundle, the number of following
  90   // nodes to be bundled with it.
  91   Bundle *_node_bundling_base;
  92 
  93   // Mapping from register to Node
  94   Node_List _reg_node;
  95 
  96   // Free list for pinch nodes.
  97   Node_List _pinch_free_list;
  98 
  99   // Latency from the beginning of the containing basic block (base 1)
 100   // for each node.
 101   unsigned short *_node_latency;
 102 
 103   // Number of uses of this node within the containing basic block.
 104   short *_uses;
 105 
 106   // Schedulable portion of current block.  Skips Region/Phi/CreateEx up
 107   // front, branch+proj at end.  Also skips Catch/CProj (same as
 108   // branch-at-end), plus just-prior exception-throwing call.
 109   uint _bb_start, _bb_end;
 110 
 111   // Latency from the end of the basic block as scheduled
 112   unsigned short *_current_latency;
 113 
 114   // Remember the next node
 115   Node *_next_node;
 116 
 117   // Use this for an unconditional branch delay slot
 118   Node *_unconditional_delay_slot;
 119 
 120   // Pointer to a Nop
 121   MachNopNode *_nop;
 122 
 123   // Length of the current bundle, in instructions
 124   uint _bundle_instr_count;
 125 
 126   // Current Cycle number, for computing latencies and bundling
 127   uint _bundle_cycle_number;
 128 
 129   // Bundle information
 130   Pipeline_Use_Element _bundle_use_elements[resource_count];
 131   Pipeline_Use         _bundle_use;
 132 
 133   // Dump the available list
 134   void dump_available() const;
 135 
 136 public:
 137   Scheduling(Arena *arena, Compile &compile);
 138 
 139   // Destructor
 140   NOT_PRODUCT( ~Scheduling(); )
 141 
 142   // Step ahead "i" cycles
 143   void step(uint i);
 144 
 145   // Step ahead 1 cycle, and clear the bundle state (for example,
 146   // at a branch target)
 147   void step_and_clear();
 148 
 149   Bundle* node_bundling(const Node *n) {
 150     assert(valid_bundle_info(n), "oob");
 151     return (&_node_bundling_base[n->_idx]);
 152   }
 153 
 154   bool valid_bundle_info(const Node *n) const {
 155     return (_node_bundling_limit > n->_idx);
 156   }
 157 
 158   bool starts_bundle(const Node *n) const {
 159     return (_node_bundling_limit > n->_idx && _node_bundling_base[n->_idx].starts_bundle());
 160   }
 161 
 162   // Do the scheduling
 163   void DoScheduling();
 164 
 165   // Compute the local latencies walking forward over the list of
 166   // nodes for a basic block
 167   void ComputeLocalLatenciesForward(const Block *bb);
 168 
 169   // Compute the register antidependencies within a basic block
 170   void ComputeRegisterAntidependencies(Block *bb);
 171   void verify_do_def( Node *n, OptoReg::Name def, const char *msg );
 172   void verify_good_schedule( Block *b, const char *msg );
 173   void anti_do_def( Block *b, Node *def, OptoReg::Name def_reg, int is_def );
 174   void anti_do_use( Block *b, Node *use, OptoReg::Name use_reg );
 175 
 176   // Add a node to the current bundle
 177   void AddNodeToBundle(Node *n, const Block *bb);
 178 
 179   // Add a node to the list of available nodes
 180   void AddNodeToAvailableList(Node *n);
 181 
 182   // Compute the local use count for the nodes in a block, and compute
 183   // the list of instructions with no uses in the block as available
 184   void ComputeUseCount(const Block *bb);
 185 
 186   // Choose an instruction from the available list to add to the bundle
 187   Node * ChooseNodeToBundle();
 188 
 189   // See if this Node fits into the currently accumulating bundle
 190   bool NodeFitsInBundle(Node *n);
 191 
 192   // Decrement the use count for a node
 193  void DecrementUseCounts(Node *n, const Block *bb);
 194 
 195   // Garbage collect pinch nodes for reuse by other blocks.
 196   void garbage_collect_pinch_nodes();
 197   // Clean up a pinch node for reuse (helper for above).
 198   void cleanup_pinch( Node *pinch );
 199 
 200   // Information for statistics gathering
 201 #ifndef PRODUCT
 202 private:
 203   // Gather information on size of nops relative to total
 204   uint _branches, _unconditional_delays;
 205 
 206   static uint _total_nop_size, _total_method_size;
 207   static uint _total_branches, _total_unconditional_delays;
 208   static uint _total_instructions_per_bundle[Pipeline::_max_instrs_per_cycle+1];
 209 
 210 public:
 211   static void print_statistics();
 212 
 213   static void increment_instructions_per_bundle(uint i) {
 214     _total_instructions_per_bundle[i]++;
 215   }
 216 
 217   static void increment_nop_size(uint s) {
 218     _total_nop_size += s;
 219   }
 220 
 221   static void increment_method_size(uint s) {
 222     _total_method_size += s;
 223   }
 224 #endif
 225 
 226 };
 227 
 228 volatile int C2SafepointPollStubTable::_stub_size = 0;
 229 
 230 Label& C2SafepointPollStubTable::add_safepoint(uintptr_t safepoint_offset) {
 231   C2SafepointPollStub* entry = new (Compile::current()->comp_arena()) C2SafepointPollStub(safepoint_offset);
 232   _safepoints.append(entry);
 233   return entry->_stub_label;
 234 }
 235 
 236 void C2SafepointPollStubTable::emit(CodeBuffer& cb) {
 237   MacroAssembler masm(&cb);
 238   for (int i = _safepoints.length() - 1; i >= 0; i--) {
 239     // Make sure there is enough space in the code buffer
 240     if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == NULL) {
 241       ciEnv::current()->record_failure("CodeCache is full");
 242       return;
 243     }
 244 
 245     C2SafepointPollStub* entry = _safepoints.at(i);
 246     emit_stub(masm, entry);
 247   }
 248 }
 249 
 250 int C2SafepointPollStubTable::stub_size_lazy() const {
 251   int size = Atomic::load(&_stub_size);
 252 
 253   if (size != 0) {
 254     return size;
 255   }
 256 
 257   Compile* const C = Compile::current();
 258   BufferBlob* const blob = C->output()->scratch_buffer_blob();
 259   CodeBuffer cb(blob->content_begin(), C->output()->scratch_buffer_code_size());
 260   MacroAssembler masm(&cb);
 261   C2SafepointPollStub* entry = _safepoints.at(0);
 262   emit_stub(masm, entry);
 263   size += cb.insts_size();
 264 
 265   Atomic::store(&_stub_size, size);
 266 
 267   return size;
 268 }
 269 
 270 int C2SafepointPollStubTable::estimate_stub_size() const {
 271   if (_safepoints.length() == 0) {
 272     return 0;
 273   }
 274 
 275   int result = stub_size_lazy() * _safepoints.length();
 276 
 277 #ifdef ASSERT
 278   Compile* const C = Compile::current();
 279   BufferBlob* const blob = C->output()->scratch_buffer_blob();
 280   int size = 0;
 281 
 282   for (int i = _safepoints.length() - 1; i >= 0; i--) {
 283     CodeBuffer cb(blob->content_begin(), C->output()->scratch_buffer_code_size());
 284     MacroAssembler masm(&cb);
 285     C2SafepointPollStub* entry = _safepoints.at(i);
 286     emit_stub(masm, entry);
 287     size += cb.insts_size();
 288   }
 289   assert(size == result, "stubs should not have variable size");
 290 #endif
 291 
 292   return result;
 293 }
 294 
 295 PhaseOutput::PhaseOutput()
 296   : Phase(Phase::Output),
 297     _code_buffer("Compile::Fill_buffer"),
 298     _first_block_size(0),
 299     _handler_table(),
 300     _inc_table(),
 301     _oop_map_set(nullptr),
 302     _scratch_buffer_blob(nullptr),
 303     _scratch_locs_memory(nullptr),
 304     _scratch_const_size(-1),
 305     _in_scratch_emit_size(false),
 306     _frame_slots(0),
 307     _code_offsets(),
 308     _node_bundling_limit(0),
 309     _node_bundling_base(nullptr),
 310     _orig_pc_slot(0),
 311     _orig_pc_slot_offset_in_bytes(0),
 312     _buf_sizes(),
 313     _block(nullptr),
 314     _index(0) {
 315   C->set_output(this);
 316   if (C->stub_name() == nullptr) {
 317     _orig_pc_slot = C->fixed_slots() - (sizeof(address) / VMRegImpl::stack_slot_size);
 318   }
 319 }
 320 
 321 PhaseOutput::~PhaseOutput() {
 322   C->set_output(nullptr);
 323   if (_scratch_buffer_blob != nullptr) {
 324     BufferBlob::free(_scratch_buffer_blob);
 325   }
 326 }
 327 
 328 void PhaseOutput::perform_mach_node_analysis() {
 329   // Late barrier analysis must be done after schedule and bundle
 330   // Otherwise liveness based spilling will fail
 331   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 332   bs->late_barrier_analysis();
 333 
 334   pd_perform_mach_node_analysis();
 335 }
 336 
 337 // Convert Nodes to instruction bits and pass off to the VM
 338 void PhaseOutput::Output() {
 339   // RootNode goes
 340   assert( C->cfg()->get_root_block()->number_of_nodes() == 0, "" );
 341 
 342   // The number of new nodes (mostly MachNop) is proportional to
 343   // the number of java calls and inner loops which are aligned.
 344   if ( C->check_node_count((NodeLimitFudgeFactor + C->java_calls()*3 +
 345                             C->inner_loops()*(OptoLoopAlignment-1)),
 346                            "out of nodes before code generation" ) ) {
 347     return;
 348   }
 349   // Make sure I can find the Start Node
 350   Block *entry = C->cfg()->get_block(1);
 351   Block *broot = C->cfg()->get_root_block();
 352 
 353   const StartNode *start = entry->head()->as_Start();
 354 
 355   // Replace StartNode with prolog
 356   MachPrologNode *prolog = new MachPrologNode();
 357   entry->map_node(prolog, 0);
 358   C->cfg()->map_node_to_block(prolog, entry);
 359   C->cfg()->unmap_node_from_block(start); // start is no longer in any block
 360 
 361   // Virtual methods need an unverified entry point
 362 
 363   if( C->is_osr_compilation() ) {
 364     if( PoisonOSREntry ) {
 365       // TODO: Should use a ShouldNotReachHereNode...
 366       C->cfg()->insert( broot, 0, new MachBreakpointNode() );
 367     }
 368   } else {
 369     if( C->method() && !C->method()->flags().is_static() ) {
 370       // Insert unvalidated entry point
 371       C->cfg()->insert( broot, 0, new MachUEPNode() );
 372     }
 373 
 374   }
 375 
 376   // Break before main entry point
 377   if ((C->method() && C->directive()->BreakAtExecuteOption) ||
 378       (OptoBreakpoint && C->is_method_compilation())       ||
 379       (OptoBreakpointOSR && C->is_osr_compilation())       ||
 380       (OptoBreakpointC2R && !C->method())                   ) {
 381     // checking for C->method() means that OptoBreakpoint does not apply to
 382     // runtime stubs or frame converters
 383     C->cfg()->insert( entry, 1, new MachBreakpointNode() );
 384   }
 385 
 386   // Insert epilogs before every return
 387   for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
 388     Block* block = C->cfg()->get_block(i);
 389     if (!block->is_connector() && block->non_connector_successor(0) == C->cfg()->get_root_block()) { // Found a program exit point?
 390       Node* m = block->end();
 391       if (m->is_Mach() && m->as_Mach()->ideal_Opcode() != Op_Halt) {
 392         MachEpilogNode* epilog = new MachEpilogNode(m->as_Mach()->ideal_Opcode() == Op_Return);
 393         block->add_inst(epilog);
 394         C->cfg()->map_node_to_block(epilog, block);
 395       }
 396     }
 397   }
 398 
 399   // Keeper of sizing aspects
 400   _buf_sizes = BufferSizingData();
 401 
 402   // Initialize code buffer
 403   estimate_buffer_size(_buf_sizes._const);
 404   if (C->failing()) return;
 405 
 406   // Pre-compute the length of blocks and replace
 407   // long branches with short if machine supports it.
 408   // Must be done before ScheduleAndBundle due to SPARC delay slots
 409   uint* blk_starts = NEW_RESOURCE_ARRAY(uint, C->cfg()->number_of_blocks() + 1);
 410   blk_starts[0] = 0;
 411   shorten_branches(blk_starts);
 412 
 413   ScheduleAndBundle();
 414   if (C->failing()) {
 415     return;
 416   }
 417 
 418   perform_mach_node_analysis();
 419 
 420   // Complete sizing of codebuffer
 421   CodeBuffer* cb = init_buffer();
 422   if (cb == nullptr || C->failing()) {
 423     return;
 424   }
 425 
 426   BuildOopMaps();
 427 
 428   if (C->failing())  {
 429     return;
 430   }
 431 
 432   fill_buffer(cb, blk_starts);
 433 }
 434 
 435 bool PhaseOutput::need_stack_bang(int frame_size_in_bytes) const {
 436   // Determine if we need to generate a stack overflow check.
 437   // Do it if the method is not a stub function and
 438   // has java calls or has frame size > vm_page_size/8.
 439   // The debug VM checks that deoptimization doesn't trigger an
 440   // unexpected stack overflow (compiled method stack banging should
 441   // guarantee it doesn't happen) so we always need the stack bang in
 442   // a debug VM.
 443   return (C->stub_function() == nullptr &&
 444           (C->has_java_calls() || frame_size_in_bytes > os::vm_page_size()>>3
 445            DEBUG_ONLY(|| true)));
 446 }
 447 
 448 bool PhaseOutput::need_register_stack_bang() const {
 449   // Determine if we need to generate a register stack overflow check.
 450   // This is only used on architectures which have split register
 451   // and memory stacks (ie. IA64).
 452   // Bang if the method is not a stub function and has java calls
 453   return (C->stub_function() == nullptr && C->has_java_calls());
 454 }
 455 
 456 
 457 // Compute the size of first NumberOfLoopInstrToAlign instructions at the top
 458 // of a loop. When aligning a loop we need to provide enough instructions
 459 // in cpu's fetch buffer to feed decoders. The loop alignment could be
 460 // avoided if we have enough instructions in fetch buffer at the head of a loop.
 461 // By default, the size is set to 999999 by Block's constructor so that
 462 // a loop will be aligned if the size is not reset here.
 463 //
 464 // Note: Mach instructions could contain several HW instructions
 465 // so the size is estimated only.
 466 //
 467 void PhaseOutput::compute_loop_first_inst_sizes() {
 468   // The next condition is used to gate the loop alignment optimization.
 469   // Don't aligned a loop if there are enough instructions at the head of a loop
 470   // or alignment padding is larger then MaxLoopPad. By default, MaxLoopPad
 471   // is equal to OptoLoopAlignment-1 except on new Intel cpus, where it is
 472   // equal to 11 bytes which is the largest address NOP instruction.
 473   if (MaxLoopPad < OptoLoopAlignment - 1) {
 474     uint last_block = C->cfg()->number_of_blocks() - 1;
 475     for (uint i = 1; i <= last_block; i++) {
 476       Block* block = C->cfg()->get_block(i);
 477       // Check the first loop's block which requires an alignment.
 478       if (block->loop_alignment() > (uint)relocInfo::addr_unit()) {
 479         uint sum_size = 0;
 480         uint inst_cnt = NumberOfLoopInstrToAlign;
 481         inst_cnt = block->compute_first_inst_size(sum_size, inst_cnt, C->regalloc());
 482 
 483         // Check subsequent fallthrough blocks if the loop's first
 484         // block(s) does not have enough instructions.
 485         Block *nb = block;
 486         while(inst_cnt > 0 &&
 487               i < last_block &&
 488               !C->cfg()->get_block(i + 1)->has_loop_alignment() &&
 489               !nb->has_successor(block)) {
 490           i++;
 491           nb = C->cfg()->get_block(i);
 492           inst_cnt  = nb->compute_first_inst_size(sum_size, inst_cnt, C->regalloc());
 493         } // while( inst_cnt > 0 && i < last_block  )
 494 
 495         block->set_first_inst_size(sum_size);
 496       } // f( b->head()->is_Loop() )
 497     } // for( i <= last_block )
 498   } // if( MaxLoopPad < OptoLoopAlignment-1 )
 499 }
 500 
 501 // The architecture description provides short branch variants for some long
 502 // branch instructions. Replace eligible long branches with short branches.
 503 void PhaseOutput::shorten_branches(uint* blk_starts) {
 504 
 505   Compile::TracePhase tp("shorten branches", &timers[_t_shortenBranches]);
 506 
 507   // Compute size of each block, method size, and relocation information size
 508   uint nblocks  = C->cfg()->number_of_blocks();
 509 
 510   uint*      jmp_offset = NEW_RESOURCE_ARRAY(uint,nblocks);
 511   uint*      jmp_size   = NEW_RESOURCE_ARRAY(uint,nblocks);
 512   int*       jmp_nidx   = NEW_RESOURCE_ARRAY(int ,nblocks);
 513 
 514   // Collect worst case block paddings
 515   int* block_worst_case_pad = NEW_RESOURCE_ARRAY(int, nblocks);
 516   memset(block_worst_case_pad, 0, nblocks * sizeof(int));
 517 
 518   DEBUG_ONLY( uint *jmp_target = NEW_RESOURCE_ARRAY(uint,nblocks); )
 519   DEBUG_ONLY( uint *jmp_rule = NEW_RESOURCE_ARRAY(uint,nblocks); )
 520 
 521   bool has_short_branch_candidate = false;
 522 
 523   // Initialize the sizes to 0
 524   int code_size  = 0;          // Size in bytes of generated code
 525   int stub_size  = 0;          // Size in bytes of all stub entries
 526   // Size in bytes of all relocation entries, including those in local stubs.
 527   // Start with 2-bytes of reloc info for the unvalidated entry point
 528   int reloc_size = 1;          // Number of relocation entries
 529 
 530   // Make three passes.  The first computes pessimistic blk_starts,
 531   // relative jmp_offset and reloc_size information.  The second performs
 532   // short branch substitution using the pessimistic sizing.  The
 533   // third inserts nops where needed.
 534 
 535   // Step one, perform a pessimistic sizing pass.
 536   uint last_call_adr = max_juint;
 537   uint last_avoid_back_to_back_adr = max_juint;
 538   uint nop_size = (new MachNopNode())->size(C->regalloc());
 539   for (uint i = 0; i < nblocks; i++) { // For all blocks
 540     Block* block = C->cfg()->get_block(i);
 541     _block = block;
 542 
 543     // During short branch replacement, we store the relative (to blk_starts)
 544     // offset of jump in jmp_offset, rather than the absolute offset of jump.
 545     // This is so that we do not need to recompute sizes of all nodes when
 546     // we compute correct blk_starts in our next sizing pass.
 547     jmp_offset[i] = 0;
 548     jmp_size[i]   = 0;
 549     jmp_nidx[i]   = -1;
 550     DEBUG_ONLY( jmp_target[i] = 0; )
 551     DEBUG_ONLY( jmp_rule[i]   = 0; )
 552 
 553     // Sum all instruction sizes to compute block size
 554     uint last_inst = block->number_of_nodes();
 555     uint blk_size = 0;
 556     for (uint j = 0; j < last_inst; j++) {
 557       _index = j;
 558       Node* nj = block->get_node(_index);
 559       // Handle machine instruction nodes
 560       if (nj->is_Mach()) {
 561         MachNode* mach = nj->as_Mach();
 562         blk_size += (mach->alignment_required() - 1) * relocInfo::addr_unit(); // assume worst case padding
 563         reloc_size += mach->reloc();
 564         if (mach->is_MachCall()) {
 565           // add size information for trampoline stub
 566           // class CallStubImpl is platform-specific and defined in the *.ad files.
 567           stub_size  += CallStubImpl::size_call_trampoline();
 568           reloc_size += CallStubImpl::reloc_call_trampoline();
 569 
 570           MachCallNode *mcall = mach->as_MachCall();
 571           // This destination address is NOT PC-relative
 572 
 573           mcall->method_set((intptr_t)mcall->entry_point());
 574 
 575           if (mcall->is_MachCallJava() && mcall->as_MachCallJava()->_method) {
 576             stub_size  += CompiledStaticCall::to_interp_stub_size();
 577             reloc_size += CompiledStaticCall::reloc_to_interp_stub();
 578           }
 579         } else if (mach->is_MachSafePoint()) {
 580           // If call/safepoint are adjacent, account for possible
 581           // nop to disambiguate the two safepoints.
 582           // ScheduleAndBundle() can rearrange nodes in a block,
 583           // check for all offsets inside this block.
 584           if (last_call_adr >= blk_starts[i]) {
 585             blk_size += nop_size;
 586           }
 587         }
 588         if (mach->avoid_back_to_back(MachNode::AVOID_BEFORE)) {
 589           // Nop is inserted between "avoid back to back" instructions.
 590           // ScheduleAndBundle() can rearrange nodes in a block,
 591           // check for all offsets inside this block.
 592           if (last_avoid_back_to_back_adr >= blk_starts[i]) {
 593             blk_size += nop_size;
 594           }
 595         }
 596         if (mach->may_be_short_branch()) {
 597           if (!nj->is_MachBranch()) {
 598 #ifndef PRODUCT
 599             nj->dump(3);
 600 #endif
 601             Unimplemented();
 602           }
 603           assert(jmp_nidx[i] == -1, "block should have only one branch");
 604           jmp_offset[i] = blk_size;
 605           jmp_size[i]   = nj->size(C->regalloc());
 606           jmp_nidx[i]   = j;
 607           has_short_branch_candidate = true;
 608         }
 609       }
 610       blk_size += nj->size(C->regalloc());
 611       // Remember end of call offset
 612       if (nj->is_MachCall() && !nj->is_MachCallLeaf()) {
 613         last_call_adr = blk_starts[i]+blk_size;
 614       }
 615       // Remember end of avoid_back_to_back offset
 616       if (nj->is_Mach() && nj->as_Mach()->avoid_back_to_back(MachNode::AVOID_AFTER)) {
 617         last_avoid_back_to_back_adr = blk_starts[i]+blk_size;
 618       }
 619     }
 620 
 621     // When the next block starts a loop, we may insert pad NOP
 622     // instructions.  Since we cannot know our future alignment,
 623     // assume the worst.
 624     if (i < nblocks - 1) {
 625       Block* nb = C->cfg()->get_block(i + 1);
 626       int max_loop_pad = nb->code_alignment()-relocInfo::addr_unit();
 627       if (max_loop_pad > 0) {
 628         assert(is_power_of_2(max_loop_pad+relocInfo::addr_unit()), "");
 629         // Adjust last_call_adr and/or last_avoid_back_to_back_adr.
 630         // If either is the last instruction in this block, bump by
 631         // max_loop_pad in lock-step with blk_size, so sizing
 632         // calculations in subsequent blocks still can conservatively
 633         // detect that it may the last instruction in this block.
 634         if (last_call_adr == blk_starts[i]+blk_size) {
 635           last_call_adr += max_loop_pad;
 636         }
 637         if (last_avoid_back_to_back_adr == blk_starts[i]+blk_size) {
 638           last_avoid_back_to_back_adr += max_loop_pad;
 639         }
 640         blk_size += max_loop_pad;
 641         block_worst_case_pad[i + 1] = max_loop_pad;
 642       }
 643     }
 644 
 645     // Save block size; update total method size
 646     blk_starts[i+1] = blk_starts[i]+blk_size;
 647   }
 648 
 649   // Step two, replace eligible long jumps.
 650   bool progress = true;
 651   uint last_may_be_short_branch_adr = max_juint;
 652   while (has_short_branch_candidate && progress) {
 653     progress = false;
 654     has_short_branch_candidate = false;
 655     int adjust_block_start = 0;
 656     for (uint i = 0; i < nblocks; i++) {
 657       Block* block = C->cfg()->get_block(i);
 658       int idx = jmp_nidx[i];
 659       MachNode* mach = (idx == -1) ? nullptr: block->get_node(idx)->as_Mach();
 660       if (mach != nullptr && mach->may_be_short_branch()) {
 661 #ifdef ASSERT
 662         assert(jmp_size[i] > 0 && mach->is_MachBranch(), "sanity");
 663         int j;
 664         // Find the branch; ignore trailing NOPs.
 665         for (j = block->number_of_nodes()-1; j>=0; j--) {
 666           Node* n = block->get_node(j);
 667           if (!n->is_Mach() || n->as_Mach()->ideal_Opcode() != Op_Con)
 668             break;
 669         }
 670         assert(j >= 0 && j == idx && block->get_node(j) == (Node*)mach, "sanity");
 671 #endif
 672         int br_size = jmp_size[i];
 673         int br_offs = blk_starts[i] + jmp_offset[i];
 674 
 675         // This requires the TRUE branch target be in succs[0]
 676         uint bnum = block->non_connector_successor(0)->_pre_order;
 677         int offset = blk_starts[bnum] - br_offs;
 678         if (bnum > i) { // adjust following block's offset
 679           offset -= adjust_block_start;
 680         }
 681 
 682         // This block can be a loop header, account for the padding
 683         // in the previous block.
 684         int block_padding = block_worst_case_pad[i];
 685         assert(i == 0 || block_padding == 0 || br_offs >= block_padding, "Should have at least a padding on top");
 686         // In the following code a nop could be inserted before
 687         // the branch which will increase the backward distance.
 688         bool needs_padding = ((uint)(br_offs - block_padding) == last_may_be_short_branch_adr);
 689         assert(!needs_padding || jmp_offset[i] == 0, "padding only branches at the beginning of block");
 690 
 691         if (needs_padding && offset <= 0)
 692           offset -= nop_size;
 693 
 694         if (C->matcher()->is_short_branch_offset(mach->rule(), br_size, offset)) {
 695           // We've got a winner.  Replace this branch.
 696           MachNode* replacement = mach->as_MachBranch()->short_branch_version();
 697 
 698           // Update the jmp_size.
 699           int new_size = replacement->size(C->regalloc());
 700           int diff     = br_size - new_size;
 701           assert(diff >= (int)nop_size, "short_branch size should be smaller");
 702           // Conservatively take into account padding between
 703           // avoid_back_to_back branches. Previous branch could be
 704           // converted into avoid_back_to_back branch during next
 705           // rounds.
 706           if (needs_padding && replacement->avoid_back_to_back(MachNode::AVOID_BEFORE)) {
 707             jmp_offset[i] += nop_size;
 708             diff -= nop_size;
 709           }
 710           adjust_block_start += diff;
 711           block->map_node(replacement, idx);
 712           mach->subsume_by(replacement, C);
 713           mach = replacement;
 714           progress = true;
 715 
 716           jmp_size[i] = new_size;
 717           DEBUG_ONLY( jmp_target[i] = bnum; );
 718           DEBUG_ONLY( jmp_rule[i] = mach->rule(); );
 719         } else {
 720           // The jump distance is not short, try again during next iteration.
 721           has_short_branch_candidate = true;
 722         }
 723       } // (mach->may_be_short_branch())
 724       if (mach != nullptr && (mach->may_be_short_branch() ||
 725                            mach->avoid_back_to_back(MachNode::AVOID_AFTER))) {
 726         last_may_be_short_branch_adr = blk_starts[i] + jmp_offset[i] + jmp_size[i];
 727       }
 728       blk_starts[i+1] -= adjust_block_start;
 729     }
 730   }
 731 
 732 #ifdef ASSERT
 733   for (uint i = 0; i < nblocks; i++) { // For all blocks
 734     if (jmp_target[i] != 0) {
 735       int br_size = jmp_size[i];
 736       int offset = blk_starts[jmp_target[i]]-(blk_starts[i] + jmp_offset[i]);
 737       if (!C->matcher()->is_short_branch_offset(jmp_rule[i], br_size, offset)) {
 738         tty->print_cr("target (%d) - jmp_offset(%d) = offset (%d), jump_size(%d), jmp_block B%d, target_block B%d", blk_starts[jmp_target[i]], blk_starts[i] + jmp_offset[i], offset, br_size, i, jmp_target[i]);
 739       }
 740       assert(C->matcher()->is_short_branch_offset(jmp_rule[i], br_size, offset), "Displacement too large for short jmp");
 741     }
 742   }
 743 #endif
 744 
 745   // Step 3, compute the offsets of all blocks, will be done in fill_buffer()
 746   // after ScheduleAndBundle().
 747 
 748   // ------------------
 749   // Compute size for code buffer
 750   code_size = blk_starts[nblocks];
 751 
 752   // Relocation records
 753   reloc_size += 1;              // Relo entry for exception handler
 754 
 755   // Adjust reloc_size to number of record of relocation info
 756   // Min is 2 bytes, max is probably 6 or 8, with a tax up to 25% for
 757   // a relocation index.
 758   // The CodeBuffer will expand the locs array if this estimate is too low.
 759   reloc_size *= 10 / sizeof(relocInfo);
 760 
 761   _buf_sizes._reloc = reloc_size;
 762   _buf_sizes._code  = code_size;
 763   _buf_sizes._stub  = stub_size;
 764 }
 765 
 766 //------------------------------FillLocArray-----------------------------------
 767 // Create a bit of debug info and append it to the array.  The mapping is from
 768 // Java local or expression stack to constant, register or stack-slot.  For
 769 // doubles, insert 2 mappings and return 1 (to tell the caller that the next
 770 // entry has been taken care of and caller should skip it).
 771 static LocationValue *new_loc_value( PhaseRegAlloc *ra, OptoReg::Name regnum, Location::Type l_type ) {
 772   // This should never have accepted Bad before
 773   assert(OptoReg::is_valid(regnum), "location must be valid");
 774   return (OptoReg::is_reg(regnum))
 775          ? new LocationValue(Location::new_reg_loc(l_type, OptoReg::as_VMReg(regnum)) )
 776          : new LocationValue(Location::new_stk_loc(l_type,  ra->reg2offset(regnum)));
 777 }
 778 
 779 
 780 ObjectValue*
 781 PhaseOutput::sv_for_node_id(GrowableArray<ScopeValue*> *objs, int id) {
 782   for (int i = 0; i < objs->length(); i++) {
 783     assert(objs->at(i)->is_object(), "corrupt object cache");
 784     ObjectValue* sv = (ObjectValue*) objs->at(i);
 785     if (sv->id() == id) {
 786       return sv;
 787     }
 788   }
 789   // Otherwise..
 790   return nullptr;
 791 }
 792 
 793 void PhaseOutput::set_sv_for_object_node(GrowableArray<ScopeValue*> *objs,
 794                                      ObjectValue* sv ) {
 795   assert(sv_for_node_id(objs, sv->id()) == nullptr, "Precondition");
 796   objs->append(sv);
 797 }
 798 
 799 
 800 void PhaseOutput::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local,
 801                             GrowableArray<ScopeValue*> *array,
 802                             GrowableArray<ScopeValue*> *objs ) {
 803   assert( local, "use _top instead of null" );
 804   if (array->length() != idx) {
 805     assert(array->length() == idx + 1, "Unexpected array count");
 806     // Old functionality:
 807     //   return
 808     // New functionality:
 809     //   Assert if the local is not top. In product mode let the new node
 810     //   override the old entry.
 811     assert(local == C->top(), "LocArray collision");
 812     if (local == C->top()) {
 813       return;
 814     }
 815     array->pop();
 816   }
 817   const Type *t = local->bottom_type();
 818 
 819   // Is it a safepoint scalar object node?
 820   if (local->is_SafePointScalarObject()) {
 821     SafePointScalarObjectNode* spobj = local->as_SafePointScalarObject();
 822 
 823     ObjectValue* sv = sv_for_node_id(objs, spobj->_idx);
 824     if (sv == nullptr) {
 825       ciKlass* cik = t->is_oopptr()->klass();
 826       assert(cik->is_instance_klass() ||
 827              cik->is_array_klass(), "Not supported allocation.");
 828       ScopeValue* klass_sv = new ConstantOopWriteValue(cik->java_mirror()->constant_encoding());
 829       sv = spobj->is_auto_box() ? new AutoBoxObjectValue(spobj->_idx, klass_sv)
 830                                     : new ObjectValue(spobj->_idx, klass_sv);
 831       set_sv_for_object_node(objs, sv);
 832 
 833       uint first_ind = spobj->first_index(sfpt->jvms());
 834       for (uint i = 0; i < spobj->n_fields(); i++) {
 835         Node* fld_node = sfpt->in(first_ind+i);
 836         (void)FillLocArray(sv->field_values()->length(), sfpt, fld_node, sv->field_values(), objs);
 837       }
 838     }
 839     array->append(sv);
 840     return;
 841   }
 842 
 843   // Grab the register number for the local
 844   OptoReg::Name regnum = C->regalloc()->get_reg_first(local);
 845   if( OptoReg::is_valid(regnum) ) {// Got a register/stack?
 846     // Record the double as two float registers.
 847     // The register mask for such a value always specifies two adjacent
 848     // float registers, with the lower register number even.
 849     // Normally, the allocation of high and low words to these registers
 850     // is irrelevant, because nearly all operations on register pairs
 851     // (e.g., StoreD) treat them as a single unit.
 852     // Here, we assume in addition that the words in these two registers
 853     // stored "naturally" (by operations like StoreD and double stores
 854     // within the interpreter) such that the lower-numbered register
 855     // is written to the lower memory address.  This may seem like
 856     // a machine dependency, but it is not--it is a requirement on
 857     // the author of the <arch>.ad file to ensure that, for every
 858     // even/odd double-register pair to which a double may be allocated,
 859     // the word in the even single-register is stored to the first
 860     // memory word.  (Note that register numbers are completely
 861     // arbitrary, and are not tied to any machine-level encodings.)
 862 #ifdef _LP64
 863     if( t->base() == Type::DoubleBot || t->base() == Type::DoubleCon ) {
 864       array->append(new ConstantIntValue((jint)0));
 865       array->append(new_loc_value( C->regalloc(), regnum, Location::dbl ));
 866     } else if ( t->base() == Type::Long ) {
 867       array->append(new ConstantIntValue((jint)0));
 868       array->append(new_loc_value( C->regalloc(), regnum, Location::lng ));
 869     } else if ( t->base() == Type::RawPtr ) {
 870       // jsr/ret return address which must be restored into a the full
 871       // width 64-bit stack slot.
 872       array->append(new_loc_value( C->regalloc(), regnum, Location::lng ));
 873     }
 874 #else //_LP64
 875     if( t->base() == Type::DoubleBot || t->base() == Type::DoubleCon || t->base() == Type::Long ) {
 876       // Repack the double/long as two jints.
 877       // The convention the interpreter uses is that the second local
 878       // holds the first raw word of the native double representation.
 879       // This is actually reasonable, since locals and stack arrays
 880       // grow downwards in all implementations.
 881       // (If, on some machine, the interpreter's Java locals or stack
 882       // were to grow upwards, the embedded doubles would be word-swapped.)
 883       array->append(new_loc_value( C->regalloc(), OptoReg::add(regnum,1), Location::normal ));
 884       array->append(new_loc_value( C->regalloc(),              regnum   , Location::normal ));
 885     }
 886 #endif //_LP64
 887     else if( (t->base() == Type::FloatBot || t->base() == Type::FloatCon) &&
 888              OptoReg::is_reg(regnum) ) {
 889       array->append(new_loc_value( C->regalloc(), regnum, Matcher::float_in_double()
 890                                                       ? Location::float_in_dbl : Location::normal ));
 891     } else if( t->base() == Type::Int && OptoReg::is_reg(regnum) ) {
 892       array->append(new_loc_value( C->regalloc(), regnum, Matcher::int_in_long
 893                                                       ? Location::int_in_long : Location::normal ));
 894     } else if( t->base() == Type::NarrowOop ) {
 895       array->append(new_loc_value( C->regalloc(), regnum, Location::narrowoop ));
 896     } else if (t->base() == Type::VectorA || t->base() == Type::VectorS ||
 897                t->base() == Type::VectorD || t->base() == Type::VectorX ||
 898                t->base() == Type::VectorY || t->base() == Type::VectorZ) {
 899       array->append(new_loc_value( C->regalloc(), regnum, Location::vector ));
 900     } else {
 901       array->append(new_loc_value( C->regalloc(), regnum, C->regalloc()->is_oop(local) ? Location::oop : Location::normal ));
 902     }
 903     return;
 904   }
 905 
 906   // No register.  It must be constant data.
 907   switch (t->base()) {
 908     case Type::Half:              // Second half of a double
 909       ShouldNotReachHere();       // Caller should skip 2nd halves
 910       break;
 911     case Type::AnyPtr:
 912       array->append(new ConstantOopWriteValue(nullptr));
 913       break;
 914     case Type::AryPtr:
 915     case Type::InstPtr:          // fall through
 916       array->append(new ConstantOopWriteValue(t->isa_oopptr()->const_oop()->constant_encoding()));
 917       break;
 918     case Type::NarrowOop:
 919       if (t == TypeNarrowOop::NULL_PTR) {
 920         array->append(new ConstantOopWriteValue(nullptr));
 921       } else {
 922         array->append(new ConstantOopWriteValue(t->make_ptr()->isa_oopptr()->const_oop()->constant_encoding()));
 923       }
 924       break;
 925     case Type::Int:
 926       array->append(new ConstantIntValue(t->is_int()->get_con()));
 927       break;
 928     case Type::RawPtr:
 929       // A return address (T_ADDRESS).
 930       assert((intptr_t)t->is_ptr()->get_con() < (intptr_t)0x10000, "must be a valid BCI");
 931 #ifdef _LP64
 932       // Must be restored to the full-width 64-bit stack slot.
 933       array->append(new ConstantLongValue(t->is_ptr()->get_con()));
 934 #else
 935       array->append(new ConstantIntValue(t->is_ptr()->get_con()));
 936 #endif
 937       break;
 938     case Type::FloatCon: {
 939       float f = t->is_float_constant()->getf();
 940       array->append(new ConstantIntValue(jint_cast(f)));
 941       break;
 942     }
 943     case Type::DoubleCon: {
 944       jdouble d = t->is_double_constant()->getd();
 945 #ifdef _LP64
 946       array->append(new ConstantIntValue((jint)0));
 947       array->append(new ConstantDoubleValue(d));
 948 #else
 949       // Repack the double as two jints.
 950     // The convention the interpreter uses is that the second local
 951     // holds the first raw word of the native double representation.
 952     // This is actually reasonable, since locals and stack arrays
 953     // grow downwards in all implementations.
 954     // (If, on some machine, the interpreter's Java locals or stack
 955     // were to grow upwards, the embedded doubles would be word-swapped.)
 956     jlong_accessor acc;
 957     acc.long_value = jlong_cast(d);
 958     array->append(new ConstantIntValue(acc.words[1]));
 959     array->append(new ConstantIntValue(acc.words[0]));
 960 #endif
 961       break;
 962     }
 963     case Type::Long: {
 964       jlong d = t->is_long()->get_con();
 965 #ifdef _LP64
 966       array->append(new ConstantIntValue((jint)0));
 967       array->append(new ConstantLongValue(d));
 968 #else
 969       // Repack the long as two jints.
 970     // The convention the interpreter uses is that the second local
 971     // holds the first raw word of the native double representation.
 972     // This is actually reasonable, since locals and stack arrays
 973     // grow downwards in all implementations.
 974     // (If, on some machine, the interpreter's Java locals or stack
 975     // were to grow upwards, the embedded doubles would be word-swapped.)
 976     jlong_accessor acc;
 977     acc.long_value = d;
 978     array->append(new ConstantIntValue(acc.words[1]));
 979     array->append(new ConstantIntValue(acc.words[0]));
 980 #endif
 981       break;
 982     }
 983     case Type::Top:               // Add an illegal value here
 984       array->append(new LocationValue(Location()));
 985       break;
 986     default:
 987       ShouldNotReachHere();
 988       break;
 989   }
 990 }
 991 
 992 // Determine if this node starts a bundle
 993 bool PhaseOutput::starts_bundle(const Node *n) const {
 994   return (_node_bundling_limit > n->_idx &&
 995           _node_bundling_base[n->_idx].starts_bundle());
 996 }
 997 
 998 //--------------------------Process_OopMap_Node--------------------------------
 999 void PhaseOutput::Process_OopMap_Node(MachNode *mach, int current_offset) {
1000   // Handle special safepoint nodes for synchronization
1001   MachSafePointNode *sfn   = mach->as_MachSafePoint();
1002   MachCallNode      *mcall;
1003 
1004   int safepoint_pc_offset = current_offset;
1005   bool is_method_handle_invoke = false;
1006   bool is_opt_native = false;
1007   bool return_oop = false;
1008   bool has_ea_local_in_scope = sfn->_has_ea_local_in_scope;
1009   bool arg_escape = false;
1010 
1011   // Add the safepoint in the DebugInfoRecorder
1012   if( !mach->is_MachCall() ) {
1013     mcall = nullptr;
1014     C->debug_info()->add_safepoint(safepoint_pc_offset, sfn->_oop_map);
1015   } else {
1016     mcall = mach->as_MachCall();
1017 
1018     // Is the call a MethodHandle call?
1019     if (mcall->is_MachCallJava()) {
1020       if (mcall->as_MachCallJava()->_method_handle_invoke) {
1021         assert(C->has_method_handle_invokes(), "must have been set during call generation");
1022         is_method_handle_invoke = true;
1023       }
1024       arg_escape = mcall->as_MachCallJava()->_arg_escape;
1025     } else if (mcall->is_MachCallNative()) {
1026       is_opt_native = true;
1027     }
1028 
1029     // Check if a call returns an object.
1030     if (mcall->returns_pointer()) {
1031       return_oop = true;
1032     }
1033     safepoint_pc_offset += mcall->ret_addr_offset();
1034     C->debug_info()->add_safepoint(safepoint_pc_offset, mcall->_oop_map);
1035   }
1036 
1037   // Loop over the JVMState list to add scope information
1038   // Do not skip safepoints with a null method, they need monitor info
1039   JVMState* youngest_jvms = sfn->jvms();
1040   int max_depth = youngest_jvms->depth();
1041 
1042   // Allocate the object pool for scalar-replaced objects -- the map from
1043   // small-integer keys (which can be recorded in the local and ostack
1044   // arrays) to descriptions of the object state.
1045   GrowableArray<ScopeValue*> *objs = new GrowableArray<ScopeValue*>();
1046 
1047   // Visit scopes from oldest to youngest.
1048   for (int depth = 1; depth <= max_depth; depth++) {
1049     JVMState* jvms = youngest_jvms->of_depth(depth);
1050     int idx;
1051     ciMethod* method = jvms->has_method() ? jvms->method() : nullptr;
1052     // Safepoints that do not have method() set only provide oop-map and monitor info
1053     // to support GC; these do not support deoptimization.
1054     int num_locs = (method == nullptr) ? 0 : jvms->loc_size();
1055     int num_exps = (method == nullptr) ? 0 : jvms->stk_size();
1056     int num_mon  = jvms->nof_monitors();
1057     assert(method == nullptr || jvms->bci() < 0 || num_locs == method->max_locals(),
1058            "JVMS local count must match that of the method");
1059 
1060     // Add Local and Expression Stack Information
1061 
1062     // Insert locals into the locarray
1063     GrowableArray<ScopeValue*> *locarray = new GrowableArray<ScopeValue*>(num_locs);
1064     for( idx = 0; idx < num_locs; idx++ ) {
1065       FillLocArray( idx, sfn, sfn->local(jvms, idx), locarray, objs );
1066     }
1067 
1068     // Insert expression stack entries into the exparray
1069     GrowableArray<ScopeValue*> *exparray = new GrowableArray<ScopeValue*>(num_exps);
1070     for( idx = 0; idx < num_exps; idx++ ) {
1071       FillLocArray( idx,  sfn, sfn->stack(jvms, idx), exparray, objs );
1072     }
1073 
1074     // Add in mappings of the monitors
1075     assert( !method ||
1076             !method->is_synchronized() ||
1077             method->is_native() ||
1078             num_mon > 0 ||
1079             !GenerateSynchronizationCode,
1080             "monitors must always exist for synchronized methods");
1081 
1082     // Build the growable array of ScopeValues for exp stack
1083     GrowableArray<MonitorValue*> *monarray = new GrowableArray<MonitorValue*>(num_mon);
1084 
1085     // Loop over monitors and insert into array
1086     for (idx = 0; idx < num_mon; idx++) {
1087       // Grab the node that defines this monitor
1088       Node* box_node = sfn->monitor_box(jvms, idx);
1089       Node* obj_node = sfn->monitor_obj(jvms, idx);
1090 
1091       // Create ScopeValue for object
1092       ScopeValue *scval = nullptr;
1093 
1094       if (obj_node->is_SafePointScalarObject()) {
1095         SafePointScalarObjectNode* spobj = obj_node->as_SafePointScalarObject();
1096         scval = PhaseOutput::sv_for_node_id(objs, spobj->_idx);
1097         if (scval == nullptr) {
1098           const Type *t = spobj->bottom_type();
1099           ciKlass* cik = t->is_oopptr()->klass();
1100           assert(cik->is_instance_klass() ||
1101                  cik->is_array_klass(), "Not supported allocation.");
1102           ScopeValue* klass_sv = new ConstantOopWriteValue(cik->java_mirror()->constant_encoding());
1103           ObjectValue* sv = spobj->is_auto_box() ? new AutoBoxObjectValue(spobj->_idx, klass_sv)
1104                                         : new ObjectValue(spobj->_idx, klass_sv);
1105           PhaseOutput::set_sv_for_object_node(objs, sv);
1106 
1107           uint first_ind = spobj->first_index(youngest_jvms);
1108           for (uint i = 0; i < spobj->n_fields(); i++) {
1109             Node* fld_node = sfn->in(first_ind+i);
1110             (void)FillLocArray(sv->field_values()->length(), sfn, fld_node, sv->field_values(), objs);
1111           }
1112           scval = sv;
1113         }
1114       } else if (!obj_node->is_Con()) {
1115         OptoReg::Name obj_reg = C->regalloc()->get_reg_first(obj_node);
1116         if( obj_node->bottom_type()->base() == Type::NarrowOop ) {
1117           scval = new_loc_value( C->regalloc(), obj_reg, Location::narrowoop );
1118         } else {
1119           scval = new_loc_value( C->regalloc(), obj_reg, Location::oop );
1120         }
1121       } else {
1122         const TypePtr *tp = obj_node->get_ptr_type();
1123         scval = new ConstantOopWriteValue(tp->is_oopptr()->const_oop()->constant_encoding());
1124       }
1125 
1126       OptoReg::Name box_reg = BoxLockNode::reg(box_node);
1127       Location basic_lock = Location::new_stk_loc(Location::normal,C->regalloc()->reg2offset(box_reg));
1128       bool eliminated = (box_node->is_BoxLock() && box_node->as_BoxLock()->is_eliminated());
1129       monarray->append(new MonitorValue(scval, basic_lock, eliminated));
1130     }
1131 
1132     // We dump the object pool first, since deoptimization reads it in first.
1133     C->debug_info()->dump_object_pool(objs);
1134 
1135     // Build first class objects to pass to scope
1136     DebugToken *locvals = C->debug_info()->create_scope_values(locarray);
1137     DebugToken *expvals = C->debug_info()->create_scope_values(exparray);
1138     DebugToken *monvals = C->debug_info()->create_monitor_values(monarray);
1139 
1140     // Make method available for all Safepoints
1141     ciMethod* scope_method = method ? method : C->method();
1142     // Describe the scope here
1143     assert(jvms->bci() >= InvocationEntryBci && jvms->bci() <= 0x10000, "must be a valid or entry BCI");
1144     assert(!jvms->should_reexecute() || depth == max_depth, "reexecute allowed only for the youngest");
1145     // Now we can describe the scope.
1146     methodHandle null_mh;
1147     bool rethrow_exception = false;
1148     C->debug_info()->describe_scope(
1149       safepoint_pc_offset,
1150       null_mh,
1151       scope_method,
1152       jvms->bci(),
1153       jvms->should_reexecute(),
1154       rethrow_exception,
1155       is_method_handle_invoke,
1156       is_opt_native,
1157       return_oop,
1158       has_ea_local_in_scope,
1159       arg_escape,
1160       locvals,
1161       expvals,
1162       monvals
1163     );
1164   } // End jvms loop
1165 
1166   // Mark the end of the scope set.
1167   C->debug_info()->end_safepoint(safepoint_pc_offset);
1168 }
1169 
1170 
1171 
1172 // A simplified version of Process_OopMap_Node, to handle non-safepoints.
1173 class NonSafepointEmitter {
1174     Compile*  C;
1175     JVMState* _pending_jvms;
1176     int       _pending_offset;
1177 
1178     void emit_non_safepoint();
1179 
1180  public:
1181     NonSafepointEmitter(Compile* compile) {
1182       this->C = compile;
1183       _pending_jvms = nullptr;
1184       _pending_offset = 0;
1185     }
1186 
1187     void observe_instruction(Node* n, int pc_offset) {
1188       if (!C->debug_info()->recording_non_safepoints())  return;
1189 
1190       Node_Notes* nn = C->node_notes_at(n->_idx);
1191       if (nn == nullptr || nn->jvms() == nullptr)  return;
1192       if (_pending_jvms != nullptr &&
1193           _pending_jvms->same_calls_as(nn->jvms())) {
1194         // Repeated JVMS?  Stretch it up here.
1195         _pending_offset = pc_offset;
1196       } else {
1197         if (_pending_jvms != nullptr &&
1198             _pending_offset < pc_offset) {
1199           emit_non_safepoint();
1200         }
1201         _pending_jvms = nullptr;
1202         if (pc_offset > C->debug_info()->last_pc_offset()) {
1203           // This is the only way _pending_jvms can become non-null:
1204           _pending_jvms = nn->jvms();
1205           _pending_offset = pc_offset;
1206         }
1207       }
1208     }
1209 
1210     // Stay out of the way of real safepoints:
1211     void observe_safepoint(JVMState* jvms, int pc_offset) {
1212       if (_pending_jvms != nullptr &&
1213           !_pending_jvms->same_calls_as(jvms) &&
1214           _pending_offset < pc_offset) {
1215         emit_non_safepoint();
1216       }
1217       _pending_jvms = nullptr;
1218     }
1219 
1220     void flush_at_end() {
1221       if (_pending_jvms != nullptr) {
1222         emit_non_safepoint();
1223       }
1224       _pending_jvms = nullptr;
1225     }
1226 };
1227 
1228 void NonSafepointEmitter::emit_non_safepoint() {
1229   JVMState* youngest_jvms = _pending_jvms;
1230   int       pc_offset     = _pending_offset;
1231 
1232   // Clear it now:
1233   _pending_jvms = nullptr;
1234 
1235   DebugInformationRecorder* debug_info = C->debug_info();
1236   assert(debug_info->recording_non_safepoints(), "sanity");
1237 
1238   debug_info->add_non_safepoint(pc_offset);
1239   int max_depth = youngest_jvms->depth();
1240 
1241   // Visit scopes from oldest to youngest.
1242   for (int depth = 1; depth <= max_depth; depth++) {
1243     JVMState* jvms = youngest_jvms->of_depth(depth);
1244     ciMethod* method = jvms->has_method() ? jvms->method() : nullptr;
1245     assert(!jvms->should_reexecute() || depth==max_depth, "reexecute allowed only for the youngest");
1246     methodHandle null_mh;
1247     debug_info->describe_scope(pc_offset, null_mh, method, jvms->bci(), jvms->should_reexecute());
1248   }
1249 
1250   // Mark the end of the scope set.
1251   debug_info->end_non_safepoint(pc_offset);
1252 }
1253 
1254 //------------------------------init_buffer------------------------------------
1255 void PhaseOutput::estimate_buffer_size(int& const_req) {
1256 
1257   // Set the initially allocated size
1258   const_req = initial_const_capacity;
1259 
1260   // The extra spacing after the code is necessary on some platforms.
1261   // Sometimes we need to patch in a jump after the last instruction,
1262   // if the nmethod has been deoptimized.  (See 4932387, 4894843.)
1263 
1264   // Compute the byte offset where we can store the deopt pc.
1265   if (C->fixed_slots() != 0) {
1266     _orig_pc_slot_offset_in_bytes = C->regalloc()->reg2offset(OptoReg::stack2reg(_orig_pc_slot));
1267   }
1268 
1269   // Compute prolog code size
1270   _method_size = 0;
1271   _frame_slots = OptoReg::reg2stack(C->matcher()->_old_SP) + C->regalloc()->_framesize;
1272   assert(_frame_slots >= 0 && _frame_slots < 1000000, "sanity check");
1273 
1274   if (C->has_mach_constant_base_node()) {
1275     uint add_size = 0;
1276     // Fill the constant table.
1277     // Note:  This must happen before shorten_branches.
1278     for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
1279       Block* b = C->cfg()->get_block(i);
1280 
1281       for (uint j = 0; j < b->number_of_nodes(); j++) {
1282         Node* n = b->get_node(j);
1283 
1284         // If the node is a MachConstantNode evaluate the constant
1285         // value section.
1286         if (n->is_MachConstant()) {
1287           MachConstantNode* machcon = n->as_MachConstant();
1288           machcon->eval_constant(C);
1289         } else if (n->is_Mach()) {
1290           // On Power there are more nodes that issue constants.
1291           add_size += (n->as_Mach()->ins_num_consts() * 8);
1292         }
1293       }
1294     }
1295 
1296     // Calculate the offsets of the constants and the size of the
1297     // constant table (including the padding to the next section).
1298     constant_table().calculate_offsets_and_size();
1299     const_req = constant_table().size() + add_size;
1300   }
1301 
1302   // Initialize the space for the BufferBlob used to find and verify
1303   // instruction size in MachNode::emit_size()
1304   init_scratch_buffer_blob(const_req);
1305 }
1306 
1307 CodeBuffer* PhaseOutput::init_buffer() {
1308   int stub_req  = _buf_sizes._stub;
1309   int code_req  = _buf_sizes._code;
1310   int const_req = _buf_sizes._const;
1311 
1312   int pad_req   = NativeCall::instruction_size;
1313 
1314   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1315   stub_req += bs->estimate_stub_size();
1316   stub_req += safepoint_poll_table()->estimate_stub_size();
1317 
1318   // nmethod and CodeBuffer count stubs & constants as part of method's code.
1319   // class HandlerImpl is platform-specific and defined in the *.ad files.
1320   int exception_handler_req = HandlerImpl::size_exception_handler() + MAX_stubs_size; // add marginal slop for handler
1321   int deopt_handler_req     = HandlerImpl::size_deopt_handler()     + MAX_stubs_size; // add marginal slop for handler
1322   stub_req += MAX_stubs_size;   // ensure per-stub margin
1323   code_req += MAX_inst_size;    // ensure per-instruction margin
1324 
1325   if (StressCodeBuffers)
1326     code_req = const_req = stub_req = exception_handler_req = deopt_handler_req = 0x10;  // force expansion
1327 
1328   int total_req =
1329           const_req +
1330           code_req +
1331           pad_req +
1332           stub_req +
1333           exception_handler_req +
1334           deopt_handler_req;               // deopt handler
1335 
1336   if (C->has_method_handle_invokes())
1337     total_req += deopt_handler_req;  // deopt MH handler
1338 
1339   CodeBuffer* cb = code_buffer();
1340   cb->initialize(total_req, _buf_sizes._reloc);
1341 
1342   // Have we run out of code space?
1343   if ((cb->blob() == nullptr) || (!CompileBroker::should_compile_new_jobs())) {
1344     C->record_failure("CodeCache is full");
1345     return nullptr;
1346   }
1347   // Configure the code buffer.
1348   cb->initialize_consts_size(const_req);
1349   cb->initialize_stubs_size(stub_req);
1350   cb->initialize_oop_recorder(C->env()->oop_recorder());
1351 
1352   // fill in the nop array for bundling computations
1353   MachNode *_nop_list[Bundle::_nop_count];
1354   Bundle::initialize_nops(_nop_list);
1355 
1356   return cb;
1357 }
1358 
1359 //------------------------------fill_buffer------------------------------------
1360 void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) {
1361   // blk_starts[] contains offsets calculated during short branches processing,
1362   // offsets should not be increased during following steps.
1363 
1364   // Compute the size of first NumberOfLoopInstrToAlign instructions at head
1365   // of a loop. It is used to determine the padding for loop alignment.
1366   Compile::TracePhase tp("fill buffer", &timers[_t_fillBuffer]);
1367 
1368   compute_loop_first_inst_sizes();
1369 
1370   // Create oopmap set.
1371   _oop_map_set = new OopMapSet();
1372 
1373   // !!!!! This preserves old handling of oopmaps for now
1374   C->debug_info()->set_oopmaps(_oop_map_set);
1375 
1376   uint nblocks  = C->cfg()->number_of_blocks();
1377   // Count and start of implicit null check instructions
1378   uint inct_cnt = 0;
1379   uint* inct_starts = NEW_RESOURCE_ARRAY(uint, nblocks+1);
1380 
1381   // Count and start of calls
1382   uint* call_returns = NEW_RESOURCE_ARRAY(uint, nblocks+1);
1383 
1384   uint  return_offset = 0;
1385   int nop_size = (new MachNopNode())->size(C->regalloc());
1386 
1387   int previous_offset = 0;
1388   int current_offset  = 0;
1389   int last_call_offset = -1;
1390   int last_avoid_back_to_back_offset = -1;
1391 #ifdef ASSERT
1392   uint* jmp_target = NEW_RESOURCE_ARRAY(uint,nblocks);
1393   uint* jmp_offset = NEW_RESOURCE_ARRAY(uint,nblocks);
1394   uint* jmp_size   = NEW_RESOURCE_ARRAY(uint,nblocks);
1395   uint* jmp_rule   = NEW_RESOURCE_ARRAY(uint,nblocks);
1396 #endif
1397 
1398   // Create an array of unused labels, one for each basic block, if printing is enabled
1399 #if defined(SUPPORT_OPTO_ASSEMBLY)
1400   int* node_offsets      = nullptr;
1401   uint node_offset_limit = C->unique();
1402 
1403   if (C->print_assembly()) {
1404     node_offsets = NEW_RESOURCE_ARRAY(int, node_offset_limit);
1405   }
1406   if (node_offsets != nullptr) {
1407     // We need to initialize. Unused array elements may contain garbage and mess up PrintOptoAssembly.
1408     memset(node_offsets, 0, node_offset_limit*sizeof(int));
1409   }
1410 #endif
1411 
1412   NonSafepointEmitter non_safepoints(C);  // emit non-safepoints lazily
1413 
1414   // Emit the constant table.
1415   if (C->has_mach_constant_base_node()) {
1416     if (!constant_table().emit(*cb)) {
1417       C->record_failure("consts section overflow");
1418       return;
1419     }
1420   }
1421 
1422   // Create an array of labels, one for each basic block
1423   Label* blk_labels = NEW_RESOURCE_ARRAY(Label, nblocks+1);
1424   for (uint i = 0; i <= nblocks; i++) {
1425     blk_labels[i].init();
1426   }
1427 
1428   // Now fill in the code buffer
1429   Node* delay_slot = nullptr;
1430   for (uint i = 0; i < nblocks; i++) {
1431     Block* block = C->cfg()->get_block(i);
1432     _block = block;
1433     Node* head = block->head();
1434 
1435     // If this block needs to start aligned (i.e, can be reached other
1436     // than by falling-thru from the previous block), then force the
1437     // start of a new bundle.
1438     if (Pipeline::requires_bundling() && starts_bundle(head)) {
1439       cb->flush_bundle(true);
1440     }
1441 
1442 #ifdef ASSERT
1443     if (!block->is_connector()) {
1444       stringStream st;
1445       block->dump_head(C->cfg(), &st);
1446       MacroAssembler(cb).block_comment(st.as_string());
1447     }
1448     jmp_target[i] = 0;
1449     jmp_offset[i] = 0;
1450     jmp_size[i]   = 0;
1451     jmp_rule[i]   = 0;
1452 #endif
1453     int blk_offset = current_offset;
1454 
1455     // Define the label at the beginning of the basic block
1456     MacroAssembler(cb).bind(blk_labels[block->_pre_order]);
1457 
1458     uint last_inst = block->number_of_nodes();
1459 
1460     // Emit block normally, except for last instruction.
1461     // Emit means "dump code bits into code buffer".
1462     for (uint j = 0; j<last_inst; j++) {
1463       _index = j;
1464 
1465       // Get the node
1466       Node* n = block->get_node(j);
1467 
1468       // See if delay slots are supported
1469       if (valid_bundle_info(n) && node_bundling(n)->used_in_unconditional_delay()) {
1470         assert(delay_slot == nullptr, "no use of delay slot node");
1471         assert(n->size(C->regalloc()) == Pipeline::instr_unit_size(), "delay slot instruction wrong size");
1472 
1473         delay_slot = n;
1474         continue;
1475       }
1476 
1477       // If this starts a new instruction group, then flush the current one
1478       // (but allow split bundles)
1479       if (Pipeline::requires_bundling() && starts_bundle(n))
1480         cb->flush_bundle(false);
1481 
1482       // Special handling for SafePoint/Call Nodes
1483       bool is_mcall = false;
1484       if (n->is_Mach()) {
1485         MachNode *mach = n->as_Mach();
1486         is_mcall = n->is_MachCall();
1487         bool is_sfn = n->is_MachSafePoint();
1488 
1489         // If this requires all previous instructions be flushed, then do so
1490         if (is_sfn || is_mcall || mach->alignment_required() != 1) {
1491           cb->flush_bundle(true);
1492           current_offset = cb->insts_size();
1493         }
1494 
1495         // A padding may be needed again since a previous instruction
1496         // could be moved to delay slot.
1497 
1498         // align the instruction if necessary
1499         int padding = mach->compute_padding(current_offset);
1500         // Make sure safepoint node for polling is distinct from a call's
1501         // return by adding a nop if needed.
1502         if (is_sfn && !is_mcall && padding == 0 && current_offset == last_call_offset) {
1503           padding = nop_size;
1504         }
1505         if (padding == 0 && mach->avoid_back_to_back(MachNode::AVOID_BEFORE) &&
1506             current_offset == last_avoid_back_to_back_offset) {
1507           // Avoid back to back some instructions.
1508           padding = nop_size;
1509         }
1510 
1511         if (padding > 0) {
1512           assert((padding % nop_size) == 0, "padding is not a multiple of NOP size");
1513           int nops_cnt = padding / nop_size;
1514           MachNode *nop = new MachNopNode(nops_cnt);
1515           block->insert_node(nop, j++);
1516           last_inst++;
1517           C->cfg()->map_node_to_block(nop, block);
1518           // Ensure enough space.
1519           cb->insts()->maybe_expand_to_ensure_remaining(MAX_inst_size);
1520           if ((cb->blob() == nullptr) || (!CompileBroker::should_compile_new_jobs())) {
1521             C->record_failure("CodeCache is full");
1522             return;
1523           }
1524           nop->emit(*cb, C->regalloc());
1525           cb->flush_bundle(true);
1526           current_offset = cb->insts_size();
1527         }
1528 
1529         bool observe_safepoint = is_sfn;
1530         // Remember the start of the last call in a basic block
1531         if (is_mcall) {
1532           MachCallNode *mcall = mach->as_MachCall();
1533 
1534           // This destination address is NOT PC-relative
1535           mcall->method_set((intptr_t)mcall->entry_point());
1536 
1537           // Save the return address
1538           call_returns[block->_pre_order] = current_offset + mcall->ret_addr_offset();
1539 
1540           observe_safepoint = mcall->guaranteed_safepoint();
1541         }
1542 
1543         // sfn will be valid whenever mcall is valid now because of inheritance
1544         if (observe_safepoint) {
1545           // Handle special safepoint nodes for synchronization
1546           if (!is_mcall) {
1547             MachSafePointNode *sfn = mach->as_MachSafePoint();
1548             // !!!!! Stubs only need an oopmap right now, so bail out
1549             if (sfn->jvms()->method() == nullptr) {
1550               // Write the oopmap directly to the code blob??!!
1551               continue;
1552             }
1553           } // End synchronization
1554 
1555           non_safepoints.observe_safepoint(mach->as_MachSafePoint()->jvms(),
1556                                            current_offset);
1557           Process_OopMap_Node(mach, current_offset);
1558         } // End if safepoint
1559 
1560           // If this is a null check, then add the start of the previous instruction to the list
1561         else if( mach->is_MachNullCheck() ) {
1562           inct_starts[inct_cnt++] = previous_offset;
1563         }
1564 
1565           // If this is a branch, then fill in the label with the target BB's label
1566         else if (mach->is_MachBranch()) {
1567           // This requires the TRUE branch target be in succs[0]
1568           uint block_num = block->non_connector_successor(0)->_pre_order;
1569 
1570           // Try to replace long branch if delay slot is not used,
1571           // it is mostly for back branches since forward branch's
1572           // distance is not updated yet.
1573           bool delay_slot_is_used = valid_bundle_info(n) &&
1574                                     C->output()->node_bundling(n)->use_unconditional_delay();
1575           if (!delay_slot_is_used && mach->may_be_short_branch()) {
1576             assert(delay_slot == nullptr, "not expecting delay slot node");
1577             int br_size = n->size(C->regalloc());
1578             int offset = blk_starts[block_num] - current_offset;
1579             if (block_num >= i) {
1580               // Current and following block's offset are not
1581               // finalized yet, adjust distance by the difference
1582               // between calculated and final offsets of current block.
1583               offset -= (blk_starts[i] - blk_offset);
1584             }
1585             // In the following code a nop could be inserted before
1586             // the branch which will increase the backward distance.
1587             bool needs_padding = (current_offset == last_avoid_back_to_back_offset);
1588             if (needs_padding && offset <= 0)
1589               offset -= nop_size;
1590 
1591             if (C->matcher()->is_short_branch_offset(mach->rule(), br_size, offset)) {
1592               // We've got a winner.  Replace this branch.
1593               MachNode* replacement = mach->as_MachBranch()->short_branch_version();
1594 
1595               // Update the jmp_size.
1596               int new_size = replacement->size(C->regalloc());
1597               assert((br_size - new_size) >= (int)nop_size, "short_branch size should be smaller");
1598               // Insert padding between avoid_back_to_back branches.
1599               if (needs_padding && replacement->avoid_back_to_back(MachNode::AVOID_BEFORE)) {
1600                 MachNode *nop = new MachNopNode();
1601                 block->insert_node(nop, j++);
1602                 C->cfg()->map_node_to_block(nop, block);
1603                 last_inst++;
1604                 nop->emit(*cb, C->regalloc());
1605                 cb->flush_bundle(true);
1606                 current_offset = cb->insts_size();
1607               }
1608 #ifdef ASSERT
1609               jmp_target[i] = block_num;
1610               jmp_offset[i] = current_offset - blk_offset;
1611               jmp_size[i]   = new_size;
1612               jmp_rule[i]   = mach->rule();
1613 #endif
1614               block->map_node(replacement, j);
1615               mach->subsume_by(replacement, C);
1616               n    = replacement;
1617               mach = replacement;
1618             }
1619           }
1620           mach->as_MachBranch()->label_set( &blk_labels[block_num], block_num );
1621         } else if (mach->ideal_Opcode() == Op_Jump) {
1622           for (uint h = 0; h < block->_num_succs; h++) {
1623             Block* succs_block = block->_succs[h];
1624             for (uint j = 1; j < succs_block->num_preds(); j++) {
1625               Node* jpn = succs_block->pred(j);
1626               if (jpn->is_JumpProj() && jpn->in(0) == mach) {
1627                 uint block_num = succs_block->non_connector()->_pre_order;
1628                 Label *blkLabel = &blk_labels[block_num];
1629                 mach->add_case_label(jpn->as_JumpProj()->proj_no(), blkLabel);
1630               }
1631             }
1632           }
1633         }
1634 #ifdef ASSERT
1635           // Check that oop-store precedes the card-mark
1636         else if (mach->ideal_Opcode() == Op_StoreCM) {
1637           uint storeCM_idx = j;
1638           int count = 0;
1639           for (uint prec = mach->req(); prec < mach->len(); prec++) {
1640             Node *oop_store = mach->in(prec);  // Precedence edge
1641             if (oop_store == nullptr) continue;
1642             count++;
1643             uint i4;
1644             for (i4 = 0; i4 < last_inst; ++i4) {
1645               if (block->get_node(i4) == oop_store) {
1646                 break;
1647               }
1648             }
1649             // Note: This test can provide a false failure if other precedence
1650             // edges have been added to the storeCMNode.
1651             assert(i4 == last_inst || i4 < storeCM_idx, "CM card-mark executes before oop-store");
1652           }
1653           assert(count > 0, "storeCM expects at least one precedence edge");
1654         }
1655 #endif
1656         else if (!n->is_Proj()) {
1657           // Remember the beginning of the previous instruction, in case
1658           // it's followed by a flag-kill and a null-check.  Happens on
1659           // Intel all the time, with add-to-memory kind of opcodes.
1660           previous_offset = current_offset;
1661         }
1662 
1663         // Not an else-if!
1664         // If this is a trap based cmp then add its offset to the list.
1665         if (mach->is_TrapBasedCheckNode()) {
1666           inct_starts[inct_cnt++] = current_offset;
1667         }
1668       }
1669 
1670       // Verify that there is sufficient space remaining
1671       cb->insts()->maybe_expand_to_ensure_remaining(MAX_inst_size);
1672       if ((cb->blob() == nullptr) || (!CompileBroker::should_compile_new_jobs())) {
1673         C->record_failure("CodeCache is full");
1674         return;
1675       }
1676 
1677       // Save the offset for the listing
1678 #if defined(SUPPORT_OPTO_ASSEMBLY)
1679       if ((node_offsets != nullptr) && (n->_idx < node_offset_limit)) {
1680         node_offsets[n->_idx] = cb->insts_size();
1681       }
1682 #endif
1683       assert(!C->failing(), "Should not reach here if failing.");
1684 
1685       // "Normal" instruction case
1686       DEBUG_ONLY(uint instr_offset = cb->insts_size());
1687       n->emit(*cb, C->regalloc());
1688       current_offset = cb->insts_size();
1689 
1690       // Above we only verified that there is enough space in the instruction section.
1691       // However, the instruction may emit stubs that cause code buffer expansion.
1692       // Bail out here if expansion failed due to a lack of code cache space.
1693       if (C->failing()) {
1694         return;
1695       }
1696 
1697       assert(!is_mcall || (call_returns[block->_pre_order] <= (uint)current_offset),
1698              "ret_addr_offset() not within emitted code");
1699 
1700 #ifdef ASSERT
1701       uint n_size = n->size(C->regalloc());
1702       if (n_size < (current_offset-instr_offset)) {
1703         MachNode* mach = n->as_Mach();
1704         n->dump();
1705         mach->dump_format(C->regalloc(), tty);
1706         tty->print_cr(" n_size (%d), current_offset (%d), instr_offset (%d)", n_size, current_offset, instr_offset);
1707         Disassembler::decode(cb->insts_begin() + instr_offset, cb->insts_begin() + current_offset + 1, tty);
1708         tty->print_cr(" ------------------- ");
1709         BufferBlob* blob = this->scratch_buffer_blob();
1710         address blob_begin = blob->content_begin();
1711         Disassembler::decode(blob_begin, blob_begin + n_size + 1, tty);
1712         assert(false, "wrong size of mach node");
1713       }
1714 #endif
1715       non_safepoints.observe_instruction(n, current_offset);
1716 
1717       // mcall is last "call" that can be a safepoint
1718       // record it so we can see if a poll will directly follow it
1719       // in which case we'll need a pad to make the PcDesc sites unique
1720       // see  5010568. This can be slightly inaccurate but conservative
1721       // in the case that return address is not actually at current_offset.
1722       // This is a small price to pay.
1723 
1724       if (is_mcall) {
1725         last_call_offset = current_offset;
1726       }
1727 
1728       if (n->is_Mach() && n->as_Mach()->avoid_back_to_back(MachNode::AVOID_AFTER)) {
1729         // Avoid back to back some instructions.
1730         last_avoid_back_to_back_offset = current_offset;
1731       }
1732 
1733       // See if this instruction has a delay slot
1734       if (valid_bundle_info(n) && node_bundling(n)->use_unconditional_delay()) {
1735         guarantee(delay_slot != nullptr, "expecting delay slot node");
1736 
1737         // Back up 1 instruction
1738         cb->set_insts_end(cb->insts_end() - Pipeline::instr_unit_size());
1739 
1740         // Save the offset for the listing
1741 #if defined(SUPPORT_OPTO_ASSEMBLY)
1742         if ((node_offsets != nullptr) && (delay_slot->_idx < node_offset_limit)) {
1743           node_offsets[delay_slot->_idx] = cb->insts_size();
1744         }
1745 #endif
1746 
1747         // Support a SafePoint in the delay slot
1748         if (delay_slot->is_MachSafePoint()) {
1749           MachNode *mach = delay_slot->as_Mach();
1750           // !!!!! Stubs only need an oopmap right now, so bail out
1751           if (!mach->is_MachCall() && mach->as_MachSafePoint()->jvms()->method() == nullptr) {
1752             // Write the oopmap directly to the code blob??!!
1753             delay_slot = nullptr;
1754             continue;
1755           }
1756 
1757           int adjusted_offset = current_offset - Pipeline::instr_unit_size();
1758           non_safepoints.observe_safepoint(mach->as_MachSafePoint()->jvms(),
1759                                            adjusted_offset);
1760           // Generate an OopMap entry
1761           Process_OopMap_Node(mach, adjusted_offset);
1762         }
1763 
1764         // Insert the delay slot instruction
1765         delay_slot->emit(*cb, C->regalloc());
1766 
1767         // Don't reuse it
1768         delay_slot = nullptr;
1769       }
1770 
1771     } // End for all instructions in block
1772 
1773     // If the next block is the top of a loop, pad this block out to align
1774     // the loop top a little. Helps prevent pipe stalls at loop back branches.
1775     if (i < nblocks-1) {
1776       Block *nb = C->cfg()->get_block(i + 1);
1777       int padding = nb->alignment_padding(current_offset);
1778       if( padding > 0 ) {
1779         MachNode *nop = new MachNopNode(padding / nop_size);
1780         block->insert_node(nop, block->number_of_nodes());
1781         C->cfg()->map_node_to_block(nop, block);
1782         nop->emit(*cb, C->regalloc());
1783         current_offset = cb->insts_size();
1784       }
1785     }
1786     // Verify that the distance for generated before forward
1787     // short branches is still valid.
1788     guarantee((int)(blk_starts[i+1] - blk_starts[i]) >= (current_offset - blk_offset), "shouldn't increase block size");
1789 
1790     // Save new block start offset
1791     blk_starts[i] = blk_offset;
1792   } // End of for all blocks
1793   blk_starts[nblocks] = current_offset;
1794 
1795   non_safepoints.flush_at_end();
1796 
1797   // Offset too large?
1798   if (C->failing())  return;
1799 
1800   // Define a pseudo-label at the end of the code
1801   MacroAssembler(cb).bind( blk_labels[nblocks] );
1802 
1803   // Compute the size of the first block
1804   _first_block_size = blk_labels[1].loc_pos() - blk_labels[0].loc_pos();
1805 
1806 #ifdef ASSERT
1807   for (uint i = 0; i < nblocks; i++) { // For all blocks
1808     if (jmp_target[i] != 0) {
1809       int br_size = jmp_size[i];
1810       int offset = blk_starts[jmp_target[i]]-(blk_starts[i] + jmp_offset[i]);
1811       if (!C->matcher()->is_short_branch_offset(jmp_rule[i], br_size, offset)) {
1812         tty->print_cr("target (%d) - jmp_offset(%d) = offset (%d), jump_size(%d), jmp_block B%d, target_block B%d", blk_starts[jmp_target[i]], blk_starts[i] + jmp_offset[i], offset, br_size, i, jmp_target[i]);
1813         assert(false, "Displacement too large for short jmp");
1814       }
1815     }
1816   }
1817 #endif
1818 
1819   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1820   bs->emit_stubs(*cb);
1821   if (C->failing())  return;
1822 
1823   // Fill in stubs for calling the runtime from safepoint polls.
1824   safepoint_poll_table()->emit(*cb);
1825   if (C->failing())  return;
1826 
1827 #ifndef PRODUCT
1828   // Information on the size of the method, without the extraneous code
1829   Scheduling::increment_method_size(cb->insts_size());
1830 #endif
1831 
1832   // ------------------
1833   // Fill in exception table entries.
1834   FillExceptionTables(inct_cnt, call_returns, inct_starts, blk_labels);
1835 
1836   // Only java methods have exception handlers and deopt handlers
1837   // class HandlerImpl is platform-specific and defined in the *.ad files.
1838   if (C->method()) {
1839     // Emit the exception handler code.
1840     _code_offsets.set_value(CodeOffsets::Exceptions, HandlerImpl::emit_exception_handler(*cb));
1841     if (C->failing()) {
1842       return; // CodeBuffer::expand failed
1843     }
1844     // Emit the deopt handler code.
1845     _code_offsets.set_value(CodeOffsets::Deopt, HandlerImpl::emit_deopt_handler(*cb));
1846 
1847     // Emit the MethodHandle deopt handler code (if required).
1848     if (C->has_method_handle_invokes() && !C->failing()) {
1849       // We can use the same code as for the normal deopt handler, we
1850       // just need a different entry point address.
1851       _code_offsets.set_value(CodeOffsets::DeoptMH, HandlerImpl::emit_deopt_handler(*cb));
1852     }
1853   }
1854 
1855   // One last check for failed CodeBuffer::expand:
1856   if ((cb->blob() == nullptr) || (!CompileBroker::should_compile_new_jobs())) {
1857     C->record_failure("CodeCache is full");
1858     return;
1859   }
1860 
1861 #if defined(SUPPORT_ABSTRACT_ASSEMBLY) || defined(SUPPORT_ASSEMBLY) || defined(SUPPORT_OPTO_ASSEMBLY)
1862   if (C->print_assembly()) {
1863     tty->cr();
1864     tty->print_cr("============================= C2-compiled nmethod ==============================");
1865   }
1866 #endif
1867 
1868 #if defined(SUPPORT_OPTO_ASSEMBLY)
1869   // Dump the assembly code, including basic-block numbers
1870   if (C->print_assembly()) {
1871     ttyLocker ttyl;  // keep the following output all in one block
1872     if (!VMThread::should_terminate()) {  // test this under the tty lock
1873       // This output goes directly to the tty, not the compiler log.
1874       // To enable tools to match it up with the compilation activity,
1875       // be sure to tag this tty output with the compile ID.
1876       if (xtty != nullptr) {
1877         xtty->head("opto_assembly compile_id='%d'%s", C->compile_id(),
1878                    C->is_osr_compilation() ? " compile_kind='osr'" : "");
1879       }
1880       if (C->method() != nullptr) {
1881         tty->print_cr("----------------------- MetaData before Compile_id = %d ------------------------", C->compile_id());
1882         C->method()->print_metadata();
1883       } else if (C->stub_name() != nullptr) {
1884         tty->print_cr("----------------------------- RuntimeStub %s -------------------------------", C->stub_name());
1885       }
1886       tty->cr();
1887       tty->print_cr("------------------------ OptoAssembly for Compile_id = %d -----------------------", C->compile_id());
1888       dump_asm(node_offsets, node_offset_limit);
1889       tty->print_cr("--------------------------------------------------------------------------------");
1890       if (xtty != nullptr) {
1891         // print_metadata and dump_asm above may safepoint which makes us loose the ttylock.
1892         // Retake lock too make sure the end tag is coherent, and that xmlStream->pop_tag is done
1893         // thread safe
1894         ttyLocker ttyl2;
1895         xtty->tail("opto_assembly");
1896       }
1897     }
1898   }
1899 #endif
1900 }
1901 
1902 void PhaseOutput::FillExceptionTables(uint cnt, uint *call_returns, uint *inct_starts, Label *blk_labels) {
1903   _inc_table.set_size(cnt);
1904 
1905   uint inct_cnt = 0;
1906   for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
1907     Block* block = C->cfg()->get_block(i);
1908     Node *n = nullptr;
1909     int j;
1910 
1911     // Find the branch; ignore trailing NOPs.
1912     for (j = block->number_of_nodes() - 1; j >= 0; j--) {
1913       n = block->get_node(j);
1914       if (!n->is_Mach() || n->as_Mach()->ideal_Opcode() != Op_Con) {
1915         break;
1916       }
1917     }
1918 
1919     // If we didn't find anything, continue
1920     if (j < 0) {
1921       continue;
1922     }
1923 
1924     // Compute ExceptionHandlerTable subtable entry and add it
1925     // (skip empty blocks)
1926     if (n->is_Catch()) {
1927 
1928       // Get the offset of the return from the call
1929       uint call_return = call_returns[block->_pre_order];
1930 #ifdef ASSERT
1931       assert( call_return > 0, "no call seen for this basic block" );
1932       while (block->get_node(--j)->is_MachProj()) ;
1933       assert(block->get_node(j)->is_MachCall(), "CatchProj must follow call");
1934 #endif
1935       // last instruction is a CatchNode, find it's CatchProjNodes
1936       int nof_succs = block->_num_succs;
1937       // allocate space
1938       GrowableArray<intptr_t> handler_bcis(nof_succs);
1939       GrowableArray<intptr_t> handler_pcos(nof_succs);
1940       // iterate through all successors
1941       for (int j = 0; j < nof_succs; j++) {
1942         Block* s = block->_succs[j];
1943         bool found_p = false;
1944         for (uint k = 1; k < s->num_preds(); k++) {
1945           Node* pk = s->pred(k);
1946           if (pk->is_CatchProj() && pk->in(0) == n) {
1947             const CatchProjNode* p = pk->as_CatchProj();
1948             found_p = true;
1949             // add the corresponding handler bci & pco information
1950             if (p->_con != CatchProjNode::fall_through_index) {
1951               // p leads to an exception handler (and is not fall through)
1952               assert(s == C->cfg()->get_block(s->_pre_order), "bad numbering");
1953               // no duplicates, please
1954               if (!handler_bcis.contains(p->handler_bci())) {
1955                 uint block_num = s->non_connector()->_pre_order;
1956                 handler_bcis.append(p->handler_bci());
1957                 handler_pcos.append(blk_labels[block_num].loc_pos());
1958               }
1959             }
1960           }
1961         }
1962         assert(found_p, "no matching predecessor found");
1963         // Note:  Due to empty block removal, one block may have
1964         // several CatchProj inputs, from the same Catch.
1965       }
1966 
1967       // Set the offset of the return from the call
1968       assert(handler_bcis.find(-1) != -1, "must have default handler");
1969       _handler_table.add_subtable(call_return, &handler_bcis, nullptr, &handler_pcos);
1970       continue;
1971     }
1972 
1973     // Handle implicit null exception table updates
1974     if (n->is_MachNullCheck()) {
1975       uint block_num = block->non_connector_successor(0)->_pre_order;
1976       _inc_table.append(inct_starts[inct_cnt++], blk_labels[block_num].loc_pos());
1977       continue;
1978     }
1979     // Handle implicit exception table updates: trap instructions.
1980     if (n->is_Mach() && n->as_Mach()->is_TrapBasedCheckNode()) {
1981       uint block_num = block->non_connector_successor(0)->_pre_order;
1982       _inc_table.append(inct_starts[inct_cnt++], blk_labels[block_num].loc_pos());
1983       continue;
1984     }
1985   } // End of for all blocks fill in exception table entries
1986 }
1987 
1988 // Static Variables
1989 #ifndef PRODUCT
1990 uint Scheduling::_total_nop_size = 0;
1991 uint Scheduling::_total_method_size = 0;
1992 uint Scheduling::_total_branches = 0;
1993 uint Scheduling::_total_unconditional_delays = 0;
1994 uint Scheduling::_total_instructions_per_bundle[Pipeline::_max_instrs_per_cycle+1];
1995 #endif
1996 
1997 // Initializer for class Scheduling
1998 
1999 Scheduling::Scheduling(Arena *arena, Compile &compile)
2000         : _arena(arena),
2001           _cfg(compile.cfg()),
2002           _regalloc(compile.regalloc()),
2003           _scheduled(arena),
2004           _available(arena),
2005           _reg_node(arena),
2006           _pinch_free_list(arena),
2007           _next_node(nullptr),
2008           _bundle_instr_count(0),
2009           _bundle_cycle_number(0),
2010           _bundle_use(0, 0, resource_count, &_bundle_use_elements[0])
2011 #ifndef PRODUCT
2012         , _branches(0)
2013         , _unconditional_delays(0)
2014 #endif
2015 {
2016   // Create a MachNopNode
2017   _nop = new MachNopNode();
2018 
2019   // Now that the nops are in the array, save the count
2020   // (but allow entries for the nops)
2021   _node_bundling_limit = compile.unique();
2022   uint node_max = _regalloc->node_regs_max_index();
2023 
2024   compile.output()->set_node_bundling_limit(_node_bundling_limit);
2025 
2026   // This one is persistent within the Compile class
2027   _node_bundling_base = NEW_ARENA_ARRAY(compile.comp_arena(), Bundle, node_max);
2028 
2029   // Allocate space for fixed-size arrays
2030   _node_latency    = NEW_ARENA_ARRAY(arena, unsigned short, node_max);
2031   _uses            = NEW_ARENA_ARRAY(arena, short,          node_max);
2032   _current_latency = NEW_ARENA_ARRAY(arena, unsigned short, node_max);
2033 
2034   // Clear the arrays
2035   for (uint i = 0; i < node_max; i++) {
2036     ::new (&_node_bundling_base[i]) Bundle();
2037   }
2038   memset(_node_latency,       0, node_max * sizeof(unsigned short));
2039   memset(_uses,               0, node_max * sizeof(short));
2040   memset(_current_latency,    0, node_max * sizeof(unsigned short));
2041 
2042   // Clear the bundling information
2043   memcpy(_bundle_use_elements, Pipeline_Use::elaborated_elements, sizeof(Pipeline_Use::elaborated_elements));
2044 
2045   // Get the last node
2046   Block* block = _cfg->get_block(_cfg->number_of_blocks() - 1);
2047 
2048   _next_node = block->get_node(block->number_of_nodes() - 1);
2049 }
2050 
2051 #ifndef PRODUCT
2052 // Scheduling destructor
2053 Scheduling::~Scheduling() {
2054   _total_branches             += _branches;
2055   _total_unconditional_delays += _unconditional_delays;
2056 }
2057 #endif
2058 
2059 // Step ahead "i" cycles
2060 void Scheduling::step(uint i) {
2061 
2062   Bundle *bundle = node_bundling(_next_node);
2063   bundle->set_starts_bundle();
2064 
2065   // Update the bundle record, but leave the flags information alone
2066   if (_bundle_instr_count > 0) {
2067     bundle->set_instr_count(_bundle_instr_count);
2068     bundle->set_resources_used(_bundle_use.resourcesUsed());
2069   }
2070 
2071   // Update the state information
2072   _bundle_instr_count = 0;
2073   _bundle_cycle_number += i;
2074   _bundle_use.step(i);
2075 }
2076 
2077 void Scheduling::step_and_clear() {
2078   Bundle *bundle = node_bundling(_next_node);
2079   bundle->set_starts_bundle();
2080 
2081   // Update the bundle record
2082   if (_bundle_instr_count > 0) {
2083     bundle->set_instr_count(_bundle_instr_count);
2084     bundle->set_resources_used(_bundle_use.resourcesUsed());
2085 
2086     _bundle_cycle_number += 1;
2087   }
2088 
2089   // Clear the bundling information
2090   _bundle_instr_count = 0;
2091   _bundle_use.reset();
2092 
2093   memcpy(_bundle_use_elements,
2094          Pipeline_Use::elaborated_elements,
2095          sizeof(Pipeline_Use::elaborated_elements));
2096 }
2097 
2098 // Perform instruction scheduling and bundling over the sequence of
2099 // instructions in backwards order.
2100 void PhaseOutput::ScheduleAndBundle() {
2101 
2102   // Don't optimize this if it isn't a method
2103   if (!C->method())
2104     return;
2105 
2106   // Don't optimize this if scheduling is disabled
2107   if (!C->do_scheduling())
2108     return;
2109 
2110   // Scheduling code works only with pairs (8 bytes) maximum.
2111   if (C->max_vector_size() > 8)
2112     return;
2113 
2114   Compile::TracePhase tp("isched", &timers[_t_instrSched]);
2115 
2116   // Create a data structure for all the scheduling information
2117   Scheduling scheduling(Thread::current()->resource_area(), *C);
2118 
2119   // Walk backwards over each basic block, computing the needed alignment
2120   // Walk over all the basic blocks
2121   scheduling.DoScheduling();
2122 
2123 #ifndef PRODUCT
2124   if (C->trace_opto_output()) {
2125     tty->print("\n---- After ScheduleAndBundle ----\n");
2126     for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
2127       tty->print("\nBB#%03d:\n", i);
2128       Block* block = C->cfg()->get_block(i);
2129       for (uint j = 0; j < block->number_of_nodes(); j++) {
2130         Node* n = block->get_node(j);
2131         OptoReg::Name reg = C->regalloc()->get_reg_first(n);
2132         tty->print(" %-6s ", reg >= 0 && reg < REG_COUNT ? Matcher::regName[reg] : "");
2133         n->dump();
2134       }
2135     }
2136   }
2137 #endif
2138 }
2139 
2140 // Compute the latency of all the instructions.  This is fairly simple,
2141 // because we already have a legal ordering.  Walk over the instructions
2142 // from first to last, and compute the latency of the instruction based
2143 // on the latency of the preceding instruction(s).
2144 void Scheduling::ComputeLocalLatenciesForward(const Block *bb) {
2145 #ifndef PRODUCT
2146   if (_cfg->C->trace_opto_output())
2147     tty->print("# -> ComputeLocalLatenciesForward\n");
2148 #endif
2149 
2150   // Walk over all the schedulable instructions
2151   for( uint j=_bb_start; j < _bb_end; j++ ) {
2152 
2153     // This is a kludge, forcing all latency calculations to start at 1.
2154     // Used to allow latency 0 to force an instruction to the beginning
2155     // of the bb
2156     uint latency = 1;
2157     Node *use = bb->get_node(j);
2158     uint nlen = use->len();
2159 
2160     // Walk over all the inputs
2161     for ( uint k=0; k < nlen; k++ ) {
2162       Node *def = use->in(k);
2163       if (!def)
2164         continue;
2165 
2166       uint l = _node_latency[def->_idx] + use->latency(k);
2167       if (latency < l)
2168         latency = l;
2169     }
2170 
2171     _node_latency[use->_idx] = latency;
2172 
2173 #ifndef PRODUCT
2174     if (_cfg->C->trace_opto_output()) {
2175       tty->print("# latency %4d: ", latency);
2176       use->dump();
2177     }
2178 #endif
2179   }
2180 
2181 #ifndef PRODUCT
2182   if (_cfg->C->trace_opto_output())
2183     tty->print("# <- ComputeLocalLatenciesForward\n");
2184 #endif
2185 
2186 } // end ComputeLocalLatenciesForward
2187 
2188 // See if this node fits into the present instruction bundle
2189 bool Scheduling::NodeFitsInBundle(Node *n) {
2190   uint n_idx = n->_idx;
2191 
2192   // If this is the unconditional delay instruction, then it fits
2193   if (n == _unconditional_delay_slot) {
2194 #ifndef PRODUCT
2195     if (_cfg->C->trace_opto_output())
2196       tty->print("#     NodeFitsInBundle [%4d]: TRUE; is in unconditional delay slot\n", n->_idx);
2197 #endif
2198     return (true);
2199   }
2200 
2201   // If the node cannot be scheduled this cycle, skip it
2202   if (_current_latency[n_idx] > _bundle_cycle_number) {
2203 #ifndef PRODUCT
2204     if (_cfg->C->trace_opto_output())
2205       tty->print("#     NodeFitsInBundle [%4d]: FALSE; latency %4d > %d\n",
2206                  n->_idx, _current_latency[n_idx], _bundle_cycle_number);
2207 #endif
2208     return (false);
2209   }
2210 
2211   const Pipeline *node_pipeline = n->pipeline();
2212 
2213   uint instruction_count = node_pipeline->instructionCount();
2214   if (node_pipeline->mayHaveNoCode() && n->size(_regalloc) == 0)
2215     instruction_count = 0;
2216   else if (node_pipeline->hasBranchDelay() && !_unconditional_delay_slot)
2217     instruction_count++;
2218 
2219   if (_bundle_instr_count + instruction_count > Pipeline::_max_instrs_per_cycle) {
2220 #ifndef PRODUCT
2221     if (_cfg->C->trace_opto_output())
2222       tty->print("#     NodeFitsInBundle [%4d]: FALSE; too many instructions: %d > %d\n",
2223                  n->_idx, _bundle_instr_count + instruction_count, Pipeline::_max_instrs_per_cycle);
2224 #endif
2225     return (false);
2226   }
2227 
2228   // Don't allow non-machine nodes to be handled this way
2229   if (!n->is_Mach() && instruction_count == 0)
2230     return (false);
2231 
2232   // See if there is any overlap
2233   uint delay = _bundle_use.full_latency(0, node_pipeline->resourceUse());
2234 
2235   if (delay > 0) {
2236 #ifndef PRODUCT
2237     if (_cfg->C->trace_opto_output())
2238       tty->print("#     NodeFitsInBundle [%4d]: FALSE; functional units overlap\n", n_idx);
2239 #endif
2240     return false;
2241   }
2242 
2243 #ifndef PRODUCT
2244   if (_cfg->C->trace_opto_output())
2245     tty->print("#     NodeFitsInBundle [%4d]:  TRUE\n", n_idx);
2246 #endif
2247 
2248   return true;
2249 }
2250 
2251 Node * Scheduling::ChooseNodeToBundle() {
2252   uint siz = _available.size();
2253 
2254   if (siz == 0) {
2255 
2256 #ifndef PRODUCT
2257     if (_cfg->C->trace_opto_output())
2258       tty->print("#   ChooseNodeToBundle: null\n");
2259 #endif
2260     return (nullptr);
2261   }
2262 
2263   // Fast path, if only 1 instruction in the bundle
2264   if (siz == 1) {
2265 #ifndef PRODUCT
2266     if (_cfg->C->trace_opto_output()) {
2267       tty->print("#   ChooseNodeToBundle (only 1): ");
2268       _available[0]->dump();
2269     }
2270 #endif
2271     return (_available[0]);
2272   }
2273 
2274   // Don't bother, if the bundle is already full
2275   if (_bundle_instr_count < Pipeline::_max_instrs_per_cycle) {
2276     for ( uint i = 0; i < siz; i++ ) {
2277       Node *n = _available[i];
2278 
2279       // Skip projections, we'll handle them another way
2280       if (n->is_Proj())
2281         continue;
2282 
2283       // This presupposed that instructions are inserted into the
2284       // available list in a legality order; i.e. instructions that
2285       // must be inserted first are at the head of the list
2286       if (NodeFitsInBundle(n)) {
2287 #ifndef PRODUCT
2288         if (_cfg->C->trace_opto_output()) {
2289           tty->print("#   ChooseNodeToBundle: ");
2290           n->dump();
2291         }
2292 #endif
2293         return (n);
2294       }
2295     }
2296   }
2297 
2298   // Nothing fits in this bundle, choose the highest priority
2299 #ifndef PRODUCT
2300   if (_cfg->C->trace_opto_output()) {
2301     tty->print("#   ChooseNodeToBundle: ");
2302     _available[0]->dump();
2303   }
2304 #endif
2305 
2306   return _available[0];
2307 }
2308 
2309 void Scheduling::AddNodeToAvailableList(Node *n) {
2310   assert( !n->is_Proj(), "projections never directly made available" );
2311 #ifndef PRODUCT
2312   if (_cfg->C->trace_opto_output()) {
2313     tty->print("#   AddNodeToAvailableList: ");
2314     n->dump();
2315   }
2316 #endif
2317 
2318   int latency = _current_latency[n->_idx];
2319 
2320   // Insert in latency order (insertion sort)
2321   uint i;
2322   for ( i=0; i < _available.size(); i++ )
2323     if (_current_latency[_available[i]->_idx] > latency)
2324       break;
2325 
2326   // Special Check for compares following branches
2327   if( n->is_Mach() && _scheduled.size() > 0 ) {
2328     int op = n->as_Mach()->ideal_Opcode();
2329     Node *last = _scheduled[0];
2330     if( last->is_MachIf() && last->in(1) == n &&
2331         ( op == Op_CmpI ||
2332           op == Op_CmpU ||
2333           op == Op_CmpUL ||
2334           op == Op_CmpP ||
2335           op == Op_CmpF ||
2336           op == Op_CmpD ||
2337           op == Op_CmpL ) ) {
2338 
2339       // Recalculate position, moving to front of same latency
2340       for ( i=0 ; i < _available.size(); i++ )
2341         if (_current_latency[_available[i]->_idx] >= latency)
2342           break;
2343     }
2344   }
2345 
2346   // Insert the node in the available list
2347   _available.insert(i, n);
2348 
2349 #ifndef PRODUCT
2350   if (_cfg->C->trace_opto_output())
2351     dump_available();
2352 #endif
2353 }
2354 
2355 void Scheduling::DecrementUseCounts(Node *n, const Block *bb) {
2356   for ( uint i=0; i < n->len(); i++ ) {
2357     Node *def = n->in(i);
2358     if (!def) continue;
2359     if( def->is_Proj() )        // If this is a machine projection, then
2360       def = def->in(0);         // propagate usage thru to the base instruction
2361 
2362     if(_cfg->get_block_for_node(def) != bb) { // Ignore if not block-local
2363       continue;
2364     }
2365 
2366     // Compute the latency
2367     uint l = _bundle_cycle_number + n->latency(i);
2368     if (_current_latency[def->_idx] < l)
2369       _current_latency[def->_idx] = l;
2370 
2371     // If this does not have uses then schedule it
2372     if ((--_uses[def->_idx]) == 0)
2373       AddNodeToAvailableList(def);
2374   }
2375 }
2376 
2377 void Scheduling::AddNodeToBundle(Node *n, const Block *bb) {
2378 #ifndef PRODUCT
2379   if (_cfg->C->trace_opto_output()) {
2380     tty->print("#   AddNodeToBundle: ");
2381     n->dump();
2382   }
2383 #endif
2384 
2385   // Remove this from the available list
2386   uint i;
2387   for (i = 0; i < _available.size(); i++)
2388     if (_available[i] == n)
2389       break;
2390   assert(i < _available.size(), "entry in _available list not found");
2391   _available.remove(i);
2392 
2393   // See if this fits in the current bundle
2394   const Pipeline *node_pipeline = n->pipeline();
2395   const Pipeline_Use& node_usage = node_pipeline->resourceUse();
2396 
2397   // Check for instructions to be placed in the delay slot. We
2398   // do this before we actually schedule the current instruction,
2399   // because the delay slot follows the current instruction.
2400   if (Pipeline::_branch_has_delay_slot &&
2401       node_pipeline->hasBranchDelay() &&
2402       !_unconditional_delay_slot) {
2403 
2404     uint siz = _available.size();
2405 
2406     // Conditional branches can support an instruction that
2407     // is unconditionally executed and not dependent by the
2408     // branch, OR a conditionally executed instruction if
2409     // the branch is taken.  In practice, this means that
2410     // the first instruction at the branch target is
2411     // copied to the delay slot, and the branch goes to
2412     // the instruction after that at the branch target
2413     if ( n->is_MachBranch() ) {
2414 
2415       assert( !n->is_MachNullCheck(), "should not look for delay slot for Null Check" );
2416       assert( !n->is_Catch(),         "should not look for delay slot for Catch" );
2417 
2418 #ifndef PRODUCT
2419       _branches++;
2420 #endif
2421 
2422       // At least 1 instruction is on the available list
2423       // that is not dependent on the branch
2424       for (uint i = 0; i < siz; i++) {
2425         Node *d = _available[i];
2426         const Pipeline *avail_pipeline = d->pipeline();
2427 
2428         // Don't allow safepoints in the branch shadow, that will
2429         // cause a number of difficulties
2430         if ( avail_pipeline->instructionCount() == 1 &&
2431              !avail_pipeline->hasMultipleBundles() &&
2432              !avail_pipeline->hasBranchDelay() &&
2433              Pipeline::instr_has_unit_size() &&
2434              d->size(_regalloc) == Pipeline::instr_unit_size() &&
2435              NodeFitsInBundle(d) &&
2436              !node_bundling(d)->used_in_delay()) {
2437 
2438           if (d->is_Mach() && !d->is_MachSafePoint()) {
2439             // A node that fits in the delay slot was found, so we need to
2440             // set the appropriate bits in the bundle pipeline information so
2441             // that it correctly indicates resource usage.  Later, when we
2442             // attempt to add this instruction to the bundle, we will skip
2443             // setting the resource usage.
2444             _unconditional_delay_slot = d;
2445             node_bundling(n)->set_use_unconditional_delay();
2446             node_bundling(d)->set_used_in_unconditional_delay();
2447             _bundle_use.add_usage(avail_pipeline->resourceUse());
2448             _current_latency[d->_idx] = _bundle_cycle_number;
2449             _next_node = d;
2450             ++_bundle_instr_count;
2451 #ifndef PRODUCT
2452             _unconditional_delays++;
2453 #endif
2454             break;
2455           }
2456         }
2457       }
2458     }
2459 
2460     // No delay slot, add a nop to the usage
2461     if (!_unconditional_delay_slot) {
2462       // See if adding an instruction in the delay slot will overflow
2463       // the bundle.
2464       if (!NodeFitsInBundle(_nop)) {
2465 #ifndef PRODUCT
2466         if (_cfg->C->trace_opto_output())
2467           tty->print("#  *** STEP(1 instruction for delay slot) ***\n");
2468 #endif
2469         step(1);
2470       }
2471 
2472       _bundle_use.add_usage(_nop->pipeline()->resourceUse());
2473       _next_node = _nop;
2474       ++_bundle_instr_count;
2475     }
2476 
2477     // See if the instruction in the delay slot requires a
2478     // step of the bundles
2479     if (!NodeFitsInBundle(n)) {
2480 #ifndef PRODUCT
2481       if (_cfg->C->trace_opto_output())
2482         tty->print("#  *** STEP(branch won't fit) ***\n");
2483 #endif
2484       // Update the state information
2485       _bundle_instr_count = 0;
2486       _bundle_cycle_number += 1;
2487       _bundle_use.step(1);
2488     }
2489   }
2490 
2491   // Get the number of instructions
2492   uint instruction_count = node_pipeline->instructionCount();
2493   if (node_pipeline->mayHaveNoCode() && n->size(_regalloc) == 0)
2494     instruction_count = 0;
2495 
2496   // Compute the latency information
2497   uint delay = 0;
2498 
2499   if (instruction_count > 0 || !node_pipeline->mayHaveNoCode()) {
2500     int relative_latency = _current_latency[n->_idx] - _bundle_cycle_number;
2501     if (relative_latency < 0)
2502       relative_latency = 0;
2503 
2504     delay = _bundle_use.full_latency(relative_latency, node_usage);
2505 
2506     // Does not fit in this bundle, start a new one
2507     if (delay > 0) {
2508       step(delay);
2509 
2510 #ifndef PRODUCT
2511       if (_cfg->C->trace_opto_output())
2512         tty->print("#  *** STEP(%d) ***\n", delay);
2513 #endif
2514     }
2515   }
2516 
2517   // If this was placed in the delay slot, ignore it
2518   if (n != _unconditional_delay_slot) {
2519 
2520     if (delay == 0) {
2521       if (node_pipeline->hasMultipleBundles()) {
2522 #ifndef PRODUCT
2523         if (_cfg->C->trace_opto_output())
2524           tty->print("#  *** STEP(multiple instructions) ***\n");
2525 #endif
2526         step(1);
2527       }
2528 
2529       else if (instruction_count + _bundle_instr_count > Pipeline::_max_instrs_per_cycle) {
2530 #ifndef PRODUCT
2531         if (_cfg->C->trace_opto_output())
2532           tty->print("#  *** STEP(%d >= %d instructions) ***\n",
2533                      instruction_count + _bundle_instr_count,
2534                      Pipeline::_max_instrs_per_cycle);
2535 #endif
2536         step(1);
2537       }
2538     }
2539 
2540     if (node_pipeline->hasBranchDelay() && !_unconditional_delay_slot)
2541       _bundle_instr_count++;
2542 
2543     // Set the node's latency
2544     _current_latency[n->_idx] = _bundle_cycle_number;
2545 
2546     // Now merge the functional unit information
2547     if (instruction_count > 0 || !node_pipeline->mayHaveNoCode())
2548       _bundle_use.add_usage(node_usage);
2549 
2550     // Increment the number of instructions in this bundle
2551     _bundle_instr_count += instruction_count;
2552 
2553     // Remember this node for later
2554     if (n->is_Mach())
2555       _next_node = n;
2556   }
2557 
2558   // It's possible to have a BoxLock in the graph and in the _bbs mapping but
2559   // not in the bb->_nodes array.  This happens for debug-info-only BoxLocks.
2560   // 'Schedule' them (basically ignore in the schedule) but do not insert them
2561   // into the block.  All other scheduled nodes get put in the schedule here.
2562   int op = n->Opcode();
2563   if( (op == Op_Node && n->req() == 0) || // anti-dependence node OR
2564       (op != Op_Node &&         // Not an unused antidepedence node and
2565        // not an unallocated boxlock
2566        (OptoReg::is_valid(_regalloc->get_reg_first(n)) || op != Op_BoxLock)) ) {
2567 
2568     // Push any trailing projections
2569     if( bb->get_node(bb->number_of_nodes()-1) != n ) {
2570       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
2571         Node *foi = n->fast_out(i);
2572         if( foi->is_Proj() )
2573           _scheduled.push(foi);
2574       }
2575     }
2576 
2577     // Put the instruction in the schedule list
2578     _scheduled.push(n);
2579   }
2580 
2581 #ifndef PRODUCT
2582   if (_cfg->C->trace_opto_output())
2583     dump_available();
2584 #endif
2585 
2586   // Walk all the definitions, decrementing use counts, and
2587   // if a definition has a 0 use count, place it in the available list.
2588   DecrementUseCounts(n,bb);
2589 }
2590 
2591 // This method sets the use count within a basic block.  We will ignore all
2592 // uses outside the current basic block.  As we are doing a backwards walk,
2593 // any node we reach that has a use count of 0 may be scheduled.  This also
2594 // avoids the problem of cyclic references from phi nodes, as long as phi
2595 // nodes are at the front of the basic block.  This method also initializes
2596 // the available list to the set of instructions that have no uses within this
2597 // basic block.
2598 void Scheduling::ComputeUseCount(const Block *bb) {
2599 #ifndef PRODUCT
2600   if (_cfg->C->trace_opto_output())
2601     tty->print("# -> ComputeUseCount\n");
2602 #endif
2603 
2604   // Clear the list of available and scheduled instructions, just in case
2605   _available.clear();
2606   _scheduled.clear();
2607 
2608   // No delay slot specified
2609   _unconditional_delay_slot = nullptr;
2610 
2611 #ifdef ASSERT
2612   for( uint i=0; i < bb->number_of_nodes(); i++ )
2613     assert( _uses[bb->get_node(i)->_idx] == 0, "_use array not clean" );
2614 #endif
2615 
2616   // Force the _uses count to never go to zero for unscheduable pieces
2617   // of the block
2618   for( uint k = 0; k < _bb_start; k++ )
2619     _uses[bb->get_node(k)->_idx] = 1;
2620   for( uint l = _bb_end; l < bb->number_of_nodes(); l++ )
2621     _uses[bb->get_node(l)->_idx] = 1;
2622 
2623   // Iterate backwards over the instructions in the block.  Don't count the
2624   // branch projections at end or the block header instructions.
2625   for( uint j = _bb_end-1; j >= _bb_start; j-- ) {
2626     Node *n = bb->get_node(j);
2627     if( n->is_Proj() ) continue; // Projections handled another way
2628 
2629     // Account for all uses
2630     for ( uint k = 0; k < n->len(); k++ ) {
2631       Node *inp = n->in(k);
2632       if (!inp) continue;
2633       assert(inp != n, "no cycles allowed" );
2634       if (_cfg->get_block_for_node(inp) == bb) { // Block-local use?
2635         if (inp->is_Proj()) { // Skip through Proj's
2636           inp = inp->in(0);
2637         }
2638         ++_uses[inp->_idx];     // Count 1 block-local use
2639       }
2640     }
2641 
2642     // If this instruction has a 0 use count, then it is available
2643     if (!_uses[n->_idx]) {
2644       _current_latency[n->_idx] = _bundle_cycle_number;
2645       AddNodeToAvailableList(n);
2646     }
2647 
2648 #ifndef PRODUCT
2649     if (_cfg->C->trace_opto_output()) {
2650       tty->print("#   uses: %3d: ", _uses[n->_idx]);
2651       n->dump();
2652     }
2653 #endif
2654   }
2655 
2656 #ifndef PRODUCT
2657   if (_cfg->C->trace_opto_output())
2658     tty->print("# <- ComputeUseCount\n");
2659 #endif
2660 }
2661 
2662 // This routine performs scheduling on each basic block in reverse order,
2663 // using instruction latencies and taking into account function unit
2664 // availability.
2665 void Scheduling::DoScheduling() {
2666 #ifndef PRODUCT
2667   if (_cfg->C->trace_opto_output())
2668     tty->print("# -> DoScheduling\n");
2669 #endif
2670 
2671   Block *succ_bb = nullptr;
2672   Block *bb;
2673   Compile* C = Compile::current();
2674 
2675   // Walk over all the basic blocks in reverse order
2676   for (int i = _cfg->number_of_blocks() - 1; i >= 0; succ_bb = bb, i--) {
2677     bb = _cfg->get_block(i);
2678 
2679 #ifndef PRODUCT
2680     if (_cfg->C->trace_opto_output()) {
2681       tty->print("#  Schedule BB#%03d (initial)\n", i);
2682       for (uint j = 0; j < bb->number_of_nodes(); j++) {
2683         bb->get_node(j)->dump();
2684       }
2685     }
2686 #endif
2687 
2688     // On the head node, skip processing
2689     if (bb == _cfg->get_root_block()) {
2690       continue;
2691     }
2692 
2693     // Skip empty, connector blocks
2694     if (bb->is_connector())
2695       continue;
2696 
2697     // If the following block is not the sole successor of
2698     // this one, then reset the pipeline information
2699     if (bb->_num_succs != 1 || bb->non_connector_successor(0) != succ_bb) {
2700 #ifndef PRODUCT
2701       if (_cfg->C->trace_opto_output()) {
2702         tty->print("*** bundle start of next BB, node %d, for %d instructions\n",
2703                    _next_node->_idx, _bundle_instr_count);
2704       }
2705 #endif
2706       step_and_clear();
2707     }
2708 
2709     // Leave untouched the starting instruction, any Phis, a CreateEx node
2710     // or Top.  bb->get_node(_bb_start) is the first schedulable instruction.
2711     _bb_end = bb->number_of_nodes()-1;
2712     for( _bb_start=1; _bb_start <= _bb_end; _bb_start++ ) {
2713       Node *n = bb->get_node(_bb_start);
2714       // Things not matched, like Phinodes and ProjNodes don't get scheduled.
2715       // Also, MachIdealNodes do not get scheduled
2716       if( !n->is_Mach() ) continue;     // Skip non-machine nodes
2717       MachNode *mach = n->as_Mach();
2718       int iop = mach->ideal_Opcode();
2719       if( iop == Op_CreateEx ) continue; // CreateEx is pinned
2720       if( iop == Op_Con ) continue;      // Do not schedule Top
2721       if( iop == Op_Node &&     // Do not schedule PhiNodes, ProjNodes
2722           mach->pipeline() == MachNode::pipeline_class() &&
2723           !n->is_SpillCopy() && !n->is_MachMerge() )  // Breakpoints, Prolog, etc
2724         continue;
2725       break;                    // Funny loop structure to be sure...
2726     }
2727     // Compute last "interesting" instruction in block - last instruction we
2728     // might schedule.  _bb_end points just after last schedulable inst.  We
2729     // normally schedule conditional branches (despite them being forced last
2730     // in the block), because they have delay slots we can fill.  Calls all
2731     // have their delay slots filled in the template expansions, so we don't
2732     // bother scheduling them.
2733     Node *last = bb->get_node(_bb_end);
2734     // Ignore trailing NOPs.
2735     while (_bb_end > 0 && last->is_Mach() &&
2736            last->as_Mach()->ideal_Opcode() == Op_Con) {
2737       last = bb->get_node(--_bb_end);
2738     }
2739     assert(!last->is_Mach() || last->as_Mach()->ideal_Opcode() != Op_Con, "");
2740     if( last->is_Catch() ||
2741         (last->is_Mach() && last->as_Mach()->ideal_Opcode() == Op_Halt) ) {
2742       // There might be a prior call.  Skip it.
2743       while (_bb_start < _bb_end && bb->get_node(--_bb_end)->is_MachProj());
2744     } else if( last->is_MachNullCheck() ) {
2745       // Backup so the last null-checked memory instruction is
2746       // outside the schedulable range. Skip over the nullcheck,
2747       // projection, and the memory nodes.
2748       Node *mem = last->in(1);
2749       do {
2750         _bb_end--;
2751       } while (mem != bb->get_node(_bb_end));
2752     } else {
2753       // Set _bb_end to point after last schedulable inst.
2754       _bb_end++;
2755     }
2756 
2757     assert( _bb_start <= _bb_end, "inverted block ends" );
2758 
2759     // Compute the register antidependencies for the basic block
2760     ComputeRegisterAntidependencies(bb);
2761     if (C->failing())  return;  // too many D-U pinch points
2762 
2763     // Compute intra-bb latencies for the nodes
2764     ComputeLocalLatenciesForward(bb);
2765 
2766     // Compute the usage within the block, and set the list of all nodes
2767     // in the block that have no uses within the block.
2768     ComputeUseCount(bb);
2769 
2770     // Schedule the remaining instructions in the block
2771     while ( _available.size() > 0 ) {
2772       Node *n = ChooseNodeToBundle();
2773       guarantee(n != nullptr, "no nodes available");
2774       AddNodeToBundle(n,bb);
2775     }
2776 
2777     assert( _scheduled.size() == _bb_end - _bb_start, "wrong number of instructions" );
2778 #ifdef ASSERT
2779     for( uint l = _bb_start; l < _bb_end; l++ ) {
2780       Node *n = bb->get_node(l);
2781       uint m;
2782       for( m = 0; m < _bb_end-_bb_start; m++ )
2783         if( _scheduled[m] == n )
2784           break;
2785       assert( m < _bb_end-_bb_start, "instruction missing in schedule" );
2786     }
2787 #endif
2788 
2789     // Now copy the instructions (in reverse order) back to the block
2790     for ( uint k = _bb_start; k < _bb_end; k++ )
2791       bb->map_node(_scheduled[_bb_end-k-1], k);
2792 
2793 #ifndef PRODUCT
2794     if (_cfg->C->trace_opto_output()) {
2795       tty->print("#  Schedule BB#%03d (final)\n", i);
2796       uint current = 0;
2797       for (uint j = 0; j < bb->number_of_nodes(); j++) {
2798         Node *n = bb->get_node(j);
2799         if( valid_bundle_info(n) ) {
2800           Bundle *bundle = node_bundling(n);
2801           if (bundle->instr_count() > 0 || bundle->flags() > 0) {
2802             tty->print("*** Bundle: ");
2803             bundle->dump();
2804           }
2805           n->dump();
2806         }
2807       }
2808     }
2809 #endif
2810 #ifdef ASSERT
2811     verify_good_schedule(bb,"after block local scheduling");
2812 #endif
2813   }
2814 
2815 #ifndef PRODUCT
2816   if (_cfg->C->trace_opto_output())
2817     tty->print("# <- DoScheduling\n");
2818 #endif
2819 
2820   // Record final node-bundling array location
2821   _regalloc->C->output()->set_node_bundling_base(_node_bundling_base);
2822 
2823 } // end DoScheduling
2824 
2825 // Verify that no live-range used in the block is killed in the block by a
2826 // wrong DEF.  This doesn't verify live-ranges that span blocks.
2827 
2828 // Check for edge existence.  Used to avoid adding redundant precedence edges.
2829 static bool edge_from_to( Node *from, Node *to ) {
2830   for( uint i=0; i<from->len(); i++ )
2831     if( from->in(i) == to )
2832       return true;
2833   return false;
2834 }
2835 
2836 #ifdef ASSERT
2837 void Scheduling::verify_do_def( Node *n, OptoReg::Name def, const char *msg ) {
2838   // Check for bad kills
2839   if( OptoReg::is_valid(def) ) { // Ignore stores & control flow
2840     Node *prior_use = _reg_node[def];
2841     if( prior_use && !edge_from_to(prior_use,n) ) {
2842       tty->print("%s = ",OptoReg::as_VMReg(def)->name());
2843       n->dump();
2844       tty->print_cr("...");
2845       prior_use->dump();
2846       assert(edge_from_to(prior_use,n), "%s", msg);
2847     }
2848     _reg_node.map(def,nullptr); // Kill live USEs
2849   }
2850 }
2851 
2852 void Scheduling::verify_good_schedule( Block *b, const char *msg ) {
2853 
2854   // Zap to something reasonable for the verify code
2855   _reg_node.clear();
2856 
2857   // Walk over the block backwards.  Check to make sure each DEF doesn't
2858   // kill a live value (other than the one it's supposed to).  Add each
2859   // USE to the live set.
2860   for( uint i = b->number_of_nodes()-1; i >= _bb_start; i-- ) {
2861     Node *n = b->get_node(i);
2862     int n_op = n->Opcode();
2863     if( n_op == Op_MachProj && n->ideal_reg() == MachProjNode::fat_proj ) {
2864       // Fat-proj kills a slew of registers
2865       RegMaskIterator rmi(n->out_RegMask());
2866       while (rmi.has_next()) {
2867         OptoReg::Name kill = rmi.next();
2868         verify_do_def(n, kill, msg);
2869       }
2870     } else if( n_op != Op_Node ) { // Avoid brand new antidependence nodes
2871       // Get DEF'd registers the normal way
2872       verify_do_def( n, _regalloc->get_reg_first(n), msg );
2873       verify_do_def( n, _regalloc->get_reg_second(n), msg );
2874     }
2875 
2876     // Now make all USEs live
2877     for( uint i=1; i<n->req(); i++ ) {
2878       Node *def = n->in(i);
2879       assert(def != 0, "input edge required");
2880       OptoReg::Name reg_lo = _regalloc->get_reg_first(def);
2881       OptoReg::Name reg_hi = _regalloc->get_reg_second(def);
2882       if( OptoReg::is_valid(reg_lo) ) {
2883         assert(!_reg_node[reg_lo] || edge_from_to(_reg_node[reg_lo],def), "%s", msg);
2884         _reg_node.map(reg_lo,n);
2885       }
2886       if( OptoReg::is_valid(reg_hi) ) {
2887         assert(!_reg_node[reg_hi] || edge_from_to(_reg_node[reg_hi],def), "%s", msg);
2888         _reg_node.map(reg_hi,n);
2889       }
2890     }
2891 
2892   }
2893 
2894   // Zap to something reasonable for the Antidependence code
2895   _reg_node.clear();
2896 }
2897 #endif
2898 
2899 // Conditionally add precedence edges.  Avoid putting edges on Projs.
2900 static void add_prec_edge_from_to( Node *from, Node *to ) {
2901   if( from->is_Proj() ) {       // Put precedence edge on Proj's input
2902     assert( from->req() == 1 && (from->len() == 1 || from->in(1)==0), "no precedence edges on projections" );
2903     from = from->in(0);
2904   }
2905   if( from != to &&             // No cycles (for things like LD L0,[L0+4] )
2906       !edge_from_to( from, to ) ) // Avoid duplicate edge
2907     from->add_prec(to);
2908 }
2909 
2910 void Scheduling::anti_do_def( Block *b, Node *def, OptoReg::Name def_reg, int is_def ) {
2911   if( !OptoReg::is_valid(def_reg) ) // Ignore stores & control flow
2912     return;
2913 
2914   if (OptoReg::is_reg(def_reg)) {
2915     VMReg vmreg = OptoReg::as_VMReg(def_reg);
2916     if (vmreg->is_reg() && !vmreg->is_concrete() && !vmreg->prev()->is_concrete()) {
2917       // This is one of the high slots of a vector register.
2918       // ScheduleAndBundle already checked there are no live wide
2919       // vectors in this method so it can be safely ignored.
2920       return;
2921     }
2922   }
2923 
2924   Node *pinch = _reg_node[def_reg]; // Get pinch point
2925   if ((pinch == nullptr) || _cfg->get_block_for_node(pinch) != b || // No pinch-point yet?
2926       is_def ) {    // Check for a true def (not a kill)
2927     _reg_node.map(def_reg,def); // Record def/kill as the optimistic pinch-point
2928     return;
2929   }
2930 
2931   Node *kill = def;             // Rename 'def' to more descriptive 'kill'
2932   debug_only( def = (Node*)((intptr_t)0xdeadbeef); )
2933 
2934   // After some number of kills there _may_ be a later def
2935   Node *later_def = nullptr;
2936 
2937   Compile* C = Compile::current();
2938 
2939   // Finding a kill requires a real pinch-point.
2940   // Check for not already having a pinch-point.
2941   // Pinch points are Op_Node's.
2942   if( pinch->Opcode() != Op_Node ) { // Or later-def/kill as pinch-point?
2943     later_def = pinch;            // Must be def/kill as optimistic pinch-point
2944     if ( _pinch_free_list.size() > 0) {
2945       pinch = _pinch_free_list.pop();
2946     } else {
2947       pinch = new Node(1); // Pinch point to-be
2948     }
2949     if (pinch->_idx >= _regalloc->node_regs_max_index()) {
2950       _cfg->C->record_method_not_compilable("too many D-U pinch points");
2951       return;
2952     }
2953     _cfg->map_node_to_block(pinch, b);      // Pretend it's valid in this block (lazy init)
2954     _reg_node.map(def_reg,pinch); // Record pinch-point
2955     //regalloc()->set_bad(pinch->_idx); // Already initialized this way.
2956     if( later_def->outcnt() == 0 || later_def->ideal_reg() == MachProjNode::fat_proj ) { // Distinguish def from kill
2957       pinch->init_req(0, C->top());     // set not null for the next call
2958       add_prec_edge_from_to(later_def,pinch); // Add edge from kill to pinch
2959       later_def = nullptr;           // and no later def
2960     }
2961     pinch->set_req(0,later_def);  // Hook later def so we can find it
2962   } else {                        // Else have valid pinch point
2963     if( pinch->in(0) )            // If there is a later-def
2964       later_def = pinch->in(0);   // Get it
2965   }
2966 
2967   // Add output-dependence edge from later def to kill
2968   if( later_def )               // If there is some original def
2969     add_prec_edge_from_to(later_def,kill); // Add edge from def to kill
2970 
2971   // See if current kill is also a use, and so is forced to be the pinch-point.
2972   if( pinch->Opcode() == Op_Node ) {
2973     Node *uses = kill->is_Proj() ? kill->in(0) : kill;
2974     for( uint i=1; i<uses->req(); i++ ) {
2975       if( _regalloc->get_reg_first(uses->in(i)) == def_reg ||
2976           _regalloc->get_reg_second(uses->in(i)) == def_reg ) {
2977         // Yes, found a use/kill pinch-point
2978         pinch->set_req(0,nullptr);  //
2979         pinch->replace_by(kill); // Move anti-dep edges up
2980         pinch = kill;
2981         _reg_node.map(def_reg,pinch);
2982         return;
2983       }
2984     }
2985   }
2986 
2987   // Add edge from kill to pinch-point
2988   add_prec_edge_from_to(kill,pinch);
2989 }
2990 
2991 void Scheduling::anti_do_use( Block *b, Node *use, OptoReg::Name use_reg ) {
2992   if( !OptoReg::is_valid(use_reg) ) // Ignore stores & control flow
2993     return;
2994   Node *pinch = _reg_node[use_reg]; // Get pinch point
2995   // Check for no later def_reg/kill in block
2996   if ((pinch != nullptr) && _cfg->get_block_for_node(pinch) == b &&
2997       // Use has to be block-local as well
2998       _cfg->get_block_for_node(use) == b) {
2999     if( pinch->Opcode() == Op_Node && // Real pinch-point (not optimistic?)
3000         pinch->req() == 1 ) {   // pinch not yet in block?
3001       pinch->del_req(0);        // yank pointer to later-def, also set flag
3002       // Insert the pinch-point in the block just after the last use
3003       b->insert_node(pinch, b->find_node(use) + 1);
3004       _bb_end++;                // Increase size scheduled region in block
3005     }
3006 
3007     add_prec_edge_from_to(pinch,use);
3008   }
3009 }
3010 
3011 // We insert antidependences between the reads and following write of
3012 // allocated registers to prevent illegal code motion. Hopefully, the
3013 // number of added references should be fairly small, especially as we
3014 // are only adding references within the current basic block.
3015 void Scheduling::ComputeRegisterAntidependencies(Block *b) {
3016 
3017 #ifdef ASSERT
3018   verify_good_schedule(b,"before block local scheduling");
3019 #endif
3020 
3021   // A valid schedule, for each register independently, is an endless cycle
3022   // of: a def, then some uses (connected to the def by true dependencies),
3023   // then some kills (defs with no uses), finally the cycle repeats with a new
3024   // def.  The uses are allowed to float relative to each other, as are the
3025   // kills.  No use is allowed to slide past a kill (or def).  This requires
3026   // antidependencies between all uses of a single def and all kills that
3027   // follow, up to the next def.  More edges are redundant, because later defs
3028   // & kills are already serialized with true or antidependencies.  To keep
3029   // the edge count down, we add a 'pinch point' node if there's more than
3030   // one use or more than one kill/def.
3031 
3032   // We add dependencies in one bottom-up pass.
3033 
3034   // For each instruction we handle it's DEFs/KILLs, then it's USEs.
3035 
3036   // For each DEF/KILL, we check to see if there's a prior DEF/KILL for this
3037   // register.  If not, we record the DEF/KILL in _reg_node, the
3038   // register-to-def mapping.  If there is a prior DEF/KILL, we insert a
3039   // "pinch point", a new Node that's in the graph but not in the block.
3040   // We put edges from the prior and current DEF/KILLs to the pinch point.
3041   // We put the pinch point in _reg_node.  If there's already a pinch point
3042   // we merely add an edge from the current DEF/KILL to the pinch point.
3043 
3044   // After doing the DEF/KILLs, we handle USEs.  For each used register, we
3045   // put an edge from the pinch point to the USE.
3046 
3047   // To be expedient, the _reg_node array is pre-allocated for the whole
3048   // compilation.  _reg_node is lazily initialized; it either contains a null,
3049   // or a valid def/kill/pinch-point, or a leftover node from some prior
3050   // block.  Leftover node from some prior block is treated like a null (no
3051   // prior def, so no anti-dependence needed).  Valid def is distinguished by
3052   // it being in the current block.
3053   bool fat_proj_seen = false;
3054   uint last_safept = _bb_end-1;
3055   Node* end_node         = (_bb_end-1 >= _bb_start) ? b->get_node(last_safept) : nullptr;
3056   Node* last_safept_node = end_node;
3057   for( uint i = _bb_end-1; i >= _bb_start; i-- ) {
3058     Node *n = b->get_node(i);
3059     int is_def = n->outcnt();   // def if some uses prior to adding precedence edges
3060     if( n->is_MachProj() && n->ideal_reg() == MachProjNode::fat_proj ) {
3061       // Fat-proj kills a slew of registers
3062       // This can add edges to 'n' and obscure whether or not it was a def,
3063       // hence the is_def flag.
3064       fat_proj_seen = true;
3065       RegMaskIterator rmi(n->out_RegMask());
3066       while (rmi.has_next()) {
3067         OptoReg::Name kill = rmi.next();
3068         anti_do_def(b, n, kill, is_def);
3069       }
3070     } else {
3071       // Get DEF'd registers the normal way
3072       anti_do_def( b, n, _regalloc->get_reg_first(n), is_def );
3073       anti_do_def( b, n, _regalloc->get_reg_second(n), is_def );
3074     }
3075 
3076     // Kill projections on a branch should appear to occur on the
3077     // branch, not afterwards, so grab the masks from the projections
3078     // and process them.
3079     if (n->is_MachBranch() || (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_Jump)) {
3080       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
3081         Node* use = n->fast_out(i);
3082         if (use->is_Proj()) {
3083           RegMaskIterator rmi(use->out_RegMask());
3084           while (rmi.has_next()) {
3085             OptoReg::Name kill = rmi.next();
3086             anti_do_def(b, n, kill, false);
3087           }
3088         }
3089       }
3090     }
3091 
3092     // Check each register used by this instruction for a following DEF/KILL
3093     // that must occur afterward and requires an anti-dependence edge.
3094     for( uint j=0; j<n->req(); j++ ) {
3095       Node *def = n->in(j);
3096       if( def ) {
3097         assert( !def->is_MachProj() || def->ideal_reg() != MachProjNode::fat_proj, "" );
3098         anti_do_use( b, n, _regalloc->get_reg_first(def) );
3099         anti_do_use( b, n, _regalloc->get_reg_second(def) );
3100       }
3101     }
3102     // Do not allow defs of new derived values to float above GC
3103     // points unless the base is definitely available at the GC point.
3104 
3105     Node *m = b->get_node(i);
3106 
3107     // Add precedence edge from following safepoint to use of derived pointer
3108     if( last_safept_node != end_node &&
3109         m != last_safept_node) {
3110       for (uint k = 1; k < m->req(); k++) {
3111         const Type *t = m->in(k)->bottom_type();
3112         if( t->isa_oop_ptr() &&
3113             t->is_ptr()->offset() != 0 ) {
3114           last_safept_node->add_prec( m );
3115           break;
3116         }
3117       }
3118     }
3119 
3120     if( n->jvms() ) {           // Precedence edge from derived to safept
3121       // Check if last_safept_node was moved by pinch-point insertion in anti_do_use()
3122       if( b->get_node(last_safept) != last_safept_node ) {
3123         last_safept = b->find_node(last_safept_node);
3124       }
3125       for( uint j=last_safept; j > i; j-- ) {
3126         Node *mach = b->get_node(j);
3127         if( mach->is_Mach() && mach->as_Mach()->ideal_Opcode() == Op_AddP )
3128           mach->add_prec( n );
3129       }
3130       last_safept = i;
3131       last_safept_node = m;
3132     }
3133   }
3134 
3135   if (fat_proj_seen) {
3136     // Garbage collect pinch nodes that were not consumed.
3137     // They are usually created by a fat kill MachProj for a call.
3138     garbage_collect_pinch_nodes();
3139   }
3140 }
3141 
3142 // Garbage collect pinch nodes for reuse by other blocks.
3143 //
3144 // The block scheduler's insertion of anti-dependence
3145 // edges creates many pinch nodes when the block contains
3146 // 2 or more Calls.  A pinch node is used to prevent a
3147 // combinatorial explosion of edges.  If a set of kills for a
3148 // register is anti-dependent on a set of uses (or defs), rather
3149 // than adding an edge in the graph between each pair of kill
3150 // and use (or def), a pinch is inserted between them:
3151 //
3152 //            use1   use2  use3
3153 //                \   |   /
3154 //                 \  |  /
3155 //                  pinch
3156 //                 /  |  \
3157 //                /   |   \
3158 //            kill1 kill2 kill3
3159 //
3160 // One pinch node is created per register killed when
3161 // the second call is encountered during a backwards pass
3162 // over the block.  Most of these pinch nodes are never
3163 // wired into the graph because the register is never
3164 // used or def'ed in the block.
3165 //
3166 void Scheduling::garbage_collect_pinch_nodes() {
3167 #ifndef PRODUCT
3168   if (_cfg->C->trace_opto_output()) tty->print("Reclaimed pinch nodes:");
3169 #endif
3170   int trace_cnt = 0;
3171   for (uint k = 0; k < _reg_node.Size(); k++) {
3172     Node* pinch = _reg_node[k];
3173     if ((pinch != nullptr) && pinch->Opcode() == Op_Node &&
3174         // no predecence input edges
3175         (pinch->req() == pinch->len() || pinch->in(pinch->req()) == nullptr) ) {
3176       cleanup_pinch(pinch);
3177       _pinch_free_list.push(pinch);
3178       _reg_node.map(k, nullptr);
3179 #ifndef PRODUCT
3180       if (_cfg->C->trace_opto_output()) {
3181         trace_cnt++;
3182         if (trace_cnt > 40) {
3183           tty->print("\n");
3184           trace_cnt = 0;
3185         }
3186         tty->print(" %d", pinch->_idx);
3187       }
3188 #endif
3189     }
3190   }
3191 #ifndef PRODUCT
3192   if (_cfg->C->trace_opto_output()) tty->print("\n");
3193 #endif
3194 }
3195 
3196 // Clean up a pinch node for reuse.
3197 void Scheduling::cleanup_pinch( Node *pinch ) {
3198   assert (pinch && pinch->Opcode() == Op_Node && pinch->req() == 1, "just checking");
3199 
3200   for (DUIterator_Last imin, i = pinch->last_outs(imin); i >= imin; ) {
3201     Node* use = pinch->last_out(i);
3202     uint uses_found = 0;
3203     for (uint j = use->req(); j < use->len(); j++) {
3204       if (use->in(j) == pinch) {
3205         use->rm_prec(j);
3206         uses_found++;
3207       }
3208     }
3209     assert(uses_found > 0, "must be a precedence edge");
3210     i -= uses_found;    // we deleted 1 or more copies of this edge
3211   }
3212   // May have a later_def entry
3213   pinch->set_req(0, nullptr);
3214 }
3215 
3216 #ifndef PRODUCT
3217 
3218 void Scheduling::dump_available() const {
3219   tty->print("#Availist  ");
3220   for (uint i = 0; i < _available.size(); i++)
3221     tty->print(" N%d/l%d", _available[i]->_idx,_current_latency[_available[i]->_idx]);
3222   tty->cr();
3223 }
3224 
3225 // Print Scheduling Statistics
3226 void Scheduling::print_statistics() {
3227   // Print the size added by nops for bundling
3228   tty->print("Nops added %d bytes to total of %d bytes",
3229              _total_nop_size, _total_method_size);
3230   if (_total_method_size > 0)
3231     tty->print(", for %.2f%%",
3232                ((double)_total_nop_size) / ((double) _total_method_size) * 100.0);
3233   tty->print("\n");
3234 
3235   // Print the number of branch shadows filled
3236   if (Pipeline::_branch_has_delay_slot) {
3237     tty->print("Of %d branches, %d had unconditional delay slots filled",
3238                _total_branches, _total_unconditional_delays);
3239     if (_total_branches > 0)
3240       tty->print(", for %.2f%%",
3241                  ((double)_total_unconditional_delays) / ((double)_total_branches) * 100.0);
3242     tty->print("\n");
3243   }
3244 
3245   uint total_instructions = 0, total_bundles = 0;
3246 
3247   for (uint i = 1; i <= Pipeline::_max_instrs_per_cycle; i++) {
3248     uint bundle_count   = _total_instructions_per_bundle[i];
3249     total_instructions += bundle_count * i;
3250     total_bundles      += bundle_count;
3251   }
3252 
3253   if (total_bundles > 0)
3254     tty->print("Average ILP (excluding nops) is %.2f\n",
3255                ((double)total_instructions) / ((double)total_bundles));
3256 }
3257 #endif
3258 
3259 //-----------------------init_scratch_buffer_blob------------------------------
3260 // Construct a temporary BufferBlob and cache it for this compile.
3261 void PhaseOutput::init_scratch_buffer_blob(int const_size) {
3262   // If there is already a scratch buffer blob allocated and the
3263   // constant section is big enough, use it.  Otherwise free the
3264   // current and allocate a new one.
3265   BufferBlob* blob = scratch_buffer_blob();
3266   if ((blob != nullptr) && (const_size <= _scratch_const_size)) {
3267     // Use the current blob.
3268   } else {
3269     if (blob != nullptr) {
3270       BufferBlob::free(blob);
3271     }
3272 
3273     ResourceMark rm;
3274     _scratch_const_size = const_size;
3275     int size = C2Compiler::initial_code_buffer_size(const_size);
3276     blob = BufferBlob::create("Compile::scratch_buffer", size);
3277     // Record the buffer blob for next time.
3278     set_scratch_buffer_blob(blob);
3279     // Have we run out of code space?
3280     if (scratch_buffer_blob() == nullptr) {
3281       // Let CompilerBroker disable further compilations.
3282       C->record_failure("Not enough space for scratch buffer in CodeCache");
3283       return;
3284     }
3285   }
3286 
3287   // Initialize the relocation buffers
3288   relocInfo* locs_buf = (relocInfo*) blob->content_end() - MAX_locs_size;
3289   set_scratch_locs_memory(locs_buf);
3290 }
3291 
3292 
3293 //-----------------------scratch_emit_size-------------------------------------
3294 // Helper function that computes size by emitting code
3295 uint PhaseOutput::scratch_emit_size(const Node* n) {
3296   // Start scratch_emit_size section.
3297   set_in_scratch_emit_size(true);
3298 
3299   // Emit into a trash buffer and count bytes emitted.
3300   // This is a pretty expensive way to compute a size,
3301   // but it works well enough if seldom used.
3302   // All common fixed-size instructions are given a size
3303   // method by the AD file.
3304   // Note that the scratch buffer blob and locs memory are
3305   // allocated at the beginning of the compile task, and
3306   // may be shared by several calls to scratch_emit_size.
3307   // The allocation of the scratch buffer blob is particularly
3308   // expensive, since it has to grab the code cache lock.
3309   BufferBlob* blob = this->scratch_buffer_blob();
3310   assert(blob != nullptr, "Initialize BufferBlob at start");
3311   assert(blob->size() > MAX_inst_size, "sanity");
3312   relocInfo* locs_buf = scratch_locs_memory();
3313   address blob_begin = blob->content_begin();
3314   address blob_end   = (address)locs_buf;
3315   assert(blob->contains(blob_end), "sanity");
3316   CodeBuffer buf(blob_begin, blob_end - blob_begin);
3317   buf.initialize_consts_size(_scratch_const_size);
3318   buf.initialize_stubs_size(MAX_stubs_size);
3319   assert(locs_buf != nullptr, "sanity");
3320   int lsize = MAX_locs_size / 3;
3321   buf.consts()->initialize_shared_locs(&locs_buf[lsize * 0], lsize);
3322   buf.insts()->initialize_shared_locs( &locs_buf[lsize * 1], lsize);
3323   buf.stubs()->initialize_shared_locs( &locs_buf[lsize * 2], lsize);
3324   // Mark as scratch buffer.
3325   buf.consts()->set_scratch_emit();
3326   buf.insts()->set_scratch_emit();
3327   buf.stubs()->set_scratch_emit();
3328 
3329   // Do the emission.
3330 
3331   Label fakeL; // Fake label for branch instructions.
3332   Label*   saveL = nullptr;
3333   uint save_bnum = 0;
3334   bool is_branch = n->is_MachBranch();
3335   if (is_branch) {
3336     MacroAssembler masm(&buf);
3337     masm.bind(fakeL);
3338     n->as_MachBranch()->save_label(&saveL, &save_bnum);
3339     n->as_MachBranch()->label_set(&fakeL, 0);
3340   }
3341   n->emit(buf, C->regalloc());
3342 
3343   // Emitting into the scratch buffer should not fail
3344   assert (!C->failing(), "Must not have pending failure. Reason is: %s", C->failure_reason());
3345 
3346   if (is_branch) // Restore label.
3347     n->as_MachBranch()->label_set(saveL, save_bnum);
3348 
3349   // End scratch_emit_size section.
3350   set_in_scratch_emit_size(false);
3351 
3352   return buf.insts_size();
3353 }
3354 
3355 void PhaseOutput::install() {
3356   if (!C->should_install_code()) {
3357     return;
3358   } else if (C->stub_function() != nullptr) {
3359     install_stub(C->stub_name());
3360   } else {
3361     install_code(C->method(),
3362                  C->entry_bci(),
3363                  CompileBroker::compiler2(),
3364                  C->has_unsafe_access(),
3365                  SharedRuntime::is_wide_vector(C->max_vector_size()),
3366                  C->rtm_state());
3367   }
3368 }
3369 
3370 void PhaseOutput::install_code(ciMethod*         target,
3371                                int               entry_bci,
3372                                AbstractCompiler* compiler,
3373                                bool              has_unsafe_access,
3374                                bool              has_wide_vectors,
3375                                RTMState          rtm_state) {
3376   // Check if we want to skip execution of all compiled code.
3377   {
3378 #ifndef PRODUCT
3379     if (OptoNoExecute) {
3380       C->record_method_not_compilable("+OptoNoExecute");  // Flag as failed
3381       return;
3382     }
3383 #endif
3384     Compile::TracePhase tp("install_code", &timers[_t_registerMethod]);
3385 
3386     if (C->is_osr_compilation()) {
3387       _code_offsets.set_value(CodeOffsets::Verified_Entry, 0);
3388       _code_offsets.set_value(CodeOffsets::OSR_Entry, _first_block_size);
3389     } else {
3390       _code_offsets.set_value(CodeOffsets::Verified_Entry, _first_block_size);
3391       _code_offsets.set_value(CodeOffsets::OSR_Entry, 0);
3392     }
3393 
3394     C->env()->register_method(target,
3395                                      entry_bci,
3396                                      &_code_offsets,
3397                                      _orig_pc_slot_offset_in_bytes,
3398                                      code_buffer(),
3399                                      frame_size_in_words(),
3400                                      oop_map_set(),
3401                                      &_handler_table,
3402                                      inc_table(),
3403                                      compiler,
3404                                      has_unsafe_access,
3405                                      SharedRuntime::is_wide_vector(C->max_vector_size()),
3406                                      C->rtm_state(),
3407                                      C->native_invokers());
3408 
3409     if (C->log() != nullptr) { // Print code cache state into compiler log
3410       C->log()->code_cache_state();
3411     }
3412   }
3413 }
3414 void PhaseOutput::install_stub(const char* stub_name) {
3415   // Entry point will be accessed using stub_entry_point();
3416   if (code_buffer() == nullptr) {
3417     Matcher::soft_match_failure();
3418   } else {
3419     if (PrintAssembly && (WizardMode || Verbose))
3420       tty->print_cr("### Stub::%s", stub_name);
3421 
3422     if (!C->failing()) {
3423       assert(C->fixed_slots() == 0, "no fixed slots used for runtime stubs");
3424 
3425       // Make the NMethod
3426       // For now we mark the frame as never safe for profile stackwalking
3427       RuntimeStub *rs = RuntimeStub::new_runtime_stub(stub_name,
3428                                                       code_buffer(),
3429                                                       CodeOffsets::frame_never_safe,
3430                                                       // _code_offsets.value(CodeOffsets::Frame_Complete),
3431                                                       frame_size_in_words(),
3432                                                       oop_map_set(),
3433                                                       false);
3434       assert(rs != nullptr && rs->is_runtime_stub(), "sanity check");
3435 
3436       C->set_stub_entry_point(rs->entry_point());
3437     }
3438   }
3439 }
3440 
3441 // Support for bundling info
3442 Bundle* PhaseOutput::node_bundling(const Node *n) {
3443   assert(valid_bundle_info(n), "oob");
3444   return &_node_bundling_base[n->_idx];
3445 }
3446 
3447 bool PhaseOutput::valid_bundle_info(const Node *n) {
3448   return (_node_bundling_limit > n->_idx);
3449 }
3450 
3451 //------------------------------frame_size_in_words-----------------------------
3452 // frame_slots in units of words
3453 int PhaseOutput::frame_size_in_words() const {
3454   // shift is 0 in LP32 and 1 in LP64
3455   const int shift = (LogBytesPerWord - LogBytesPerInt);
3456   int words = _frame_slots >> shift;
3457   assert( words << shift == _frame_slots, "frame size must be properly aligned in LP64" );
3458   return words;
3459 }
3460 
3461 // To bang the stack of this compiled method we use the stack size
3462 // that the interpreter would need in case of a deoptimization. This
3463 // removes the need to bang the stack in the deoptimization blob which
3464 // in turn simplifies stack overflow handling.
3465 int PhaseOutput::bang_size_in_bytes() const {
3466   return MAX2(frame_size_in_bytes() + os::extra_bang_size_in_bytes(), C->interpreter_frame_size());
3467 }
3468 
3469 //------------------------------dump_asm---------------------------------------
3470 // Dump formatted assembly
3471 #if defined(SUPPORT_OPTO_ASSEMBLY)
3472 void PhaseOutput::dump_asm_on(outputStream* st, int* pcs, uint pc_limit) {
3473 
3474   int pc_digits = 3; // #chars required for pc
3475   int sb_chars  = 3; // #chars for "start bundle" indicator
3476   int tab_size  = 8;
3477   if (pcs != nullptr) {
3478     int max_pc = 0;
3479     for (uint i = 0; i < pc_limit; i++) {
3480       max_pc = (max_pc < pcs[i]) ? pcs[i] : max_pc;
3481     }
3482     pc_digits  = ((max_pc < 4096) ? 3 : ((max_pc < 65536) ? 4 : ((max_pc < 65536*256) ? 6 : 8))); // #chars required for pc
3483   }
3484   int prefix_len = ((pc_digits + sb_chars + tab_size - 1)/tab_size)*tab_size;
3485 
3486   bool cut_short = false;
3487   st->print_cr("#");
3488   st->print("#  ");  C->tf()->dump_on(st);  st->cr();
3489   st->print_cr("#");
3490 
3491   // For all blocks
3492   int pc = 0x0;                 // Program counter
3493   char starts_bundle = ' ';
3494   C->regalloc()->dump_frame();
3495 
3496   Node *n = nullptr;
3497   for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
3498     if (VMThread::should_terminate()) {
3499       cut_short = true;
3500       break;
3501     }
3502     Block* block = C->cfg()->get_block(i);
3503     if (block->is_connector() && !Verbose) {
3504       continue;
3505     }
3506     n = block->head();
3507     if ((pcs != nullptr) && (n->_idx < pc_limit)) {
3508       pc = pcs[n->_idx];
3509       st->print("%*.*x", pc_digits, pc_digits, pc);
3510     }
3511     st->fill_to(prefix_len);
3512     block->dump_head(C->cfg(), st);
3513     if (block->is_connector()) {
3514       st->fill_to(prefix_len);
3515       st->print_cr("# Empty connector block");
3516     } else if (block->num_preds() == 2 && block->pred(1)->is_CatchProj() && block->pred(1)->as_CatchProj()->_con == CatchProjNode::fall_through_index) {
3517       st->fill_to(prefix_len);
3518       st->print_cr("# Block is sole successor of call");
3519     }
3520 
3521     // For all instructions
3522     Node *delay = nullptr;
3523     for (uint j = 0; j < block->number_of_nodes(); j++) {
3524       if (VMThread::should_terminate()) {
3525         cut_short = true;
3526         break;
3527       }
3528       n = block->get_node(j);
3529       if (valid_bundle_info(n)) {
3530         Bundle* bundle = node_bundling(n);
3531         if (bundle->used_in_unconditional_delay()) {
3532           delay = n;
3533           continue;
3534         }
3535         if (bundle->starts_bundle()) {
3536           starts_bundle = '+';
3537         }
3538       }
3539 
3540       if (WizardMode) {
3541         n->dump();
3542       }
3543 
3544       if( !n->is_Region() &&    // Dont print in the Assembly
3545           !n->is_Phi() &&       // a few noisely useless nodes
3546           !n->is_Proj() &&
3547           !n->is_MachTemp() &&
3548           !n->is_SafePointScalarObject() &&
3549           !n->is_Catch() &&     // Would be nice to print exception table targets
3550           !n->is_MergeMem() &&  // Not very interesting
3551           !n->is_top() &&       // Debug info table constants
3552           !(n->is_Con() && !n->is_Mach())// Debug info table constants
3553           ) {
3554         if ((pcs != nullptr) && (n->_idx < pc_limit)) {
3555           pc = pcs[n->_idx];
3556           st->print("%*.*x", pc_digits, pc_digits, pc);
3557         } else {
3558           st->fill_to(pc_digits);
3559         }
3560         st->print(" %c ", starts_bundle);
3561         starts_bundle = ' ';
3562         st->fill_to(prefix_len);
3563         n->format(C->regalloc(), st);
3564         st->cr();
3565       }
3566 
3567       // If we have an instruction with a delay slot, and have seen a delay,
3568       // then back up and print it
3569       if (valid_bundle_info(n) && node_bundling(n)->use_unconditional_delay()) {
3570         // Coverity finding - Explicit null dereferenced.
3571         guarantee(delay != nullptr, "no unconditional delay instruction");
3572         if (WizardMode) delay->dump();
3573 
3574         if (node_bundling(delay)->starts_bundle())
3575           starts_bundle = '+';
3576         if ((pcs != nullptr) && (n->_idx < pc_limit)) {
3577           pc = pcs[n->_idx];
3578           st->print("%*.*x", pc_digits, pc_digits, pc);
3579         } else {
3580           st->fill_to(pc_digits);
3581         }
3582         st->print(" %c ", starts_bundle);
3583         starts_bundle = ' ';
3584         st->fill_to(prefix_len);
3585         delay->format(C->regalloc(), st);
3586         st->cr();
3587         delay = nullptr;
3588       }
3589 
3590       // Dump the exception table as well
3591       if( n->is_Catch() && (Verbose || WizardMode) ) {
3592         // Print the exception table for this offset
3593         _handler_table.print_subtable_for(pc);
3594       }
3595       st->bol(); // Make sure we start on a new line
3596     }
3597     st->cr(); // one empty line between blocks
3598     assert(cut_short || delay == nullptr, "no unconditional delay branch");
3599   } // End of per-block dump
3600 
3601   if (cut_short)  st->print_cr("*** disassembly is cut short ***");
3602 }
3603 #endif
3604 
3605 #ifndef PRODUCT
3606 void PhaseOutput::print_statistics() {
3607   Scheduling::print_statistics();
3608 }
3609 #endif