1 /*
   2  * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.inline.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "code/compiledIC.hpp"
  29 #include "code/debugInfo.hpp"
  30 #include "code/debugInfoRec.hpp"
  31 #include "compiler/compileBroker.hpp"
  32 #include "compiler/compilerDirectives.hpp"
  33 #include "compiler/disassembler.hpp"
  34 #include "compiler/oopMap.hpp"
  35 #include "gc/shared/barrierSet.hpp"
  36 #include "gc/shared/c2/barrierSetC2.hpp"
  37 #include "memory/allocation.inline.hpp"
  38 #include "memory/allocation.hpp"
  39 #include "opto/ad.hpp"
  40 #include "opto/block.hpp"
  41 #include "opto/c2compiler.hpp"
  42 #include "opto/callnode.hpp"
  43 #include "opto/cfgnode.hpp"
  44 #include "opto/locknode.hpp"
  45 #include "opto/machnode.hpp"
  46 #include "opto/node.hpp"
  47 #include "opto/optoreg.hpp"
  48 #include "opto/output.hpp"
  49 #include "opto/regalloc.hpp"
  50 #include "opto/runtime.hpp"
  51 #include "opto/subnode.hpp"
  52 #include "opto/type.hpp"
  53 #include "runtime/handles.inline.hpp"
  54 #include "runtime/sharedRuntime.hpp"
  55 #include "utilities/macros.hpp"
  56 #include "utilities/powerOfTwo.hpp"
  57 #include "utilities/xmlstream.hpp"
  58 
  59 #ifndef PRODUCT
  60 #define DEBUG_ARG(x) , x
  61 #else
  62 #define DEBUG_ARG(x)
  63 #endif
  64 
  65 //------------------------------Scheduling----------------------------------
  66 // This class contains all the information necessary to implement instruction
  67 // scheduling and bundling.
  68 class Scheduling {
  69 
  70 private:
  71   // Arena to use
  72   Arena *_arena;
  73 
  74   // Control-Flow Graph info
  75   PhaseCFG *_cfg;
  76 
  77   // Register Allocation info
  78   PhaseRegAlloc *_regalloc;
  79 
  80   // Number of nodes in the method
  81   uint _node_bundling_limit;
  82 
  83   // List of scheduled nodes. Generated in reverse order
  84   Node_List _scheduled;
  85 
  86   // List of nodes currently available for choosing for scheduling
  87   Node_List _available;
  88 
  89   // For each instruction beginning a bundle, the number of following
  90   // nodes to be bundled with it.
  91   Bundle *_node_bundling_base;
  92 
  93   // Mapping from register to Node
  94   Node_List _reg_node;
  95 
  96   // Free list for pinch nodes.
  97   Node_List _pinch_free_list;
  98 
  99   // Latency from the beginning of the containing basic block (base 1)
 100   // for each node.
 101   unsigned short *_node_latency;
 102 
 103   // Number of uses of this node within the containing basic block.
 104   short *_uses;
 105 
 106   // Schedulable portion of current block.  Skips Region/Phi/CreateEx up
 107   // front, branch+proj at end.  Also skips Catch/CProj (same as
 108   // branch-at-end), plus just-prior exception-throwing call.
 109   uint _bb_start, _bb_end;
 110 
 111   // Latency from the end of the basic block as scheduled
 112   unsigned short *_current_latency;
 113 
 114   // Remember the next node
 115   Node *_next_node;
 116 
 117   // Use this for an unconditional branch delay slot
 118   Node *_unconditional_delay_slot;
 119 
 120   // Pointer to a Nop
 121   MachNopNode *_nop;
 122 
 123   // Length of the current bundle, in instructions
 124   uint _bundle_instr_count;
 125 
 126   // Current Cycle number, for computing latencies and bundling
 127   uint _bundle_cycle_number;
 128 
 129   // Bundle information
 130   Pipeline_Use_Element _bundle_use_elements[resource_count];
 131   Pipeline_Use         _bundle_use;
 132 
 133   // Dump the available list
 134   void dump_available() const;
 135 
 136 public:
 137   Scheduling(Arena *arena, Compile &compile);
 138 
 139   // Destructor
 140   NOT_PRODUCT( ~Scheduling(); )
 141 
 142   // Step ahead "i" cycles
 143   void step(uint i);
 144 
 145   // Step ahead 1 cycle, and clear the bundle state (for example,
 146   // at a branch target)
 147   void step_and_clear();
 148 
 149   Bundle* node_bundling(const Node *n) {
 150     assert(valid_bundle_info(n), "oob");
 151     return (&_node_bundling_base[n->_idx]);
 152   }
 153 
 154   bool valid_bundle_info(const Node *n) const {
 155     return (_node_bundling_limit > n->_idx);
 156   }
 157 
 158   bool starts_bundle(const Node *n) const {
 159     return (_node_bundling_limit > n->_idx && _node_bundling_base[n->_idx].starts_bundle());
 160   }
 161 
 162   // Do the scheduling
 163   void DoScheduling();
 164 
 165   // Compute the local latencies walking forward over the list of
 166   // nodes for a basic block
 167   void ComputeLocalLatenciesForward(const Block *bb);
 168 
 169   // Compute the register antidependencies within a basic block
 170   void ComputeRegisterAntidependencies(Block *bb);
 171   void verify_do_def( Node *n, OptoReg::Name def, const char *msg );
 172   void verify_good_schedule( Block *b, const char *msg );
 173   void anti_do_def( Block *b, Node *def, OptoReg::Name def_reg, int is_def );
 174   void anti_do_use( Block *b, Node *use, OptoReg::Name use_reg );
 175 
 176   // Add a node to the current bundle
 177   void AddNodeToBundle(Node *n, const Block *bb);
 178 
 179   // Add a node to the list of available nodes
 180   void AddNodeToAvailableList(Node *n);
 181 
 182   // Compute the local use count for the nodes in a block, and compute
 183   // the list of instructions with no uses in the block as available
 184   void ComputeUseCount(const Block *bb);
 185 
 186   // Choose an instruction from the available list to add to the bundle
 187   Node * ChooseNodeToBundle();
 188 
 189   // See if this Node fits into the currently accumulating bundle
 190   bool NodeFitsInBundle(Node *n);
 191 
 192   // Decrement the use count for a node
 193  void DecrementUseCounts(Node *n, const Block *bb);
 194 
 195   // Garbage collect pinch nodes for reuse by other blocks.
 196   void garbage_collect_pinch_nodes();
 197   // Clean up a pinch node for reuse (helper for above).
 198   void cleanup_pinch( Node *pinch );
 199 
 200   // Information for statistics gathering
 201 #ifndef PRODUCT
 202 private:
 203   // Gather information on size of nops relative to total
 204   uint _branches, _unconditional_delays;
 205 
 206   static uint _total_nop_size, _total_method_size;
 207   static uint _total_branches, _total_unconditional_delays;
 208   static uint _total_instructions_per_bundle[Pipeline::_max_instrs_per_cycle+1];
 209 
 210 public:
 211   static void print_statistics();
 212 
 213   static void increment_instructions_per_bundle(uint i) {
 214     _total_instructions_per_bundle[i]++;
 215   }
 216 
 217   static void increment_nop_size(uint s) {
 218     _total_nop_size += s;
 219   }
 220 
 221   static void increment_method_size(uint s) {
 222     _total_method_size += s;
 223   }
 224 #endif
 225 
 226 };
 227 
 228 volatile int C2SafepointPollStubTable::_stub_size = 0;
 229 
 230 Label& C2SafepointPollStubTable::add_safepoint(uintptr_t safepoint_offset) {
 231   C2SafepointPollStub* entry = new (Compile::current()->comp_arena()) C2SafepointPollStub(safepoint_offset);
 232   _safepoints.append(entry);
 233   return entry->_stub_label;
 234 }
 235 
 236 void C2SafepointPollStubTable::emit(CodeBuffer& cb) {
 237   MacroAssembler masm(&cb);
 238   for (int i = _safepoints.length() - 1; i >= 0; i--) {
 239     // Make sure there is enough space in the code buffer
 240     if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == NULL) {
 241       ciEnv::current()->record_failure("CodeCache is full");
 242       return;
 243     }
 244 
 245     C2SafepointPollStub* entry = _safepoints.at(i);
 246     emit_stub(masm, entry);
 247   }
 248 }
 249 
 250 int C2SafepointPollStubTable::stub_size_lazy() const {
 251   int size = Atomic::load(&_stub_size);
 252 
 253   if (size != 0) {
 254     return size;
 255   }
 256 
 257   Compile* const C = Compile::current();
 258   BufferBlob* const blob = C->output()->scratch_buffer_blob();
 259   CodeBuffer cb(blob->content_begin(), C->output()->scratch_buffer_code_size());
 260   MacroAssembler masm(&cb);
 261   C2SafepointPollStub* entry = _safepoints.at(0);
 262   emit_stub(masm, entry);
 263   size += cb.insts_size();
 264 
 265   Atomic::store(&_stub_size, size);
 266 
 267   return size;
 268 }
 269 
 270 int C2SafepointPollStubTable::estimate_stub_size() const {
 271   if (_safepoints.length() == 0) {
 272     return 0;
 273   }
 274 
 275   int result = stub_size_lazy() * _safepoints.length();
 276 
 277 #ifdef ASSERT
 278   Compile* const C = Compile::current();
 279   BufferBlob* const blob = C->output()->scratch_buffer_blob();
 280   int size = 0;
 281 
 282   for (int i = _safepoints.length() - 1; i >= 0; i--) {
 283     CodeBuffer cb(blob->content_begin(), C->output()->scratch_buffer_code_size());
 284     MacroAssembler masm(&cb);
 285     C2SafepointPollStub* entry = _safepoints.at(i);
 286     emit_stub(masm, entry);
 287     size += cb.insts_size();
 288   }
 289   assert(size == result, "stubs should not have variable size");
 290 #endif
 291 
 292   return result;
 293 }
 294 
 295 PhaseOutput::PhaseOutput()
 296   : Phase(Phase::Output),
 297     _code_buffer("Compile::Fill_buffer"),
 298     _first_block_size(0),
 299     _handler_table(),
 300     _inc_table(),
 301     _oop_map_set(NULL),
 302     _scratch_buffer_blob(NULL),
 303     _scratch_locs_memory(NULL),
 304     _scratch_const_size(-1),
 305     _in_scratch_emit_size(false),
 306     _frame_slots(0),
 307     _code_offsets(),
 308     _node_bundling_limit(0),
 309     _node_bundling_base(NULL),
 310     _orig_pc_slot(0),
 311     _orig_pc_slot_offset_in_bytes(0),
 312     _buf_sizes(),
 313     _block(NULL),
 314     _index(0) {
 315   C->set_output(this);
 316   if (C->stub_name() == NULL) {
 317     _orig_pc_slot = C->fixed_slots() - (sizeof(address) / VMRegImpl::stack_slot_size);
 318   }
 319 }
 320 
 321 PhaseOutput::~PhaseOutput() {
 322   C->set_output(NULL);
 323   if (_scratch_buffer_blob != NULL) {
 324     BufferBlob::free(_scratch_buffer_blob);
 325   }
 326 }
 327 
 328 void PhaseOutput::perform_mach_node_analysis() {
 329   // Late barrier analysis must be done after schedule and bundle
 330   // Otherwise liveness based spilling will fail
 331   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
 332   bs->late_barrier_analysis();
 333 
 334   pd_perform_mach_node_analysis();
 335 }
 336 
 337 // Convert Nodes to instruction bits and pass off to the VM
 338 void PhaseOutput::Output() {
 339   // RootNode goes
 340   assert( C->cfg()->get_root_block()->number_of_nodes() == 0, "" );
 341 
 342   // The number of new nodes (mostly MachNop) is proportional to
 343   // the number of java calls and inner loops which are aligned.
 344   if ( C->check_node_count((NodeLimitFudgeFactor + C->java_calls()*3 +
 345                             C->inner_loops()*(OptoLoopAlignment-1)),
 346                            "out of nodes before code generation" ) ) {
 347     return;
 348   }
 349   // Make sure I can find the Start Node
 350   Block *entry = C->cfg()->get_block(1);
 351   Block *broot = C->cfg()->get_root_block();
 352 
 353   const StartNode *start = entry->head()->as_Start();
 354 
 355   // Replace StartNode with prolog
 356   MachPrologNode *prolog = new MachPrologNode();
 357   entry->map_node(prolog, 0);
 358   C->cfg()->map_node_to_block(prolog, entry);
 359   C->cfg()->unmap_node_from_block(start); // start is no longer in any block
 360 
 361   // Virtual methods need an unverified entry point
 362 
 363   if( C->is_osr_compilation() ) {
 364     if( PoisonOSREntry ) {
 365       // TODO: Should use a ShouldNotReachHereNode...
 366       C->cfg()->insert( broot, 0, new MachBreakpointNode() );
 367     }
 368   } else {
 369     if( C->method() && !C->method()->flags().is_static() ) {
 370       // Insert unvalidated entry point
 371       C->cfg()->insert( broot, 0, new MachUEPNode() );
 372     }
 373 
 374   }
 375 
 376   // Break before main entry point
 377   if ((C->method() && C->directive()->BreakAtExecuteOption) ||
 378       (OptoBreakpoint && C->is_method_compilation())       ||
 379       (OptoBreakpointOSR && C->is_osr_compilation())       ||
 380       (OptoBreakpointC2R && !C->method())                   ) {
 381     // checking for C->method() means that OptoBreakpoint does not apply to
 382     // runtime stubs or frame converters
 383     C->cfg()->insert( entry, 1, new MachBreakpointNode() );
 384   }
 385 
 386   // Insert epilogs before every return
 387   for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
 388     Block* block = C->cfg()->get_block(i);
 389     if (!block->is_connector() && block->non_connector_successor(0) == C->cfg()->get_root_block()) { // Found a program exit point?
 390       Node* m = block->end();
 391       if (m->is_Mach() && m->as_Mach()->ideal_Opcode() != Op_Halt) {
 392         MachEpilogNode* epilog = new MachEpilogNode(m->as_Mach()->ideal_Opcode() == Op_Return);
 393         block->add_inst(epilog);
 394         C->cfg()->map_node_to_block(epilog, block);
 395       }
 396     }
 397   }
 398 
 399   // Keeper of sizing aspects
 400   _buf_sizes = BufferSizingData();
 401 
 402   // Initialize code buffer
 403   estimate_buffer_size(_buf_sizes._const);
 404   if (C->failing()) return;
 405 
 406   // Pre-compute the length of blocks and replace
 407   // long branches with short if machine supports it.
 408   // Must be done before ScheduleAndBundle due to SPARC delay slots
 409   uint* blk_starts = NEW_RESOURCE_ARRAY(uint, C->cfg()->number_of_blocks() + 1);
 410   blk_starts[0] = 0;
 411   shorten_branches(blk_starts);
 412 
 413   ScheduleAndBundle();
 414   if (C->failing()) {
 415     return;
 416   }
 417 
 418   perform_mach_node_analysis();
 419 
 420   // Complete sizing of codebuffer
 421   CodeBuffer* cb = init_buffer();
 422   if (cb == NULL || C->failing()) {
 423     return;
 424   }
 425 
 426   BuildOopMaps();
 427 
 428   if (C->failing())  {
 429     return;
 430   }
 431 
 432   fill_buffer(cb, blk_starts);
 433 }
 434 
 435 bool PhaseOutput::need_stack_bang(int frame_size_in_bytes) const {
 436   // Determine if we need to generate a stack overflow check.
 437   // Do it if the method is not a stub function and
 438   // has java calls or has frame size > vm_page_size/8.
 439   // The debug VM checks that deoptimization doesn't trigger an
 440   // unexpected stack overflow (compiled method stack banging should
 441   // guarantee it doesn't happen) so we always need the stack bang in
 442   // a debug VM.
 443   return (C->stub_function() == NULL &&
 444           (C->has_java_calls() || frame_size_in_bytes > os::vm_page_size()>>3
 445            DEBUG_ONLY(|| true)));
 446 }
 447 
 448 bool PhaseOutput::need_register_stack_bang() const {
 449   // Determine if we need to generate a register stack overflow check.
 450   // This is only used on architectures which have split register
 451   // and memory stacks (ie. IA64).
 452   // Bang if the method is not a stub function and has java calls
 453   return (C->stub_function() == NULL && C->has_java_calls());
 454 }
 455 
 456 
 457 // Compute the size of first NumberOfLoopInstrToAlign instructions at the top
 458 // of a loop. When aligning a loop we need to provide enough instructions
 459 // in cpu's fetch buffer to feed decoders. The loop alignment could be
 460 // avoided if we have enough instructions in fetch buffer at the head of a loop.
 461 // By default, the size is set to 999999 by Block's constructor so that
 462 // a loop will be aligned if the size is not reset here.
 463 //
 464 // Note: Mach instructions could contain several HW instructions
 465 // so the size is estimated only.
 466 //
 467 void PhaseOutput::compute_loop_first_inst_sizes() {
 468   // The next condition is used to gate the loop alignment optimization.
 469   // Don't aligned a loop if there are enough instructions at the head of a loop
 470   // or alignment padding is larger then MaxLoopPad. By default, MaxLoopPad
 471   // is equal to OptoLoopAlignment-1 except on new Intel cpus, where it is
 472   // equal to 11 bytes which is the largest address NOP instruction.
 473   if (MaxLoopPad < OptoLoopAlignment - 1) {
 474     uint last_block = C->cfg()->number_of_blocks() - 1;
 475     for (uint i = 1; i <= last_block; i++) {
 476       Block* block = C->cfg()->get_block(i);
 477       // Check the first loop's block which requires an alignment.
 478       if (block->loop_alignment() > (uint)relocInfo::addr_unit()) {
 479         uint sum_size = 0;
 480         uint inst_cnt = NumberOfLoopInstrToAlign;
 481         inst_cnt = block->compute_first_inst_size(sum_size, inst_cnt, C->regalloc());
 482 
 483         // Check subsequent fallthrough blocks if the loop's first
 484         // block(s) does not have enough instructions.
 485         Block *nb = block;
 486         while(inst_cnt > 0 &&
 487               i < last_block &&
 488               !C->cfg()->get_block(i + 1)->has_loop_alignment() &&
 489               !nb->has_successor(block)) {
 490           i++;
 491           nb = C->cfg()->get_block(i);
 492           inst_cnt  = nb->compute_first_inst_size(sum_size, inst_cnt, C->regalloc());
 493         } // while( inst_cnt > 0 && i < last_block  )
 494 
 495         block->set_first_inst_size(sum_size);
 496       } // f( b->head()->is_Loop() )
 497     } // for( i <= last_block )
 498   } // if( MaxLoopPad < OptoLoopAlignment-1 )
 499 }
 500 
 501 // The architecture description provides short branch variants for some long
 502 // branch instructions. Replace eligible long branches with short branches.
 503 void PhaseOutput::shorten_branches(uint* blk_starts) {
 504 
 505   Compile::TracePhase tp("shorten branches", &timers[_t_shortenBranches]);
 506 
 507   // Compute size of each block, method size, and relocation information size
 508   uint nblocks  = C->cfg()->number_of_blocks();
 509 
 510   uint*      jmp_offset = NEW_RESOURCE_ARRAY(uint,nblocks);
 511   uint*      jmp_size   = NEW_RESOURCE_ARRAY(uint,nblocks);
 512   int*       jmp_nidx   = NEW_RESOURCE_ARRAY(int ,nblocks);
 513 
 514   // Collect worst case block paddings
 515   int* block_worst_case_pad = NEW_RESOURCE_ARRAY(int, nblocks);
 516   memset(block_worst_case_pad, 0, nblocks * sizeof(int));
 517 
 518   DEBUG_ONLY( uint *jmp_target = NEW_RESOURCE_ARRAY(uint,nblocks); )
 519   DEBUG_ONLY( uint *jmp_rule = NEW_RESOURCE_ARRAY(uint,nblocks); )
 520 
 521   bool has_short_branch_candidate = false;
 522 
 523   // Initialize the sizes to 0
 524   int code_size  = 0;          // Size in bytes of generated code
 525   int stub_size  = 0;          // Size in bytes of all stub entries
 526   // Size in bytes of all relocation entries, including those in local stubs.
 527   // Start with 2-bytes of reloc info for the unvalidated entry point
 528   int reloc_size = 1;          // Number of relocation entries
 529 
 530   // Make three passes.  The first computes pessimistic blk_starts,
 531   // relative jmp_offset and reloc_size information.  The second performs
 532   // short branch substitution using the pessimistic sizing.  The
 533   // third inserts nops where needed.
 534 
 535   // Step one, perform a pessimistic sizing pass.
 536   uint last_call_adr = max_juint;
 537   uint last_avoid_back_to_back_adr = max_juint;
 538   uint nop_size = (new MachNopNode())->size(C->regalloc());
 539   for (uint i = 0; i < nblocks; i++) { // For all blocks
 540     Block* block = C->cfg()->get_block(i);
 541     _block = block;
 542 
 543     // During short branch replacement, we store the relative (to blk_starts)
 544     // offset of jump in jmp_offset, rather than the absolute offset of jump.
 545     // This is so that we do not need to recompute sizes of all nodes when
 546     // we compute correct blk_starts in our next sizing pass.
 547     jmp_offset[i] = 0;
 548     jmp_size[i]   = 0;
 549     jmp_nidx[i]   = -1;
 550     DEBUG_ONLY( jmp_target[i] = 0; )
 551     DEBUG_ONLY( jmp_rule[i]   = 0; )
 552 
 553     // Sum all instruction sizes to compute block size
 554     uint last_inst = block->number_of_nodes();
 555     uint blk_size = 0;
 556     for (uint j = 0; j < last_inst; j++) {
 557       _index = j;
 558       Node* nj = block->get_node(_index);
 559       // Handle machine instruction nodes
 560       if (nj->is_Mach()) {
 561         MachNode* mach = nj->as_Mach();
 562         blk_size += (mach->alignment_required() - 1) * relocInfo::addr_unit(); // assume worst case padding
 563         reloc_size += mach->reloc();
 564         if (mach->is_MachCall()) {
 565           // add size information for trampoline stub
 566           // class CallStubImpl is platform-specific and defined in the *.ad files.
 567           stub_size  += CallStubImpl::size_call_trampoline();
 568           reloc_size += CallStubImpl::reloc_call_trampoline();
 569 
 570           MachCallNode *mcall = mach->as_MachCall();
 571           // This destination address is NOT PC-relative
 572 
 573           mcall->method_set((intptr_t)mcall->entry_point());
 574 
 575           if (mcall->is_MachCallJava() && mcall->as_MachCallJava()->_method) {
 576             stub_size  += CompiledStaticCall::to_interp_stub_size();
 577             reloc_size += CompiledStaticCall::reloc_to_interp_stub();
 578           }
 579         } else if (mach->is_MachSafePoint()) {
 580           // If call/safepoint are adjacent, account for possible
 581           // nop to disambiguate the two safepoints.
 582           // ScheduleAndBundle() can rearrange nodes in a block,
 583           // check for all offsets inside this block.
 584           if (last_call_adr >= blk_starts[i]) {
 585             blk_size += nop_size;
 586           }
 587         }
 588         if (mach->avoid_back_to_back(MachNode::AVOID_BEFORE)) {
 589           // Nop is inserted between "avoid back to back" instructions.
 590           // ScheduleAndBundle() can rearrange nodes in a block,
 591           // check for all offsets inside this block.
 592           if (last_avoid_back_to_back_adr >= blk_starts[i]) {
 593             blk_size += nop_size;
 594           }
 595         }
 596         if (mach->may_be_short_branch()) {
 597           if (!nj->is_MachBranch()) {
 598 #ifndef PRODUCT
 599             nj->dump(3);
 600 #endif
 601             Unimplemented();
 602           }
 603           assert(jmp_nidx[i] == -1, "block should have only one branch");
 604           jmp_offset[i] = blk_size;
 605           jmp_size[i]   = nj->size(C->regalloc());
 606           jmp_nidx[i]   = j;
 607           has_short_branch_candidate = true;
 608         }
 609       }
 610       blk_size += nj->size(C->regalloc());
 611       // Remember end of call offset
 612       if (nj->is_MachCall() && !nj->is_MachCallLeaf()) {
 613         last_call_adr = blk_starts[i]+blk_size;
 614       }
 615       // Remember end of avoid_back_to_back offset
 616       if (nj->is_Mach() && nj->as_Mach()->avoid_back_to_back(MachNode::AVOID_AFTER)) {
 617         last_avoid_back_to_back_adr = blk_starts[i]+blk_size;
 618       }
 619     }
 620 
 621     // When the next block starts a loop, we may insert pad NOP
 622     // instructions.  Since we cannot know our future alignment,
 623     // assume the worst.
 624     if (i < nblocks - 1) {
 625       Block* nb = C->cfg()->get_block(i + 1);
 626       int max_loop_pad = nb->code_alignment()-relocInfo::addr_unit();
 627       if (max_loop_pad > 0) {
 628         assert(is_power_of_2(max_loop_pad+relocInfo::addr_unit()), "");
 629         // Adjust last_call_adr and/or last_avoid_back_to_back_adr.
 630         // If either is the last instruction in this block, bump by
 631         // max_loop_pad in lock-step with blk_size, so sizing
 632         // calculations in subsequent blocks still can conservatively
 633         // detect that it may the last instruction in this block.
 634         if (last_call_adr == blk_starts[i]+blk_size) {
 635           last_call_adr += max_loop_pad;
 636         }
 637         if (last_avoid_back_to_back_adr == blk_starts[i]+blk_size) {
 638           last_avoid_back_to_back_adr += max_loop_pad;
 639         }
 640         blk_size += max_loop_pad;
 641         block_worst_case_pad[i + 1] = max_loop_pad;
 642       }
 643     }
 644 
 645     // Save block size; update total method size
 646     blk_starts[i+1] = blk_starts[i]+blk_size;
 647   }
 648 
 649   // Step two, replace eligible long jumps.
 650   bool progress = true;
 651   uint last_may_be_short_branch_adr = max_juint;
 652   while (has_short_branch_candidate && progress) {
 653     progress = false;
 654     has_short_branch_candidate = false;
 655     int adjust_block_start = 0;
 656     for (uint i = 0; i < nblocks; i++) {
 657       Block* block = C->cfg()->get_block(i);
 658       int idx = jmp_nidx[i];
 659       MachNode* mach = (idx == -1) ? NULL: block->get_node(idx)->as_Mach();
 660       if (mach != NULL && mach->may_be_short_branch()) {
 661 #ifdef ASSERT
 662         assert(jmp_size[i] > 0 && mach->is_MachBranch(), "sanity");
 663         int j;
 664         // Find the branch; ignore trailing NOPs.
 665         for (j = block->number_of_nodes()-1; j>=0; j--) {
 666           Node* n = block->get_node(j);
 667           if (!n->is_Mach() || n->as_Mach()->ideal_Opcode() != Op_Con)
 668             break;
 669         }
 670         assert(j >= 0 && j == idx && block->get_node(j) == (Node*)mach, "sanity");
 671 #endif
 672         int br_size = jmp_size[i];
 673         int br_offs = blk_starts[i] + jmp_offset[i];
 674 
 675         // This requires the TRUE branch target be in succs[0]
 676         uint bnum = block->non_connector_successor(0)->_pre_order;
 677         int offset = blk_starts[bnum] - br_offs;
 678         if (bnum > i) { // adjust following block's offset
 679           offset -= adjust_block_start;
 680         }
 681 
 682         // This block can be a loop header, account for the padding
 683         // in the previous block.
 684         int block_padding = block_worst_case_pad[i];
 685         assert(i == 0 || block_padding == 0 || br_offs >= block_padding, "Should have at least a padding on top");
 686         // In the following code a nop could be inserted before
 687         // the branch which will increase the backward distance.
 688         bool needs_padding = ((uint)(br_offs - block_padding) == last_may_be_short_branch_adr);
 689         assert(!needs_padding || jmp_offset[i] == 0, "padding only branches at the beginning of block");
 690 
 691         if (needs_padding && offset <= 0)
 692           offset -= nop_size;
 693 
 694         if (C->matcher()->is_short_branch_offset(mach->rule(), br_size, offset)) {
 695           // We've got a winner.  Replace this branch.
 696           MachNode* replacement = mach->as_MachBranch()->short_branch_version();
 697 
 698           // Update the jmp_size.
 699           int new_size = replacement->size(C->regalloc());
 700           int diff     = br_size - new_size;
 701           assert(diff >= (int)nop_size, "short_branch size should be smaller");
 702           // Conservatively take into account padding between
 703           // avoid_back_to_back branches. Previous branch could be
 704           // converted into avoid_back_to_back branch during next
 705           // rounds.
 706           if (needs_padding && replacement->avoid_back_to_back(MachNode::AVOID_BEFORE)) {
 707             jmp_offset[i] += nop_size;
 708             diff -= nop_size;
 709           }
 710           adjust_block_start += diff;
 711           block->map_node(replacement, idx);
 712           mach->subsume_by(replacement, C);
 713           mach = replacement;
 714           progress = true;
 715 
 716           jmp_size[i] = new_size;
 717           DEBUG_ONLY( jmp_target[i] = bnum; );
 718           DEBUG_ONLY( jmp_rule[i] = mach->rule(); );
 719         } else {
 720           // The jump distance is not short, try again during next iteration.
 721           has_short_branch_candidate = true;
 722         }
 723       } // (mach->may_be_short_branch())
 724       if (mach != NULL && (mach->may_be_short_branch() ||
 725                            mach->avoid_back_to_back(MachNode::AVOID_AFTER))) {
 726         last_may_be_short_branch_adr = blk_starts[i] + jmp_offset[i] + jmp_size[i];
 727       }
 728       blk_starts[i+1] -= adjust_block_start;
 729     }
 730   }
 731 
 732 #ifdef ASSERT
 733   for (uint i = 0; i < nblocks; i++) { // For all blocks
 734     if (jmp_target[i] != 0) {
 735       int br_size = jmp_size[i];
 736       int offset = blk_starts[jmp_target[i]]-(blk_starts[i] + jmp_offset[i]);
 737       if (!C->matcher()->is_short_branch_offset(jmp_rule[i], br_size, offset)) {
 738         tty->print_cr("target (%d) - jmp_offset(%d) = offset (%d), jump_size(%d), jmp_block B%d, target_block B%d", blk_starts[jmp_target[i]], blk_starts[i] + jmp_offset[i], offset, br_size, i, jmp_target[i]);
 739       }
 740       assert(C->matcher()->is_short_branch_offset(jmp_rule[i], br_size, offset), "Displacement too large for short jmp");
 741     }
 742   }
 743 #endif
 744 
 745   // Step 3, compute the offsets of all blocks, will be done in fill_buffer()
 746   // after ScheduleAndBundle().
 747 
 748   // ------------------
 749   // Compute size for code buffer
 750   code_size = blk_starts[nblocks];
 751 
 752   // Relocation records
 753   reloc_size += 1;              // Relo entry for exception handler
 754 
 755   // Adjust reloc_size to number of record of relocation info
 756   // Min is 2 bytes, max is probably 6 or 8, with a tax up to 25% for
 757   // a relocation index.
 758   // The CodeBuffer will expand the locs array if this estimate is too low.
 759   reloc_size *= 10 / sizeof(relocInfo);
 760 
 761   _buf_sizes._reloc = reloc_size;
 762   _buf_sizes._code  = code_size;
 763   _buf_sizes._stub  = stub_size;
 764 }
 765 
 766 //------------------------------FillLocArray-----------------------------------
 767 // Create a bit of debug info and append it to the array.  The mapping is from
 768 // Java local or expression stack to constant, register or stack-slot.  For
 769 // doubles, insert 2 mappings and return 1 (to tell the caller that the next
 770 // entry has been taken care of and caller should skip it).
 771 static LocationValue *new_loc_value( PhaseRegAlloc *ra, OptoReg::Name regnum, Location::Type l_type ) {
 772   // This should never have accepted Bad before
 773   assert(OptoReg::is_valid(regnum), "location must be valid");
 774   return (OptoReg::is_reg(regnum))
 775          ? new LocationValue(Location::new_reg_loc(l_type, OptoReg::as_VMReg(regnum)) )
 776          : new LocationValue(Location::new_stk_loc(l_type,  ra->reg2offset(regnum)));
 777 }
 778 
 779 
 780 ObjectValue*
 781 PhaseOutput::sv_for_node_id(GrowableArray<ScopeValue*> *objs, int id) {
 782   for (int i = 0; i < objs->length(); i++) {
 783     assert(objs->at(i)->is_object(), "corrupt object cache");
 784     ObjectValue* sv = (ObjectValue*) objs->at(i);
 785     if (sv->id() == id) {
 786       return sv;
 787     }
 788   }
 789   // Otherwise..
 790   return NULL;
 791 }
 792 
 793 void PhaseOutput::set_sv_for_object_node(GrowableArray<ScopeValue*> *objs,
 794                                      ObjectValue* sv ) {
 795   assert(sv_for_node_id(objs, sv->id()) == NULL, "Precondition");
 796   objs->append(sv);
 797 }
 798 
 799 
 800 void PhaseOutput::FillLocArray( int idx, MachSafePointNode* sfpt, Node *local,
 801                             GrowableArray<ScopeValue*> *array,
 802                             GrowableArray<ScopeValue*> *objs ) {
 803   assert( local, "use _top instead of null" );
 804   if (array->length() != idx) {
 805     assert(array->length() == idx + 1, "Unexpected array count");
 806     // Old functionality:
 807     //   return
 808     // New functionality:
 809     //   Assert if the local is not top. In product mode let the new node
 810     //   override the old entry.
 811     assert(local == C->top(), "LocArray collision");
 812     if (local == C->top()) {
 813       return;
 814     }
 815     array->pop();
 816   }
 817   const Type *t = local->bottom_type();
 818 
 819   // Is it a safepoint scalar object node?
 820   if (local->is_SafePointScalarObject()) {
 821     SafePointScalarObjectNode* spobj = local->as_SafePointScalarObject();
 822 
 823     ObjectValue* sv = sv_for_node_id(objs, spobj->_idx);
 824     if (sv == NULL) {
 825       ciKlass* cik = t->is_oopptr()->klass();
 826       assert(cik->is_instance_klass() ||
 827              cik->is_array_klass(), "Not supported allocation.");
 828       ScopeValue* klass_sv = new ConstantOopWriteValue(cik->java_mirror()->constant_encoding());
 829       sv = spobj->is_auto_box() ? new AutoBoxObjectValue(spobj->_idx, klass_sv)
 830                                     : new ObjectValue(spobj->_idx, klass_sv);
 831       set_sv_for_object_node(objs, sv);
 832 
 833       uint first_ind = spobj->first_index(sfpt->jvms());
 834       for (uint i = 0; i < spobj->n_fields(); i++) {
 835         Node* fld_node = sfpt->in(first_ind+i);
 836         (void)FillLocArray(sv->field_values()->length(), sfpt, fld_node, sv->field_values(), objs);
 837       }
 838     }
 839     array->append(sv);
 840     return;
 841   }
 842 
 843   // Grab the register number for the local
 844   OptoReg::Name regnum = C->regalloc()->get_reg_first(local);
 845   if( OptoReg::is_valid(regnum) ) {// Got a register/stack?
 846     // Record the double as two float registers.
 847     // The register mask for such a value always specifies two adjacent
 848     // float registers, with the lower register number even.
 849     // Normally, the allocation of high and low words to these registers
 850     // is irrelevant, because nearly all operations on register pairs
 851     // (e.g., StoreD) treat them as a single unit.
 852     // Here, we assume in addition that the words in these two registers
 853     // stored "naturally" (by operations like StoreD and double stores
 854     // within the interpreter) such that the lower-numbered register
 855     // is written to the lower memory address.  This may seem like
 856     // a machine dependency, but it is not--it is a requirement on
 857     // the author of the <arch>.ad file to ensure that, for every
 858     // even/odd double-register pair to which a double may be allocated,
 859     // the word in the even single-register is stored to the first
 860     // memory word.  (Note that register numbers are completely
 861     // arbitrary, and are not tied to any machine-level encodings.)
 862 #ifdef _LP64
 863     if( t->base() == Type::DoubleBot || t->base() == Type::DoubleCon ) {
 864       array->append(new ConstantIntValue((jint)0));
 865       array->append(new_loc_value( C->regalloc(), regnum, Location::dbl ));
 866     } else if ( t->base() == Type::Long ) {
 867       array->append(new ConstantIntValue((jint)0));
 868       array->append(new_loc_value( C->regalloc(), regnum, Location::lng ));
 869     } else if ( t->base() == Type::RawPtr ) {
 870       // jsr/ret return address which must be restored into a the full
 871       // width 64-bit stack slot.
 872       array->append(new_loc_value( C->regalloc(), regnum, Location::lng ));
 873     }
 874 #else //_LP64
 875     if( t->base() == Type::DoubleBot || t->base() == Type::DoubleCon || t->base() == Type::Long ) {
 876       // Repack the double/long as two jints.
 877       // The convention the interpreter uses is that the second local
 878       // holds the first raw word of the native double representation.
 879       // This is actually reasonable, since locals and stack arrays
 880       // grow downwards in all implementations.
 881       // (If, on some machine, the interpreter's Java locals or stack
 882       // were to grow upwards, the embedded doubles would be word-swapped.)
 883       array->append(new_loc_value( C->regalloc(), OptoReg::add(regnum,1), Location::normal ));
 884       array->append(new_loc_value( C->regalloc(),              regnum   , Location::normal ));
 885     }
 886 #endif //_LP64
 887     else if( (t->base() == Type::FloatBot || t->base() == Type::FloatCon) &&
 888              OptoReg::is_reg(regnum) ) {
 889       array->append(new_loc_value( C->regalloc(), regnum, Matcher::float_in_double()
 890                                                       ? Location::float_in_dbl : Location::normal ));
 891     } else if( t->base() == Type::Int && OptoReg::is_reg(regnum) ) {
 892       array->append(new_loc_value( C->regalloc(), regnum, Matcher::int_in_long
 893                                                       ? Location::int_in_long : Location::normal ));
 894     } else if( t->base() == Type::NarrowOop ) {
 895       array->append(new_loc_value( C->regalloc(), regnum, Location::narrowoop ));
 896     } else if (t->base() == Type::VectorA || t->base() == Type::VectorS ||
 897                t->base() == Type::VectorD || t->base() == Type::VectorX ||
 898                t->base() == Type::VectorY || t->base() == Type::VectorZ) {
 899       array->append(new_loc_value( C->regalloc(), regnum, Location::vector ));
 900     } else {
 901       array->append(new_loc_value( C->regalloc(), regnum, C->regalloc()->is_oop(local) ? Location::oop : Location::normal ));
 902     }
 903     return;
 904   }
 905 
 906   // No register.  It must be constant data.
 907   switch (t->base()) {
 908     case Type::Half:              // Second half of a double
 909       ShouldNotReachHere();       // Caller should skip 2nd halves
 910       break;
 911     case Type::AnyPtr:
 912       array->append(new ConstantOopWriteValue(NULL));
 913       break;
 914     case Type::AryPtr:
 915     case Type::InstPtr:          // fall through
 916       array->append(new ConstantOopWriteValue(t->isa_oopptr()->const_oop()->constant_encoding()));
 917       break;
 918     case Type::NarrowOop:
 919       if (t == TypeNarrowOop::NULL_PTR) {
 920         array->append(new ConstantOopWriteValue(NULL));
 921       } else {
 922         array->append(new ConstantOopWriteValue(t->make_ptr()->isa_oopptr()->const_oop()->constant_encoding()));
 923       }
 924       break;
 925     case Type::Int:
 926       array->append(new ConstantIntValue(t->is_int()->get_con()));
 927       break;
 928     case Type::RawPtr:
 929       // A return address (T_ADDRESS).
 930       assert((intptr_t)t->is_ptr()->get_con() < (intptr_t)0x10000, "must be a valid BCI");
 931 #ifdef _LP64
 932       // Must be restored to the full-width 64-bit stack slot.
 933       array->append(new ConstantLongValue(t->is_ptr()->get_con()));
 934 #else
 935       array->append(new ConstantIntValue(t->is_ptr()->get_con()));
 936 #endif
 937       break;
 938     case Type::FloatCon: {
 939       float f = t->is_float_constant()->getf();
 940       array->append(new ConstantIntValue(jint_cast(f)));
 941       break;
 942     }
 943     case Type::DoubleCon: {
 944       jdouble d = t->is_double_constant()->getd();
 945 #ifdef _LP64
 946       array->append(new ConstantIntValue((jint)0));
 947       array->append(new ConstantDoubleValue(d));
 948 #else
 949       // Repack the double as two jints.
 950     // The convention the interpreter uses is that the second local
 951     // holds the first raw word of the native double representation.
 952     // This is actually reasonable, since locals and stack arrays
 953     // grow downwards in all implementations.
 954     // (If, on some machine, the interpreter's Java locals or stack
 955     // were to grow upwards, the embedded doubles would be word-swapped.)
 956     jlong_accessor acc;
 957     acc.long_value = jlong_cast(d);
 958     array->append(new ConstantIntValue(acc.words[1]));
 959     array->append(new ConstantIntValue(acc.words[0]));
 960 #endif
 961       break;
 962     }
 963     case Type::Long: {
 964       jlong d = t->is_long()->get_con();
 965 #ifdef _LP64
 966       array->append(new ConstantIntValue((jint)0));
 967       array->append(new ConstantLongValue(d));
 968 #else
 969       // Repack the long as two jints.
 970     // The convention the interpreter uses is that the second local
 971     // holds the first raw word of the native double representation.
 972     // This is actually reasonable, since locals and stack arrays
 973     // grow downwards in all implementations.
 974     // (If, on some machine, the interpreter's Java locals or stack
 975     // were to grow upwards, the embedded doubles would be word-swapped.)
 976     jlong_accessor acc;
 977     acc.long_value = d;
 978     array->append(new ConstantIntValue(acc.words[1]));
 979     array->append(new ConstantIntValue(acc.words[0]));
 980 #endif
 981       break;
 982     }
 983     case Type::Top:               // Add an illegal value here
 984       array->append(new LocationValue(Location()));
 985       break;
 986     default:
 987       ShouldNotReachHere();
 988       break;
 989   }
 990 }
 991 
 992 // Determine if this node starts a bundle
 993 bool PhaseOutput::starts_bundle(const Node *n) const {
 994   return (_node_bundling_limit > n->_idx &&
 995           _node_bundling_base[n->_idx].starts_bundle());
 996 }
 997 
 998 //--------------------------Process_OopMap_Node--------------------------------
 999 void PhaseOutput::Process_OopMap_Node(MachNode *mach, int current_offset) {
1000   // Handle special safepoint nodes for synchronization
1001   MachSafePointNode *sfn   = mach->as_MachSafePoint();
1002   MachCallNode      *mcall;
1003 
1004   int safepoint_pc_offset = current_offset;
1005   bool is_method_handle_invoke = false;
1006   bool is_opt_native = false;
1007   bool return_oop = false;
1008   bool has_ea_local_in_scope = sfn->_has_ea_local_in_scope;
1009   bool arg_escape = false;
1010 
1011   // Add the safepoint in the DebugInfoRecorder
1012   if( !mach->is_MachCall() ) {
1013     mcall = NULL;
1014     C->debug_info()->add_safepoint(safepoint_pc_offset, sfn->_oop_map);
1015   } else {
1016     mcall = mach->as_MachCall();
1017 
1018     // Is the call a MethodHandle call?
1019     if (mcall->is_MachCallJava()) {
1020       if (mcall->as_MachCallJava()->_method_handle_invoke) {
1021         assert(C->has_method_handle_invokes(), "must have been set during call generation");
1022         is_method_handle_invoke = true;
1023       }
1024       arg_escape = mcall->as_MachCallJava()->_arg_escape;
1025     } else if (mcall->is_MachCallNative()) {
1026       is_opt_native = true;
1027     }
1028 
1029     // Check if a call returns an object.
1030     if (mcall->returns_pointer()) {
1031       return_oop = true;
1032     }
1033     safepoint_pc_offset += mcall->ret_addr_offset();
1034     C->debug_info()->add_safepoint(safepoint_pc_offset, mcall->_oop_map);
1035   }
1036 
1037   // Loop over the JVMState list to add scope information
1038   // Do not skip safepoints with a NULL method, they need monitor info
1039   JVMState* youngest_jvms = sfn->jvms();
1040   int max_depth = youngest_jvms->depth();
1041 
1042   // Allocate the object pool for scalar-replaced objects -- the map from
1043   // small-integer keys (which can be recorded in the local and ostack
1044   // arrays) to descriptions of the object state.
1045   GrowableArray<ScopeValue*> *objs = new GrowableArray<ScopeValue*>();
1046 
1047   // Visit scopes from oldest to youngest.
1048   for (int depth = 1; depth <= max_depth; depth++) {
1049     JVMState* jvms = youngest_jvms->of_depth(depth);
1050     int idx;
1051     ciMethod* method = jvms->has_method() ? jvms->method() : NULL;
1052     // Safepoints that do not have method() set only provide oop-map and monitor info
1053     // to support GC; these do not support deoptimization.
1054     int num_locs = (method == NULL) ? 0 : jvms->loc_size();
1055     int num_exps = (method == NULL) ? 0 : jvms->stk_size();
1056     int num_mon  = jvms->nof_monitors();
1057     assert(method == NULL || jvms->bci() < 0 || num_locs == method->max_locals(),
1058            "JVMS local count must match that of the method");
1059 
1060     // Add Local and Expression Stack Information
1061 
1062     // Insert locals into the locarray
1063     GrowableArray<ScopeValue*> *locarray = new GrowableArray<ScopeValue*>(num_locs);
1064     for( idx = 0; idx < num_locs; idx++ ) {
1065       FillLocArray( idx, sfn, sfn->local(jvms, idx), locarray, objs );
1066     }
1067 
1068     // Insert expression stack entries into the exparray
1069     GrowableArray<ScopeValue*> *exparray = new GrowableArray<ScopeValue*>(num_exps);
1070     for( idx = 0; idx < num_exps; idx++ ) {
1071       FillLocArray( idx,  sfn, sfn->stack(jvms, idx), exparray, objs );
1072     }
1073 
1074     // Add in mappings of the monitors
1075     assert( !method ||
1076             !method->is_synchronized() ||
1077             method->is_native() ||
1078             num_mon > 0 ||
1079             !GenerateSynchronizationCode,
1080             "monitors must always exist for synchronized methods");
1081 
1082     // Build the growable array of ScopeValues for exp stack
1083     GrowableArray<MonitorValue*> *monarray = new GrowableArray<MonitorValue*>(num_mon);
1084 
1085     // Loop over monitors and insert into array
1086     for (idx = 0; idx < num_mon; idx++) {
1087       // Grab the node that defines this monitor
1088       Node* box_node = sfn->monitor_box(jvms, idx);
1089       Node* obj_node = sfn->monitor_obj(jvms, idx);
1090 
1091       // Create ScopeValue for object
1092       ScopeValue *scval = NULL;
1093 
1094       if (obj_node->is_SafePointScalarObject()) {
1095         SafePointScalarObjectNode* spobj = obj_node->as_SafePointScalarObject();
1096         scval = PhaseOutput::sv_for_node_id(objs, spobj->_idx);
1097         if (scval == NULL) {
1098           const Type *t = spobj->bottom_type();
1099           ciKlass* cik = t->is_oopptr()->klass();
1100           assert(cik->is_instance_klass() ||
1101                  cik->is_array_klass(), "Not supported allocation.");
1102           ScopeValue* klass_sv = new ConstantOopWriteValue(cik->java_mirror()->constant_encoding());
1103           ObjectValue* sv = spobj->is_auto_box() ? new AutoBoxObjectValue(spobj->_idx, klass_sv)
1104                                         : new ObjectValue(spobj->_idx, klass_sv);
1105           PhaseOutput::set_sv_for_object_node(objs, sv);
1106 
1107           uint first_ind = spobj->first_index(youngest_jvms);
1108           for (uint i = 0; i < spobj->n_fields(); i++) {
1109             Node* fld_node = sfn->in(first_ind+i);
1110             (void)FillLocArray(sv->field_values()->length(), sfn, fld_node, sv->field_values(), objs);
1111           }
1112           scval = sv;
1113         }
1114       } else if (!obj_node->is_Con()) {
1115         OptoReg::Name obj_reg = C->regalloc()->get_reg_first(obj_node);
1116         if( obj_node->bottom_type()->base() == Type::NarrowOop ) {
1117           scval = new_loc_value( C->regalloc(), obj_reg, Location::narrowoop );
1118         } else {
1119           scval = new_loc_value( C->regalloc(), obj_reg, Location::oop );
1120         }
1121       } else {
1122         const TypePtr *tp = obj_node->get_ptr_type();
1123         scval = new ConstantOopWriteValue(tp->is_oopptr()->const_oop()->constant_encoding());
1124       }
1125 
1126       OptoReg::Name box_reg = BoxLockNode::reg(box_node);
1127       Location basic_lock = Location::new_stk_loc(Location::normal,C->regalloc()->reg2offset(box_reg));
1128       bool eliminated = (box_node->is_BoxLock() && box_node->as_BoxLock()->is_eliminated());
1129       monarray->append(new MonitorValue(scval, basic_lock, eliminated));
1130     }
1131 
1132     // We dump the object pool first, since deoptimization reads it in first.
1133     C->debug_info()->dump_object_pool(objs);
1134 
1135     // Build first class objects to pass to scope
1136     DebugToken *locvals = C->debug_info()->create_scope_values(locarray);
1137     DebugToken *expvals = C->debug_info()->create_scope_values(exparray);
1138     DebugToken *monvals = C->debug_info()->create_monitor_values(monarray);
1139 
1140     // Make method available for all Safepoints
1141     ciMethod* scope_method = method ? method : C->method();
1142     // Describe the scope here
1143     assert(jvms->bci() >= InvocationEntryBci && jvms->bci() <= 0x10000, "must be a valid or entry BCI");
1144     assert(!jvms->should_reexecute() || depth == max_depth, "reexecute allowed only for the youngest");
1145     // Now we can describe the scope.
1146     methodHandle null_mh;
1147     bool rethrow_exception = false;
1148     C->debug_info()->describe_scope(
1149       safepoint_pc_offset,
1150       null_mh,
1151       scope_method,
1152       jvms->bci(),
1153       jvms->should_reexecute(),
1154       rethrow_exception,
1155       is_method_handle_invoke,
1156       is_opt_native,
1157       return_oop,
1158       has_ea_local_in_scope,
1159       arg_escape,
1160       locvals,
1161       expvals,
1162       monvals
1163     );
1164   } // End jvms loop
1165 
1166   // Mark the end of the scope set.
1167   C->debug_info()->end_safepoint(safepoint_pc_offset);
1168 }
1169 
1170 
1171 
1172 // A simplified version of Process_OopMap_Node, to handle non-safepoints.
1173 class NonSafepointEmitter {
1174     Compile*  C;
1175     JVMState* _pending_jvms;
1176     int       _pending_offset;
1177 
1178     void emit_non_safepoint();
1179 
1180  public:
1181     NonSafepointEmitter(Compile* compile) {
1182       this->C = compile;
1183       _pending_jvms = NULL;
1184       _pending_offset = 0;
1185     }
1186 
1187     void observe_instruction(Node* n, int pc_offset) {
1188       if (!C->debug_info()->recording_non_safepoints())  return;
1189 
1190       Node_Notes* nn = C->node_notes_at(n->_idx);
1191       if (nn == NULL || nn->jvms() == NULL)  return;
1192       if (_pending_jvms != NULL &&
1193           _pending_jvms->same_calls_as(nn->jvms())) {
1194         // Repeated JVMS?  Stretch it up here.
1195         _pending_offset = pc_offset;
1196       } else {
1197         if (_pending_jvms != NULL &&
1198             _pending_offset < pc_offset) {
1199           emit_non_safepoint();
1200         }
1201         _pending_jvms = NULL;
1202         if (pc_offset > C->debug_info()->last_pc_offset()) {
1203           // This is the only way _pending_jvms can become non-NULL:
1204           _pending_jvms = nn->jvms();
1205           _pending_offset = pc_offset;
1206         }
1207       }
1208     }
1209 
1210     // Stay out of the way of real safepoints:
1211     void observe_safepoint(JVMState* jvms, int pc_offset) {
1212       if (_pending_jvms != NULL &&
1213           !_pending_jvms->same_calls_as(jvms) &&
1214           _pending_offset < pc_offset) {
1215         emit_non_safepoint();
1216       }
1217       _pending_jvms = NULL;
1218     }
1219 
1220     void flush_at_end() {
1221       if (_pending_jvms != NULL) {
1222         emit_non_safepoint();
1223       }
1224       _pending_jvms = NULL;
1225     }
1226 };
1227 
1228 void NonSafepointEmitter::emit_non_safepoint() {
1229   JVMState* youngest_jvms = _pending_jvms;
1230   int       pc_offset     = _pending_offset;
1231 
1232   // Clear it now:
1233   _pending_jvms = NULL;
1234 
1235   DebugInformationRecorder* debug_info = C->debug_info();
1236   assert(debug_info->recording_non_safepoints(), "sanity");
1237 
1238   debug_info->add_non_safepoint(pc_offset);
1239   int max_depth = youngest_jvms->depth();
1240 
1241   // Visit scopes from oldest to youngest.
1242   for (int depth = 1; depth <= max_depth; depth++) {
1243     JVMState* jvms = youngest_jvms->of_depth(depth);
1244     ciMethod* method = jvms->has_method() ? jvms->method() : NULL;
1245     assert(!jvms->should_reexecute() || depth==max_depth, "reexecute allowed only for the youngest");
1246     methodHandle null_mh;
1247     debug_info->describe_scope(pc_offset, null_mh, method, jvms->bci(), jvms->should_reexecute());
1248   }
1249 
1250   // Mark the end of the scope set.
1251   debug_info->end_non_safepoint(pc_offset);
1252 }
1253 
1254 //------------------------------init_buffer------------------------------------
1255 void PhaseOutput::estimate_buffer_size(int& const_req) {
1256 
1257   // Set the initially allocated size
1258   const_req = initial_const_capacity;
1259 
1260   // The extra spacing after the code is necessary on some platforms.
1261   // Sometimes we need to patch in a jump after the last instruction,
1262   // if the nmethod has been deoptimized.  (See 4932387, 4894843.)
1263 
1264   // Compute the byte offset where we can store the deopt pc.
1265   if (C->fixed_slots() != 0) {
1266     _orig_pc_slot_offset_in_bytes = C->regalloc()->reg2offset(OptoReg::stack2reg(_orig_pc_slot));
1267   }
1268 
1269   // Compute prolog code size
1270   _method_size = 0;
1271   _frame_slots = OptoReg::reg2stack(C->matcher()->_old_SP) + C->regalloc()->_framesize;
1272   assert(_frame_slots >= 0 && _frame_slots < 1000000, "sanity check");
1273 
1274   if (C->has_mach_constant_base_node()) {
1275     uint add_size = 0;
1276     // Fill the constant table.
1277     // Note:  This must happen before shorten_branches.
1278     for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
1279       Block* b = C->cfg()->get_block(i);
1280 
1281       for (uint j = 0; j < b->number_of_nodes(); j++) {
1282         Node* n = b->get_node(j);
1283 
1284         // If the node is a MachConstantNode evaluate the constant
1285         // value section.
1286         if (n->is_MachConstant()) {
1287           MachConstantNode* machcon = n->as_MachConstant();
1288           machcon->eval_constant(C);
1289         } else if (n->is_Mach()) {
1290           // On Power there are more nodes that issue constants.
1291           add_size += (n->as_Mach()->ins_num_consts() * 8);
1292         }
1293       }
1294     }
1295 
1296     // Calculate the offsets of the constants and the size of the
1297     // constant table (including the padding to the next section).
1298     constant_table().calculate_offsets_and_size();
1299     const_req = constant_table().size() + add_size;
1300   }
1301 
1302   // Initialize the space for the BufferBlob used to find and verify
1303   // instruction size in MachNode::emit_size()
1304   init_scratch_buffer_blob(const_req);
1305 }
1306 
1307 CodeBuffer* PhaseOutput::init_buffer() {
1308   int stub_req  = _buf_sizes._stub;
1309   int code_req  = _buf_sizes._code;
1310   int const_req = _buf_sizes._const;
1311 
1312   int pad_req   = NativeCall::instruction_size;
1313 
1314   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1315   stub_req += bs->estimate_stub_size();
1316   stub_req += safepoint_poll_table()->estimate_stub_size();
1317 
1318   // nmethod and CodeBuffer count stubs & constants as part of method's code.
1319   // class HandlerImpl is platform-specific and defined in the *.ad files.
1320   int exception_handler_req = HandlerImpl::size_exception_handler() + MAX_stubs_size; // add marginal slop for handler
1321   int deopt_handler_req     = HandlerImpl::size_deopt_handler()     + MAX_stubs_size; // add marginal slop for handler
1322   stub_req += MAX_stubs_size;   // ensure per-stub margin
1323   code_req += MAX_inst_size;    // ensure per-instruction margin
1324 
1325   if (StressCodeBuffers)
1326     code_req = const_req = stub_req = exception_handler_req = deopt_handler_req = 0x10;  // force expansion
1327 
1328   int total_req =
1329           const_req +
1330           code_req +
1331           pad_req +
1332           stub_req +
1333           exception_handler_req +
1334           deopt_handler_req;               // deopt handler
1335 
1336   if (C->has_method_handle_invokes())
1337     total_req += deopt_handler_req;  // deopt MH handler
1338 
1339   CodeBuffer* cb = code_buffer();
1340   cb->initialize(total_req, _buf_sizes._reloc);
1341 
1342   // Have we run out of code space?
1343   if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) {
1344     C->record_failure("CodeCache is full");
1345     return NULL;
1346   }
1347   // Configure the code buffer.
1348   cb->initialize_consts_size(const_req);
1349   cb->initialize_stubs_size(stub_req);
1350   cb->initialize_oop_recorder(C->env()->oop_recorder());
1351 
1352   // fill in the nop array for bundling computations
1353   MachNode *_nop_list[Bundle::_nop_count];
1354   Bundle::initialize_nops(_nop_list);
1355 
1356   return cb;
1357 }
1358 
1359 //------------------------------fill_buffer------------------------------------
1360 void PhaseOutput::fill_buffer(CodeBuffer* cb, uint* blk_starts) {
1361   // blk_starts[] contains offsets calculated during short branches processing,
1362   // offsets should not be increased during following steps.
1363 
1364   // Compute the size of first NumberOfLoopInstrToAlign instructions at head
1365   // of a loop. It is used to determine the padding for loop alignment.
1366   Compile::TracePhase tp("fill buffer", &timers[_t_fillBuffer]);
1367 
1368   compute_loop_first_inst_sizes();
1369 
1370   // Create oopmap set.
1371   _oop_map_set = new OopMapSet();
1372 
1373   // !!!!! This preserves old handling of oopmaps for now
1374   C->debug_info()->set_oopmaps(_oop_map_set);
1375 
1376   uint nblocks  = C->cfg()->number_of_blocks();
1377   // Count and start of implicit null check instructions
1378   uint inct_cnt = 0;
1379   uint* inct_starts = NEW_RESOURCE_ARRAY(uint, nblocks+1);
1380 
1381   // Count and start of calls
1382   uint* call_returns = NEW_RESOURCE_ARRAY(uint, nblocks+1);
1383 
1384   uint  return_offset = 0;
1385   int nop_size = (new MachNopNode())->size(C->regalloc());
1386 
1387   int previous_offset = 0;
1388   int current_offset  = 0;
1389   int last_call_offset = -1;
1390   int last_avoid_back_to_back_offset = -1;
1391 #ifdef ASSERT
1392   uint* jmp_target = NEW_RESOURCE_ARRAY(uint,nblocks);
1393   uint* jmp_offset = NEW_RESOURCE_ARRAY(uint,nblocks);
1394   uint* jmp_size   = NEW_RESOURCE_ARRAY(uint,nblocks);
1395   uint* jmp_rule   = NEW_RESOURCE_ARRAY(uint,nblocks);
1396 #endif
1397 
1398   // Create an array of unused labels, one for each basic block, if printing is enabled
1399 #if defined(SUPPORT_OPTO_ASSEMBLY)
1400   int* node_offsets      = NULL;
1401   uint node_offset_limit = C->unique();
1402 
1403   if (C->print_assembly()) {
1404     node_offsets = NEW_RESOURCE_ARRAY(int, node_offset_limit);
1405   }
1406   if (node_offsets != NULL) {
1407     // We need to initialize. Unused array elements may contain garbage and mess up PrintOptoAssembly.
1408     memset(node_offsets, 0, node_offset_limit*sizeof(int));
1409   }
1410 #endif
1411 
1412   NonSafepointEmitter non_safepoints(C);  // emit non-safepoints lazily
1413 
1414   // Emit the constant table.
1415   if (C->has_mach_constant_base_node()) {
1416     if (!constant_table().emit(*cb)) {
1417       C->record_failure("consts section overflow");
1418       return;
1419     }
1420   }
1421 
1422   // Create an array of labels, one for each basic block
1423   Label* blk_labels = NEW_RESOURCE_ARRAY(Label, nblocks+1);
1424   for (uint i = 0; i <= nblocks; i++) {
1425     blk_labels[i].init();
1426   }
1427 
1428   // Now fill in the code buffer
1429   Node* delay_slot = NULL;
1430   for (uint i = 0; i < nblocks; i++) {
1431     Block* block = C->cfg()->get_block(i);
1432     _block = block;
1433     Node* head = block->head();
1434 
1435     // If this block needs to start aligned (i.e, can be reached other
1436     // than by falling-thru from the previous block), then force the
1437     // start of a new bundle.
1438     if (Pipeline::requires_bundling() && starts_bundle(head)) {
1439       cb->flush_bundle(true);
1440     }
1441 
1442 #ifdef ASSERT
1443     if (!block->is_connector()) {
1444       stringStream st;
1445       block->dump_head(C->cfg(), &st);
1446       MacroAssembler(cb).block_comment(st.as_string());
1447     }
1448     jmp_target[i] = 0;
1449     jmp_offset[i] = 0;
1450     jmp_size[i]   = 0;
1451     jmp_rule[i]   = 0;
1452 #endif
1453     int blk_offset = current_offset;
1454 
1455     // Define the label at the beginning of the basic block
1456     MacroAssembler(cb).bind(blk_labels[block->_pre_order]);
1457 
1458     uint last_inst = block->number_of_nodes();
1459 
1460     // Emit block normally, except for last instruction.
1461     // Emit means "dump code bits into code buffer".
1462     for (uint j = 0; j<last_inst; j++) {
1463       _index = j;
1464 
1465       // Get the node
1466       Node* n = block->get_node(j);
1467 
1468       // See if delay slots are supported
1469       if (valid_bundle_info(n) && node_bundling(n)->used_in_unconditional_delay()) {
1470         assert(delay_slot == NULL, "no use of delay slot node");
1471         assert(n->size(C->regalloc()) == Pipeline::instr_unit_size(), "delay slot instruction wrong size");
1472 
1473         delay_slot = n;
1474         continue;
1475       }
1476 
1477       // If this starts a new instruction group, then flush the current one
1478       // (but allow split bundles)
1479       if (Pipeline::requires_bundling() && starts_bundle(n))
1480         cb->flush_bundle(false);
1481 
1482       // Special handling for SafePoint/Call Nodes
1483       bool is_mcall = false;
1484       if (n->is_Mach()) {
1485         MachNode *mach = n->as_Mach();
1486         is_mcall = n->is_MachCall();
1487         bool is_sfn = n->is_MachSafePoint();
1488 
1489         // If this requires all previous instructions be flushed, then do so
1490         if (is_sfn || is_mcall || mach->alignment_required() != 1) {
1491           cb->flush_bundle(true);
1492           current_offset = cb->insts_size();
1493         }
1494 
1495         // A padding may be needed again since a previous instruction
1496         // could be moved to delay slot.
1497 
1498         // align the instruction if necessary
1499         int padding = mach->compute_padding(current_offset);
1500         // Make sure safepoint node for polling is distinct from a call's
1501         // return by adding a nop if needed.
1502         if (is_sfn && !is_mcall && padding == 0 && current_offset == last_call_offset) {
1503           padding = nop_size;
1504         }
1505         if (padding == 0 && mach->avoid_back_to_back(MachNode::AVOID_BEFORE) &&
1506             current_offset == last_avoid_back_to_back_offset) {
1507           // Avoid back to back some instructions.
1508           padding = nop_size;
1509         }
1510 
1511         if (padding > 0) {
1512           assert((padding % nop_size) == 0, "padding is not a multiple of NOP size");
1513           int nops_cnt = padding / nop_size;
1514           MachNode *nop = new MachNopNode(nops_cnt);
1515           block->insert_node(nop, j++);
1516           last_inst++;
1517           C->cfg()->map_node_to_block(nop, block);
1518           // Ensure enough space.
1519           cb->insts()->maybe_expand_to_ensure_remaining(MAX_inst_size);
1520           if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) {
1521             C->record_failure("CodeCache is full");
1522             return;
1523           }
1524           nop->emit(*cb, C->regalloc());
1525           cb->flush_bundle(true);
1526           current_offset = cb->insts_size();
1527         }
1528 
1529         bool observe_safepoint = is_sfn;
1530         // Remember the start of the last call in a basic block
1531         if (is_mcall) {
1532           MachCallNode *mcall = mach->as_MachCall();
1533 
1534           // This destination address is NOT PC-relative
1535           mcall->method_set((intptr_t)mcall->entry_point());
1536 
1537           // Save the return address
1538           call_returns[block->_pre_order] = current_offset + mcall->ret_addr_offset();
1539 
1540           observe_safepoint = mcall->guaranteed_safepoint();
1541         }
1542 
1543         // sfn will be valid whenever mcall is valid now because of inheritance
1544         if (observe_safepoint) {
1545           // Handle special safepoint nodes for synchronization
1546           if (!is_mcall) {
1547             MachSafePointNode *sfn = mach->as_MachSafePoint();
1548             // !!!!! Stubs only need an oopmap right now, so bail out
1549             if (sfn->jvms()->method() == NULL) {
1550               // Write the oopmap directly to the code blob??!!
1551               continue;
1552             }
1553           } // End synchronization
1554 
1555           non_safepoints.observe_safepoint(mach->as_MachSafePoint()->jvms(),
1556                                            current_offset);
1557           Process_OopMap_Node(mach, current_offset);
1558         } // End if safepoint
1559 
1560           // If this is a null check, then add the start of the previous instruction to the list
1561         else if( mach->is_MachNullCheck() ) {
1562           inct_starts[inct_cnt++] = previous_offset;
1563         }
1564 
1565           // If this is a branch, then fill in the label with the target BB's label
1566         else if (mach->is_MachBranch()) {
1567           // This requires the TRUE branch target be in succs[0]
1568           uint block_num = block->non_connector_successor(0)->_pre_order;
1569 
1570           // Try to replace long branch if delay slot is not used,
1571           // it is mostly for back branches since forward branch's
1572           // distance is not updated yet.
1573           bool delay_slot_is_used = valid_bundle_info(n) &&
1574                                     C->output()->node_bundling(n)->use_unconditional_delay();
1575           if (!delay_slot_is_used && mach->may_be_short_branch()) {
1576             assert(delay_slot == NULL, "not expecting delay slot node");
1577             int br_size = n->size(C->regalloc());
1578             int offset = blk_starts[block_num] - current_offset;
1579             if (block_num >= i) {
1580               // Current and following block's offset are not
1581               // finalized yet, adjust distance by the difference
1582               // between calculated and final offsets of current block.
1583               offset -= (blk_starts[i] - blk_offset);
1584             }
1585             // In the following code a nop could be inserted before
1586             // the branch which will increase the backward distance.
1587             bool needs_padding = (current_offset == last_avoid_back_to_back_offset);
1588             if (needs_padding && offset <= 0)
1589               offset -= nop_size;
1590 
1591             if (C->matcher()->is_short_branch_offset(mach->rule(), br_size, offset)) {
1592               // We've got a winner.  Replace this branch.
1593               MachNode* replacement = mach->as_MachBranch()->short_branch_version();
1594 
1595               // Update the jmp_size.
1596               int new_size = replacement->size(C->regalloc());
1597               assert((br_size - new_size) >= (int)nop_size, "short_branch size should be smaller");
1598               // Insert padding between avoid_back_to_back branches.
1599               if (needs_padding && replacement->avoid_back_to_back(MachNode::AVOID_BEFORE)) {
1600                 MachNode *nop = new MachNopNode();
1601                 block->insert_node(nop, j++);
1602                 C->cfg()->map_node_to_block(nop, block);
1603                 last_inst++;
1604                 nop->emit(*cb, C->regalloc());
1605                 cb->flush_bundle(true);
1606                 current_offset = cb->insts_size();
1607               }
1608 #ifdef ASSERT
1609               jmp_target[i] = block_num;
1610               jmp_offset[i] = current_offset - blk_offset;
1611               jmp_size[i]   = new_size;
1612               jmp_rule[i]   = mach->rule();
1613 #endif
1614               block->map_node(replacement, j);
1615               mach->subsume_by(replacement, C);
1616               n    = replacement;
1617               mach = replacement;
1618             }
1619           }
1620           mach->as_MachBranch()->label_set( &blk_labels[block_num], block_num );
1621         } else if (mach->ideal_Opcode() == Op_Jump) {
1622           for (uint h = 0; h < block->_num_succs; h++) {
1623             Block* succs_block = block->_succs[h];
1624             for (uint j = 1; j < succs_block->num_preds(); j++) {
1625               Node* jpn = succs_block->pred(j);
1626               if (jpn->is_JumpProj() && jpn->in(0) == mach) {
1627                 uint block_num = succs_block->non_connector()->_pre_order;
1628                 Label *blkLabel = &blk_labels[block_num];
1629                 mach->add_case_label(jpn->as_JumpProj()->proj_no(), blkLabel);
1630               }
1631             }
1632           }
1633         }
1634 #ifdef ASSERT
1635           // Check that oop-store precedes the card-mark
1636         else if (mach->ideal_Opcode() == Op_StoreCM) {
1637           uint storeCM_idx = j;
1638           int count = 0;
1639           for (uint prec = mach->req(); prec < mach->len(); prec++) {
1640             Node *oop_store = mach->in(prec);  // Precedence edge
1641             if (oop_store == NULL) continue;
1642             count++;
1643             uint i4;
1644             for (i4 = 0; i4 < last_inst; ++i4) {
1645               if (block->get_node(i4) == oop_store) {
1646                 break;
1647               }
1648             }
1649             // Note: This test can provide a false failure if other precedence
1650             // edges have been added to the storeCMNode.
1651             assert(i4 == last_inst || i4 < storeCM_idx, "CM card-mark executes before oop-store");
1652           }
1653           assert(count > 0, "storeCM expects at least one precedence edge");
1654         }
1655 #endif
1656         else if (!n->is_Proj()) {
1657           // Remember the beginning of the previous instruction, in case
1658           // it's followed by a flag-kill and a null-check.  Happens on
1659           // Intel all the time, with add-to-memory kind of opcodes.
1660           previous_offset = current_offset;
1661         }
1662 
1663         // Not an else-if!
1664         // If this is a trap based cmp then add its offset to the list.
1665         if (mach->is_TrapBasedCheckNode()) {
1666           inct_starts[inct_cnt++] = current_offset;
1667         }
1668       }
1669 
1670       // Verify that there is sufficient space remaining
1671       cb->insts()->maybe_expand_to_ensure_remaining(MAX_inst_size);
1672       if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) {
1673         C->record_failure("CodeCache is full");
1674         return;
1675       }
1676 
1677       // Save the offset for the listing
1678 #if defined(SUPPORT_OPTO_ASSEMBLY)
1679       if ((node_offsets != NULL) && (n->_idx < node_offset_limit)) {
1680         node_offsets[n->_idx] = cb->insts_size();
1681       }
1682 #endif
1683       assert(!C->failing(), "Should not reach here if failing.");
1684 
1685       // "Normal" instruction case
1686       DEBUG_ONLY(uint instr_offset = cb->insts_size());
1687       n->emit(*cb, C->regalloc());
1688       current_offset = cb->insts_size();
1689 
1690       // Above we only verified that there is enough space in the instruction section.
1691       // However, the instruction may emit stubs that cause code buffer expansion.
1692       // Bail out here if expansion failed due to a lack of code cache space.
1693       if (C->failing()) {
1694         return;
1695       }
1696 
1697       assert(!is_mcall || (call_returns[block->_pre_order] <= (uint)current_offset),
1698              "ret_addr_offset() not within emitted code");
1699 
1700 #ifdef ASSERT
1701       uint n_size = n->size(C->regalloc());
1702       if (n_size < (current_offset-instr_offset)) {
1703         MachNode* mach = n->as_Mach();
1704         n->dump();
1705         mach->dump_format(C->regalloc(), tty);
1706         tty->print_cr(" n_size (%d), current_offset (%d), instr_offset (%d)", n_size, current_offset, instr_offset);
1707         Disassembler::decode(cb->insts_begin() + instr_offset, cb->insts_begin() + current_offset + 1, tty);
1708         tty->print_cr(" ------------------- ");
1709         BufferBlob* blob = this->scratch_buffer_blob();
1710         address blob_begin = blob->content_begin();
1711         Disassembler::decode(blob_begin, blob_begin + n_size + 1, tty);
1712         assert(false, "wrong size of mach node");
1713       }
1714 #endif
1715       non_safepoints.observe_instruction(n, current_offset);
1716 
1717       // mcall is last "call" that can be a safepoint
1718       // record it so we can see if a poll will directly follow it
1719       // in which case we'll need a pad to make the PcDesc sites unique
1720       // see  5010568. This can be slightly inaccurate but conservative
1721       // in the case that return address is not actually at current_offset.
1722       // This is a small price to pay.
1723 
1724       if (is_mcall) {
1725         last_call_offset = current_offset;
1726       }
1727 
1728       if (n->is_Mach() && n->as_Mach()->avoid_back_to_back(MachNode::AVOID_AFTER)) {
1729         // Avoid back to back some instructions.
1730         last_avoid_back_to_back_offset = current_offset;
1731       }
1732 
1733       // See if this instruction has a delay slot
1734       if (valid_bundle_info(n) && node_bundling(n)->use_unconditional_delay()) {
1735         guarantee(delay_slot != NULL, "expecting delay slot node");
1736 
1737         // Back up 1 instruction
1738         cb->set_insts_end(cb->insts_end() - Pipeline::instr_unit_size());
1739 
1740         // Save the offset for the listing
1741 #if defined(SUPPORT_OPTO_ASSEMBLY)
1742         if ((node_offsets != NULL) && (delay_slot->_idx < node_offset_limit)) {
1743           node_offsets[delay_slot->_idx] = cb->insts_size();
1744         }
1745 #endif
1746 
1747         // Support a SafePoint in the delay slot
1748         if (delay_slot->is_MachSafePoint()) {
1749           MachNode *mach = delay_slot->as_Mach();
1750           // !!!!! Stubs only need an oopmap right now, so bail out
1751           if (!mach->is_MachCall() && mach->as_MachSafePoint()->jvms()->method() == NULL) {
1752             // Write the oopmap directly to the code blob??!!
1753             delay_slot = NULL;
1754             continue;
1755           }
1756 
1757           int adjusted_offset = current_offset - Pipeline::instr_unit_size();
1758           non_safepoints.observe_safepoint(mach->as_MachSafePoint()->jvms(),
1759                                            adjusted_offset);
1760           // Generate an OopMap entry
1761           Process_OopMap_Node(mach, adjusted_offset);
1762         }
1763 
1764         // Insert the delay slot instruction
1765         delay_slot->emit(*cb, C->regalloc());
1766 
1767         // Don't reuse it
1768         delay_slot = NULL;
1769       }
1770 
1771     } // End for all instructions in block
1772 
1773     // If the next block is the top of a loop, pad this block out to align
1774     // the loop top a little. Helps prevent pipe stalls at loop back branches.
1775     if (i < nblocks-1) {
1776       Block *nb = C->cfg()->get_block(i + 1);
1777       int padding = nb->alignment_padding(current_offset);
1778       if( padding > 0 ) {
1779         MachNode *nop = new MachNopNode(padding / nop_size);
1780         block->insert_node(nop, block->number_of_nodes());
1781         C->cfg()->map_node_to_block(nop, block);
1782         nop->emit(*cb, C->regalloc());
1783         current_offset = cb->insts_size();
1784       }
1785     }
1786     // Verify that the distance for generated before forward
1787     // short branches is still valid.
1788     guarantee((int)(blk_starts[i+1] - blk_starts[i]) >= (current_offset - blk_offset), "shouldn't increase block size");
1789 
1790     // Save new block start offset
1791     blk_starts[i] = blk_offset;
1792   } // End of for all blocks
1793   blk_starts[nblocks] = current_offset;
1794 
1795   non_safepoints.flush_at_end();
1796 
1797   // Offset too large?
1798   if (C->failing())  return;
1799 
1800   // Define a pseudo-label at the end of the code
1801   MacroAssembler(cb).bind( blk_labels[nblocks] );
1802 
1803   // Compute the size of the first block
1804   _first_block_size = blk_labels[1].loc_pos() - blk_labels[0].loc_pos();
1805 
1806 #ifdef ASSERT
1807   for (uint i = 0; i < nblocks; i++) { // For all blocks
1808     if (jmp_target[i] != 0) {
1809       int br_size = jmp_size[i];
1810       int offset = blk_starts[jmp_target[i]]-(blk_starts[i] + jmp_offset[i]);
1811       if (!C->matcher()->is_short_branch_offset(jmp_rule[i], br_size, offset)) {
1812         tty->print_cr("target (%d) - jmp_offset(%d) = offset (%d), jump_size(%d), jmp_block B%d, target_block B%d", blk_starts[jmp_target[i]], blk_starts[i] + jmp_offset[i], offset, br_size, i, jmp_target[i]);
1813         assert(false, "Displacement too large for short jmp");
1814       }
1815     }
1816   }
1817 #endif
1818 
1819   BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2();
1820   bs->emit_stubs(*cb);
1821   if (C->failing())  return;
1822 
1823   // Fill in stubs for calling the runtime from safepoint polls.
1824   safepoint_poll_table()->emit(*cb);
1825   if (C->failing())  return;
1826 
1827 #ifndef PRODUCT
1828   // Information on the size of the method, without the extraneous code
1829   Scheduling::increment_method_size(cb->insts_size());
1830 #endif
1831 
1832   // ------------------
1833   // Fill in exception table entries.
1834   FillExceptionTables(inct_cnt, call_returns, inct_starts, blk_labels);
1835 
1836   // Only java methods have exception handlers and deopt handlers
1837   // class HandlerImpl is platform-specific and defined in the *.ad files.
1838   if (C->method()) {
1839     // Emit the exception handler code.
1840     _code_offsets.set_value(CodeOffsets::Exceptions, HandlerImpl::emit_exception_handler(*cb));
1841     if (C->failing()) {
1842       return; // CodeBuffer::expand failed
1843     }
1844     // Emit the deopt handler code.
1845     _code_offsets.set_value(CodeOffsets::Deopt, HandlerImpl::emit_deopt_handler(*cb));
1846 
1847     // Emit the MethodHandle deopt handler code (if required).
1848     if (C->has_method_handle_invokes() && !C->failing()) {
1849       // We can use the same code as for the normal deopt handler, we
1850       // just need a different entry point address.
1851       _code_offsets.set_value(CodeOffsets::DeoptMH, HandlerImpl::emit_deopt_handler(*cb));
1852     }
1853   }
1854 
1855   // One last check for failed CodeBuffer::expand:
1856   if ((cb->blob() == NULL) || (!CompileBroker::should_compile_new_jobs())) {
1857     C->record_failure("CodeCache is full");
1858     return;
1859   }
1860 
1861 #if defined(SUPPORT_ABSTRACT_ASSEMBLY) || defined(SUPPORT_ASSEMBLY) || defined(SUPPORT_OPTO_ASSEMBLY)
1862   if (C->print_assembly()) {
1863     tty->cr();
1864     tty->print_cr("============================= C2-compiled nmethod ==============================");
1865   }
1866 #endif
1867 
1868 #if defined(SUPPORT_OPTO_ASSEMBLY)
1869   // Dump the assembly code, including basic-block numbers
1870   if (C->print_assembly()) {
1871     ttyLocker ttyl;  // keep the following output all in one block
1872     if (!VMThread::should_terminate()) {  // test this under the tty lock
1873       // This output goes directly to the tty, not the compiler log.
1874       // To enable tools to match it up with the compilation activity,
1875       // be sure to tag this tty output with the compile ID.
1876       if (xtty != NULL) {
1877         xtty->head("opto_assembly compile_id='%d'%s", C->compile_id(),
1878                    C->is_osr_compilation() ? " compile_kind='osr'" : "");
1879       }
1880       if (C->method() != NULL) {
1881         tty->print_cr("----------------------- MetaData before Compile_id = %d ------------------------", C->compile_id());
1882         C->method()->print_metadata();
1883       } else if (C->stub_name() != NULL) {
1884         tty->print_cr("----------------------------- RuntimeStub %s -------------------------------", C->stub_name());
1885       }
1886       tty->cr();
1887       tty->print_cr("------------------------ OptoAssembly for Compile_id = %d -----------------------", C->compile_id());
1888       dump_asm(node_offsets, node_offset_limit);
1889       tty->print_cr("--------------------------------------------------------------------------------");
1890       if (xtty != NULL) {
1891         // print_metadata and dump_asm above may safepoint which makes us loose the ttylock.
1892         // Retake lock too make sure the end tag is coherent, and that xmlStream->pop_tag is done
1893         // thread safe
1894         ttyLocker ttyl2;
1895         xtty->tail("opto_assembly");
1896       }
1897     }
1898   }
1899 #endif
1900 }
1901 
1902 void PhaseOutput::FillExceptionTables(uint cnt, uint *call_returns, uint *inct_starts, Label *blk_labels) {
1903   _inc_table.set_size(cnt);
1904 
1905   uint inct_cnt = 0;
1906   for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
1907     Block* block = C->cfg()->get_block(i);
1908     Node *n = NULL;
1909     int j;
1910 
1911     // Find the branch; ignore trailing NOPs.
1912     for (j = block->number_of_nodes() - 1; j >= 0; j--) {
1913       n = block->get_node(j);
1914       if (!n->is_Mach() || n->as_Mach()->ideal_Opcode() != Op_Con) {
1915         break;
1916       }
1917     }
1918 
1919     // If we didn't find anything, continue
1920     if (j < 0) {
1921       continue;
1922     }
1923 
1924     // Compute ExceptionHandlerTable subtable entry and add it
1925     // (skip empty blocks)
1926     if (n->is_Catch()) {
1927 
1928       // Get the offset of the return from the call
1929       uint call_return = call_returns[block->_pre_order];
1930 #ifdef ASSERT
1931       assert( call_return > 0, "no call seen for this basic block" );
1932       while (block->get_node(--j)->is_MachProj()) ;
1933       assert(block->get_node(j)->is_MachCall(), "CatchProj must follow call");
1934 #endif
1935       // last instruction is a CatchNode, find it's CatchProjNodes
1936       int nof_succs = block->_num_succs;
1937       // allocate space
1938       GrowableArray<intptr_t> handler_bcis(nof_succs);
1939       GrowableArray<intptr_t> handler_pcos(nof_succs);
1940       // iterate through all successors
1941       for (int j = 0; j < nof_succs; j++) {
1942         Block* s = block->_succs[j];
1943         bool found_p = false;
1944         for (uint k = 1; k < s->num_preds(); k++) {
1945           Node* pk = s->pred(k);
1946           if (pk->is_CatchProj() && pk->in(0) == n) {
1947             const CatchProjNode* p = pk->as_CatchProj();
1948             found_p = true;
1949             // add the corresponding handler bci & pco information
1950             if (p->_con != CatchProjNode::fall_through_index) {
1951               // p leads to an exception handler (and is not fall through)
1952               assert(s == C->cfg()->get_block(s->_pre_order), "bad numbering");
1953               // no duplicates, please
1954               if (!handler_bcis.contains(p->handler_bci())) {
1955                 uint block_num = s->non_connector()->_pre_order;
1956                 handler_bcis.append(p->handler_bci());
1957                 handler_pcos.append(blk_labels[block_num].loc_pos());
1958               }
1959             }
1960           }
1961         }
1962         assert(found_p, "no matching predecessor found");
1963         // Note:  Due to empty block removal, one block may have
1964         // several CatchProj inputs, from the same Catch.
1965       }
1966 
1967       // Set the offset of the return from the call
1968       assert(handler_bcis.find(-1) != -1, "must have default handler");
1969       _handler_table.add_subtable(call_return, &handler_bcis, NULL, &handler_pcos);
1970       continue;
1971     }
1972 
1973     // Handle implicit null exception table updates
1974     if (n->is_MachNullCheck()) {
1975       uint block_num = block->non_connector_successor(0)->_pre_order;
1976       _inc_table.append(inct_starts[inct_cnt++], blk_labels[block_num].loc_pos());
1977       continue;
1978     }
1979     // Handle implicit exception table updates: trap instructions.
1980     if (n->is_Mach() && n->as_Mach()->is_TrapBasedCheckNode()) {
1981       uint block_num = block->non_connector_successor(0)->_pre_order;
1982       _inc_table.append(inct_starts[inct_cnt++], blk_labels[block_num].loc_pos());
1983       continue;
1984     }
1985   } // End of for all blocks fill in exception table entries
1986 }
1987 
1988 // Static Variables
1989 #ifndef PRODUCT
1990 uint Scheduling::_total_nop_size = 0;
1991 uint Scheduling::_total_method_size = 0;
1992 uint Scheduling::_total_branches = 0;
1993 uint Scheduling::_total_unconditional_delays = 0;
1994 uint Scheduling::_total_instructions_per_bundle[Pipeline::_max_instrs_per_cycle+1];
1995 #endif
1996 
1997 // Initializer for class Scheduling
1998 
1999 Scheduling::Scheduling(Arena *arena, Compile &compile)
2000         : _arena(arena),
2001           _cfg(compile.cfg()),
2002           _regalloc(compile.regalloc()),
2003           _scheduled(arena),
2004           _available(arena),
2005           _reg_node(arena),
2006           _pinch_free_list(arena),
2007           _next_node(NULL),
2008           _bundle_instr_count(0),
2009           _bundle_cycle_number(0),
2010           _bundle_use(0, 0, resource_count, &_bundle_use_elements[0])
2011 #ifndef PRODUCT
2012         , _branches(0)
2013         , _unconditional_delays(0)
2014 #endif
2015 {
2016   // Create a MachNopNode
2017   _nop = new MachNopNode();
2018 
2019   // Now that the nops are in the array, save the count
2020   // (but allow entries for the nops)
2021   _node_bundling_limit = compile.unique();
2022   uint node_max = _regalloc->node_regs_max_index();
2023 
2024   compile.output()->set_node_bundling_limit(_node_bundling_limit);
2025 
2026   // This one is persistent within the Compile class
2027   _node_bundling_base = NEW_ARENA_ARRAY(compile.comp_arena(), Bundle, node_max);
2028 
2029   // Allocate space for fixed-size arrays
2030   _node_latency    = NEW_ARENA_ARRAY(arena, unsigned short, node_max);
2031   _uses            = NEW_ARENA_ARRAY(arena, short,          node_max);
2032   _current_latency = NEW_ARENA_ARRAY(arena, unsigned short, node_max);
2033 
2034   // Clear the arrays
2035   for (uint i = 0; i < node_max; i++) {
2036     ::new (&_node_bundling_base[i]) Bundle();
2037   }
2038   memset(_node_latency,       0, node_max * sizeof(unsigned short));
2039   memset(_uses,               0, node_max * sizeof(short));
2040   memset(_current_latency,    0, node_max * sizeof(unsigned short));
2041 
2042   // Clear the bundling information
2043   memcpy(_bundle_use_elements, Pipeline_Use::elaborated_elements, sizeof(Pipeline_Use::elaborated_elements));
2044 
2045   // Get the last node
2046   Block* block = _cfg->get_block(_cfg->number_of_blocks() - 1);
2047 
2048   _next_node = block->get_node(block->number_of_nodes() - 1);
2049 }
2050 
2051 #ifndef PRODUCT
2052 // Scheduling destructor
2053 Scheduling::~Scheduling() {
2054   _total_branches             += _branches;
2055   _total_unconditional_delays += _unconditional_delays;
2056 }
2057 #endif
2058 
2059 // Step ahead "i" cycles
2060 void Scheduling::step(uint i) {
2061 
2062   Bundle *bundle = node_bundling(_next_node);
2063   bundle->set_starts_bundle();
2064 
2065   // Update the bundle record, but leave the flags information alone
2066   if (_bundle_instr_count > 0) {
2067     bundle->set_instr_count(_bundle_instr_count);
2068     bundle->set_resources_used(_bundle_use.resourcesUsed());
2069   }
2070 
2071   // Update the state information
2072   _bundle_instr_count = 0;
2073   _bundle_cycle_number += i;
2074   _bundle_use.step(i);
2075 }
2076 
2077 void Scheduling::step_and_clear() {
2078   Bundle *bundle = node_bundling(_next_node);
2079   bundle->set_starts_bundle();
2080 
2081   // Update the bundle record
2082   if (_bundle_instr_count > 0) {
2083     bundle->set_instr_count(_bundle_instr_count);
2084     bundle->set_resources_used(_bundle_use.resourcesUsed());
2085 
2086     _bundle_cycle_number += 1;
2087   }
2088 
2089   // Clear the bundling information
2090   _bundle_instr_count = 0;
2091   _bundle_use.reset();
2092 
2093   memcpy(_bundle_use_elements,
2094          Pipeline_Use::elaborated_elements,
2095          sizeof(Pipeline_Use::elaborated_elements));
2096 }
2097 
2098 // Perform instruction scheduling and bundling over the sequence of
2099 // instructions in backwards order.
2100 void PhaseOutput::ScheduleAndBundle() {
2101 
2102   // Don't optimize this if it isn't a method
2103   if (!C->method())
2104     return;
2105 
2106   // Don't optimize this if scheduling is disabled
2107   if (!C->do_scheduling())
2108     return;
2109 
2110   // Scheduling code works only with pairs (8 bytes) maximum.
2111   // And when the scalable vector register is used, we may spill/unspill
2112   // the whole reg regardless of the max vector size.
2113   if (C->max_vector_size() > 8 ||
2114       (C->max_vector_size() > 0 && Matcher::supports_scalable_vector())) {
2115     return;
2116   }
2117 
2118   Compile::TracePhase tp("isched", &timers[_t_instrSched]);
2119 
2120   // Create a data structure for all the scheduling information
2121   Scheduling scheduling(Thread::current()->resource_area(), *C);
2122 
2123   // Walk backwards over each basic block, computing the needed alignment
2124   // Walk over all the basic blocks
2125   scheduling.DoScheduling();
2126 
2127 #ifndef PRODUCT
2128   if (C->trace_opto_output()) {
2129     tty->print("\n---- After ScheduleAndBundle ----\n");
2130     for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
2131       tty->print("\nBB#%03d:\n", i);
2132       Block* block = C->cfg()->get_block(i);
2133       for (uint j = 0; j < block->number_of_nodes(); j++) {
2134         Node* n = block->get_node(j);
2135         OptoReg::Name reg = C->regalloc()->get_reg_first(n);
2136         tty->print(" %-6s ", reg >= 0 && reg < REG_COUNT ? Matcher::regName[reg] : "");
2137         n->dump();
2138       }
2139     }
2140   }
2141 #endif
2142 }
2143 
2144 // Compute the latency of all the instructions.  This is fairly simple,
2145 // because we already have a legal ordering.  Walk over the instructions
2146 // from first to last, and compute the latency of the instruction based
2147 // on the latency of the preceding instruction(s).
2148 void Scheduling::ComputeLocalLatenciesForward(const Block *bb) {
2149 #ifndef PRODUCT
2150   if (_cfg->C->trace_opto_output())
2151     tty->print("# -> ComputeLocalLatenciesForward\n");
2152 #endif
2153 
2154   // Walk over all the schedulable instructions
2155   for( uint j=_bb_start; j < _bb_end; j++ ) {
2156 
2157     // This is a kludge, forcing all latency calculations to start at 1.
2158     // Used to allow latency 0 to force an instruction to the beginning
2159     // of the bb
2160     uint latency = 1;
2161     Node *use = bb->get_node(j);
2162     uint nlen = use->len();
2163 
2164     // Walk over all the inputs
2165     for ( uint k=0; k < nlen; k++ ) {
2166       Node *def = use->in(k);
2167       if (!def)
2168         continue;
2169 
2170       uint l = _node_latency[def->_idx] + use->latency(k);
2171       if (latency < l)
2172         latency = l;
2173     }
2174 
2175     _node_latency[use->_idx] = latency;
2176 
2177 #ifndef PRODUCT
2178     if (_cfg->C->trace_opto_output()) {
2179       tty->print("# latency %4d: ", latency);
2180       use->dump();
2181     }
2182 #endif
2183   }
2184 
2185 #ifndef PRODUCT
2186   if (_cfg->C->trace_opto_output())
2187     tty->print("# <- ComputeLocalLatenciesForward\n");
2188 #endif
2189 
2190 } // end ComputeLocalLatenciesForward
2191 
2192 // See if this node fits into the present instruction bundle
2193 bool Scheduling::NodeFitsInBundle(Node *n) {
2194   uint n_idx = n->_idx;
2195 
2196   // If this is the unconditional delay instruction, then it fits
2197   if (n == _unconditional_delay_slot) {
2198 #ifndef PRODUCT
2199     if (_cfg->C->trace_opto_output())
2200       tty->print("#     NodeFitsInBundle [%4d]: TRUE; is in unconditional delay slot\n", n->_idx);
2201 #endif
2202     return (true);
2203   }
2204 
2205   // If the node cannot be scheduled this cycle, skip it
2206   if (_current_latency[n_idx] > _bundle_cycle_number) {
2207 #ifndef PRODUCT
2208     if (_cfg->C->trace_opto_output())
2209       tty->print("#     NodeFitsInBundle [%4d]: FALSE; latency %4d > %d\n",
2210                  n->_idx, _current_latency[n_idx], _bundle_cycle_number);
2211 #endif
2212     return (false);
2213   }
2214 
2215   const Pipeline *node_pipeline = n->pipeline();
2216 
2217   uint instruction_count = node_pipeline->instructionCount();
2218   if (node_pipeline->mayHaveNoCode() && n->size(_regalloc) == 0)
2219     instruction_count = 0;
2220   else if (node_pipeline->hasBranchDelay() && !_unconditional_delay_slot)
2221     instruction_count++;
2222 
2223   if (_bundle_instr_count + instruction_count > Pipeline::_max_instrs_per_cycle) {
2224 #ifndef PRODUCT
2225     if (_cfg->C->trace_opto_output())
2226       tty->print("#     NodeFitsInBundle [%4d]: FALSE; too many instructions: %d > %d\n",
2227                  n->_idx, _bundle_instr_count + instruction_count, Pipeline::_max_instrs_per_cycle);
2228 #endif
2229     return (false);
2230   }
2231 
2232   // Don't allow non-machine nodes to be handled this way
2233   if (!n->is_Mach() && instruction_count == 0)
2234     return (false);
2235 
2236   // See if there is any overlap
2237   uint delay = _bundle_use.full_latency(0, node_pipeline->resourceUse());
2238 
2239   if (delay > 0) {
2240 #ifndef PRODUCT
2241     if (_cfg->C->trace_opto_output())
2242       tty->print("#     NodeFitsInBundle [%4d]: FALSE; functional units overlap\n", n_idx);
2243 #endif
2244     return false;
2245   }
2246 
2247 #ifndef PRODUCT
2248   if (_cfg->C->trace_opto_output())
2249     tty->print("#     NodeFitsInBundle [%4d]:  TRUE\n", n_idx);
2250 #endif
2251 
2252   return true;
2253 }
2254 
2255 Node * Scheduling::ChooseNodeToBundle() {
2256   uint siz = _available.size();
2257 
2258   if (siz == 0) {
2259 
2260 #ifndef PRODUCT
2261     if (_cfg->C->trace_opto_output())
2262       tty->print("#   ChooseNodeToBundle: NULL\n");
2263 #endif
2264     return (NULL);
2265   }
2266 
2267   // Fast path, if only 1 instruction in the bundle
2268   if (siz == 1) {
2269 #ifndef PRODUCT
2270     if (_cfg->C->trace_opto_output()) {
2271       tty->print("#   ChooseNodeToBundle (only 1): ");
2272       _available[0]->dump();
2273     }
2274 #endif
2275     return (_available[0]);
2276   }
2277 
2278   // Don't bother, if the bundle is already full
2279   if (_bundle_instr_count < Pipeline::_max_instrs_per_cycle) {
2280     for ( uint i = 0; i < siz; i++ ) {
2281       Node *n = _available[i];
2282 
2283       // Skip projections, we'll handle them another way
2284       if (n->is_Proj())
2285         continue;
2286 
2287       // This presupposed that instructions are inserted into the
2288       // available list in a legality order; i.e. instructions that
2289       // must be inserted first are at the head of the list
2290       if (NodeFitsInBundle(n)) {
2291 #ifndef PRODUCT
2292         if (_cfg->C->trace_opto_output()) {
2293           tty->print("#   ChooseNodeToBundle: ");
2294           n->dump();
2295         }
2296 #endif
2297         return (n);
2298       }
2299     }
2300   }
2301 
2302   // Nothing fits in this bundle, choose the highest priority
2303 #ifndef PRODUCT
2304   if (_cfg->C->trace_opto_output()) {
2305     tty->print("#   ChooseNodeToBundle: ");
2306     _available[0]->dump();
2307   }
2308 #endif
2309 
2310   return _available[0];
2311 }
2312 
2313 void Scheduling::AddNodeToAvailableList(Node *n) {
2314   assert( !n->is_Proj(), "projections never directly made available" );
2315 #ifndef PRODUCT
2316   if (_cfg->C->trace_opto_output()) {
2317     tty->print("#   AddNodeToAvailableList: ");
2318     n->dump();
2319   }
2320 #endif
2321 
2322   int latency = _current_latency[n->_idx];
2323 
2324   // Insert in latency order (insertion sort)
2325   uint i;
2326   for ( i=0; i < _available.size(); i++ )
2327     if (_current_latency[_available[i]->_idx] > latency)
2328       break;
2329 
2330   // Special Check for compares following branches
2331   if( n->is_Mach() && _scheduled.size() > 0 ) {
2332     int op = n->as_Mach()->ideal_Opcode();
2333     Node *last = _scheduled[0];
2334     if( last->is_MachIf() && last->in(1) == n &&
2335         ( op == Op_CmpI ||
2336           op == Op_CmpU ||
2337           op == Op_CmpUL ||
2338           op == Op_CmpP ||
2339           op == Op_CmpF ||
2340           op == Op_CmpD ||
2341           op == Op_CmpL ) ) {
2342 
2343       // Recalculate position, moving to front of same latency
2344       for ( i=0 ; i < _available.size(); i++ )
2345         if (_current_latency[_available[i]->_idx] >= latency)
2346           break;
2347     }
2348   }
2349 
2350   // Insert the node in the available list
2351   _available.insert(i, n);
2352 
2353 #ifndef PRODUCT
2354   if (_cfg->C->trace_opto_output())
2355     dump_available();
2356 #endif
2357 }
2358 
2359 void Scheduling::DecrementUseCounts(Node *n, const Block *bb) {
2360   for ( uint i=0; i < n->len(); i++ ) {
2361     Node *def = n->in(i);
2362     if (!def) continue;
2363     if( def->is_Proj() )        // If this is a machine projection, then
2364       def = def->in(0);         // propagate usage thru to the base instruction
2365 
2366     if(_cfg->get_block_for_node(def) != bb) { // Ignore if not block-local
2367       continue;
2368     }
2369 
2370     // Compute the latency
2371     uint l = _bundle_cycle_number + n->latency(i);
2372     if (_current_latency[def->_idx] < l)
2373       _current_latency[def->_idx] = l;
2374 
2375     // If this does not have uses then schedule it
2376     if ((--_uses[def->_idx]) == 0)
2377       AddNodeToAvailableList(def);
2378   }
2379 }
2380 
2381 void Scheduling::AddNodeToBundle(Node *n, const Block *bb) {
2382 #ifndef PRODUCT
2383   if (_cfg->C->trace_opto_output()) {
2384     tty->print("#   AddNodeToBundle: ");
2385     n->dump();
2386   }
2387 #endif
2388 
2389   // Remove this from the available list
2390   uint i;
2391   for (i = 0; i < _available.size(); i++)
2392     if (_available[i] == n)
2393       break;
2394   assert(i < _available.size(), "entry in _available list not found");
2395   _available.remove(i);
2396 
2397   // See if this fits in the current bundle
2398   const Pipeline *node_pipeline = n->pipeline();
2399   const Pipeline_Use& node_usage = node_pipeline->resourceUse();
2400 
2401   // Check for instructions to be placed in the delay slot. We
2402   // do this before we actually schedule the current instruction,
2403   // because the delay slot follows the current instruction.
2404   if (Pipeline::_branch_has_delay_slot &&
2405       node_pipeline->hasBranchDelay() &&
2406       !_unconditional_delay_slot) {
2407 
2408     uint siz = _available.size();
2409 
2410     // Conditional branches can support an instruction that
2411     // is unconditionally executed and not dependent by the
2412     // branch, OR a conditionally executed instruction if
2413     // the branch is taken.  In practice, this means that
2414     // the first instruction at the branch target is
2415     // copied to the delay slot, and the branch goes to
2416     // the instruction after that at the branch target
2417     if ( n->is_MachBranch() ) {
2418 
2419       assert( !n->is_MachNullCheck(), "should not look for delay slot for Null Check" );
2420       assert( !n->is_Catch(),         "should not look for delay slot for Catch" );
2421 
2422 #ifndef PRODUCT
2423       _branches++;
2424 #endif
2425 
2426       // At least 1 instruction is on the available list
2427       // that is not dependent on the branch
2428       for (uint i = 0; i < siz; i++) {
2429         Node *d = _available[i];
2430         const Pipeline *avail_pipeline = d->pipeline();
2431 
2432         // Don't allow safepoints in the branch shadow, that will
2433         // cause a number of difficulties
2434         if ( avail_pipeline->instructionCount() == 1 &&
2435              !avail_pipeline->hasMultipleBundles() &&
2436              !avail_pipeline->hasBranchDelay() &&
2437              Pipeline::instr_has_unit_size() &&
2438              d->size(_regalloc) == Pipeline::instr_unit_size() &&
2439              NodeFitsInBundle(d) &&
2440              !node_bundling(d)->used_in_delay()) {
2441 
2442           if (d->is_Mach() && !d->is_MachSafePoint()) {
2443             // A node that fits in the delay slot was found, so we need to
2444             // set the appropriate bits in the bundle pipeline information so
2445             // that it correctly indicates resource usage.  Later, when we
2446             // attempt to add this instruction to the bundle, we will skip
2447             // setting the resource usage.
2448             _unconditional_delay_slot = d;
2449             node_bundling(n)->set_use_unconditional_delay();
2450             node_bundling(d)->set_used_in_unconditional_delay();
2451             _bundle_use.add_usage(avail_pipeline->resourceUse());
2452             _current_latency[d->_idx] = _bundle_cycle_number;
2453             _next_node = d;
2454             ++_bundle_instr_count;
2455 #ifndef PRODUCT
2456             _unconditional_delays++;
2457 #endif
2458             break;
2459           }
2460         }
2461       }
2462     }
2463 
2464     // No delay slot, add a nop to the usage
2465     if (!_unconditional_delay_slot) {
2466       // See if adding an instruction in the delay slot will overflow
2467       // the bundle.
2468       if (!NodeFitsInBundle(_nop)) {
2469 #ifndef PRODUCT
2470         if (_cfg->C->trace_opto_output())
2471           tty->print("#  *** STEP(1 instruction for delay slot) ***\n");
2472 #endif
2473         step(1);
2474       }
2475 
2476       _bundle_use.add_usage(_nop->pipeline()->resourceUse());
2477       _next_node = _nop;
2478       ++_bundle_instr_count;
2479     }
2480 
2481     // See if the instruction in the delay slot requires a
2482     // step of the bundles
2483     if (!NodeFitsInBundle(n)) {
2484 #ifndef PRODUCT
2485       if (_cfg->C->trace_opto_output())
2486         tty->print("#  *** STEP(branch won't fit) ***\n");
2487 #endif
2488       // Update the state information
2489       _bundle_instr_count = 0;
2490       _bundle_cycle_number += 1;
2491       _bundle_use.step(1);
2492     }
2493   }
2494 
2495   // Get the number of instructions
2496   uint instruction_count = node_pipeline->instructionCount();
2497   if (node_pipeline->mayHaveNoCode() && n->size(_regalloc) == 0)
2498     instruction_count = 0;
2499 
2500   // Compute the latency information
2501   uint delay = 0;
2502 
2503   if (instruction_count > 0 || !node_pipeline->mayHaveNoCode()) {
2504     int relative_latency = _current_latency[n->_idx] - _bundle_cycle_number;
2505     if (relative_latency < 0)
2506       relative_latency = 0;
2507 
2508     delay = _bundle_use.full_latency(relative_latency, node_usage);
2509 
2510     // Does not fit in this bundle, start a new one
2511     if (delay > 0) {
2512       step(delay);
2513 
2514 #ifndef PRODUCT
2515       if (_cfg->C->trace_opto_output())
2516         tty->print("#  *** STEP(%d) ***\n", delay);
2517 #endif
2518     }
2519   }
2520 
2521   // If this was placed in the delay slot, ignore it
2522   if (n != _unconditional_delay_slot) {
2523 
2524     if (delay == 0) {
2525       if (node_pipeline->hasMultipleBundles()) {
2526 #ifndef PRODUCT
2527         if (_cfg->C->trace_opto_output())
2528           tty->print("#  *** STEP(multiple instructions) ***\n");
2529 #endif
2530         step(1);
2531       }
2532 
2533       else if (instruction_count + _bundle_instr_count > Pipeline::_max_instrs_per_cycle) {
2534 #ifndef PRODUCT
2535         if (_cfg->C->trace_opto_output())
2536           tty->print("#  *** STEP(%d >= %d instructions) ***\n",
2537                      instruction_count + _bundle_instr_count,
2538                      Pipeline::_max_instrs_per_cycle);
2539 #endif
2540         step(1);
2541       }
2542     }
2543 
2544     if (node_pipeline->hasBranchDelay() && !_unconditional_delay_slot)
2545       _bundle_instr_count++;
2546 
2547     // Set the node's latency
2548     _current_latency[n->_idx] = _bundle_cycle_number;
2549 
2550     // Now merge the functional unit information
2551     if (instruction_count > 0 || !node_pipeline->mayHaveNoCode())
2552       _bundle_use.add_usage(node_usage);
2553 
2554     // Increment the number of instructions in this bundle
2555     _bundle_instr_count += instruction_count;
2556 
2557     // Remember this node for later
2558     if (n->is_Mach())
2559       _next_node = n;
2560   }
2561 
2562   // It's possible to have a BoxLock in the graph and in the _bbs mapping but
2563   // not in the bb->_nodes array.  This happens for debug-info-only BoxLocks.
2564   // 'Schedule' them (basically ignore in the schedule) but do not insert them
2565   // into the block.  All other scheduled nodes get put in the schedule here.
2566   int op = n->Opcode();
2567   if( (op == Op_Node && n->req() == 0) || // anti-dependence node OR
2568       (op != Op_Node &&         // Not an unused antidepedence node and
2569        // not an unallocated boxlock
2570        (OptoReg::is_valid(_regalloc->get_reg_first(n)) || op != Op_BoxLock)) ) {
2571 
2572     // Push any trailing projections
2573     if( bb->get_node(bb->number_of_nodes()-1) != n ) {
2574       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
2575         Node *foi = n->fast_out(i);
2576         if( foi->is_Proj() )
2577           _scheduled.push(foi);
2578       }
2579     }
2580 
2581     // Put the instruction in the schedule list
2582     _scheduled.push(n);
2583   }
2584 
2585 #ifndef PRODUCT
2586   if (_cfg->C->trace_opto_output())
2587     dump_available();
2588 #endif
2589 
2590   // Walk all the definitions, decrementing use counts, and
2591   // if a definition has a 0 use count, place it in the available list.
2592   DecrementUseCounts(n,bb);
2593 }
2594 
2595 // This method sets the use count within a basic block.  We will ignore all
2596 // uses outside the current basic block.  As we are doing a backwards walk,
2597 // any node we reach that has a use count of 0 may be scheduled.  This also
2598 // avoids the problem of cyclic references from phi nodes, as long as phi
2599 // nodes are at the front of the basic block.  This method also initializes
2600 // the available list to the set of instructions that have no uses within this
2601 // basic block.
2602 void Scheduling::ComputeUseCount(const Block *bb) {
2603 #ifndef PRODUCT
2604   if (_cfg->C->trace_opto_output())
2605     tty->print("# -> ComputeUseCount\n");
2606 #endif
2607 
2608   // Clear the list of available and scheduled instructions, just in case
2609   _available.clear();
2610   _scheduled.clear();
2611 
2612   // No delay slot specified
2613   _unconditional_delay_slot = NULL;
2614 
2615 #ifdef ASSERT
2616   for( uint i=0; i < bb->number_of_nodes(); i++ )
2617     assert( _uses[bb->get_node(i)->_idx] == 0, "_use array not clean" );
2618 #endif
2619 
2620   // Force the _uses count to never go to zero for unscheduable pieces
2621   // of the block
2622   for( uint k = 0; k < _bb_start; k++ )
2623     _uses[bb->get_node(k)->_idx] = 1;
2624   for( uint l = _bb_end; l < bb->number_of_nodes(); l++ )
2625     _uses[bb->get_node(l)->_idx] = 1;
2626 
2627   // Iterate backwards over the instructions in the block.  Don't count the
2628   // branch projections at end or the block header instructions.
2629   for( uint j = _bb_end-1; j >= _bb_start; j-- ) {
2630     Node *n = bb->get_node(j);
2631     if( n->is_Proj() ) continue; // Projections handled another way
2632 
2633     // Account for all uses
2634     for ( uint k = 0; k < n->len(); k++ ) {
2635       Node *inp = n->in(k);
2636       if (!inp) continue;
2637       assert(inp != n, "no cycles allowed" );
2638       if (_cfg->get_block_for_node(inp) == bb) { // Block-local use?
2639         if (inp->is_Proj()) { // Skip through Proj's
2640           inp = inp->in(0);
2641         }
2642         ++_uses[inp->_idx];     // Count 1 block-local use
2643       }
2644     }
2645 
2646     // If this instruction has a 0 use count, then it is available
2647     if (!_uses[n->_idx]) {
2648       _current_latency[n->_idx] = _bundle_cycle_number;
2649       AddNodeToAvailableList(n);
2650     }
2651 
2652 #ifndef PRODUCT
2653     if (_cfg->C->trace_opto_output()) {
2654       tty->print("#   uses: %3d: ", _uses[n->_idx]);
2655       n->dump();
2656     }
2657 #endif
2658   }
2659 
2660 #ifndef PRODUCT
2661   if (_cfg->C->trace_opto_output())
2662     tty->print("# <- ComputeUseCount\n");
2663 #endif
2664 }
2665 
2666 // This routine performs scheduling on each basic block in reverse order,
2667 // using instruction latencies and taking into account function unit
2668 // availability.
2669 void Scheduling::DoScheduling() {
2670 #ifndef PRODUCT
2671   if (_cfg->C->trace_opto_output())
2672     tty->print("# -> DoScheduling\n");
2673 #endif
2674 
2675   Block *succ_bb = NULL;
2676   Block *bb;
2677   Compile* C = Compile::current();
2678 
2679   // Walk over all the basic blocks in reverse order
2680   for (int i = _cfg->number_of_blocks() - 1; i >= 0; succ_bb = bb, i--) {
2681     bb = _cfg->get_block(i);
2682 
2683 #ifndef PRODUCT
2684     if (_cfg->C->trace_opto_output()) {
2685       tty->print("#  Schedule BB#%03d (initial)\n", i);
2686       for (uint j = 0; j < bb->number_of_nodes(); j++) {
2687         bb->get_node(j)->dump();
2688       }
2689     }
2690 #endif
2691 
2692     // On the head node, skip processing
2693     if (bb == _cfg->get_root_block()) {
2694       continue;
2695     }
2696 
2697     // Skip empty, connector blocks
2698     if (bb->is_connector())
2699       continue;
2700 
2701     // If the following block is not the sole successor of
2702     // this one, then reset the pipeline information
2703     if (bb->_num_succs != 1 || bb->non_connector_successor(0) != succ_bb) {
2704 #ifndef PRODUCT
2705       if (_cfg->C->trace_opto_output()) {
2706         tty->print("*** bundle start of next BB, node %d, for %d instructions\n",
2707                    _next_node->_idx, _bundle_instr_count);
2708       }
2709 #endif
2710       step_and_clear();
2711     }
2712 
2713     // Leave untouched the starting instruction, any Phis, a CreateEx node
2714     // or Top.  bb->get_node(_bb_start) is the first schedulable instruction.
2715     _bb_end = bb->number_of_nodes()-1;
2716     for( _bb_start=1; _bb_start <= _bb_end; _bb_start++ ) {
2717       Node *n = bb->get_node(_bb_start);
2718       // Things not matched, like Phinodes and ProjNodes don't get scheduled.
2719       // Also, MachIdealNodes do not get scheduled
2720       if( !n->is_Mach() ) continue;     // Skip non-machine nodes
2721       MachNode *mach = n->as_Mach();
2722       int iop = mach->ideal_Opcode();
2723       if( iop == Op_CreateEx ) continue; // CreateEx is pinned
2724       if( iop == Op_Con ) continue;      // Do not schedule Top
2725       if( iop == Op_Node &&     // Do not schedule PhiNodes, ProjNodes
2726           mach->pipeline() == MachNode::pipeline_class() &&
2727           !n->is_SpillCopy() && !n->is_MachMerge() )  // Breakpoints, Prolog, etc
2728         continue;
2729       break;                    // Funny loop structure to be sure...
2730     }
2731     // Compute last "interesting" instruction in block - last instruction we
2732     // might schedule.  _bb_end points just after last schedulable inst.  We
2733     // normally schedule conditional branches (despite them being forced last
2734     // in the block), because they have delay slots we can fill.  Calls all
2735     // have their delay slots filled in the template expansions, so we don't
2736     // bother scheduling them.
2737     Node *last = bb->get_node(_bb_end);
2738     // Ignore trailing NOPs.
2739     while (_bb_end > 0 && last->is_Mach() &&
2740            last->as_Mach()->ideal_Opcode() == Op_Con) {
2741       last = bb->get_node(--_bb_end);
2742     }
2743     assert(!last->is_Mach() || last->as_Mach()->ideal_Opcode() != Op_Con, "");
2744     if( last->is_Catch() ||
2745         (last->is_Mach() && last->as_Mach()->ideal_Opcode() == Op_Halt) ) {
2746       // There might be a prior call.  Skip it.
2747       while (_bb_start < _bb_end && bb->get_node(--_bb_end)->is_MachProj());
2748     } else if( last->is_MachNullCheck() ) {
2749       // Backup so the last null-checked memory instruction is
2750       // outside the schedulable range. Skip over the nullcheck,
2751       // projection, and the memory nodes.
2752       Node *mem = last->in(1);
2753       do {
2754         _bb_end--;
2755       } while (mem != bb->get_node(_bb_end));
2756     } else {
2757       // Set _bb_end to point after last schedulable inst.
2758       _bb_end++;
2759     }
2760 
2761     assert( _bb_start <= _bb_end, "inverted block ends" );
2762 
2763     // Compute the register antidependencies for the basic block
2764     ComputeRegisterAntidependencies(bb);
2765     if (C->failing())  return;  // too many D-U pinch points
2766 
2767     // Compute intra-bb latencies for the nodes
2768     ComputeLocalLatenciesForward(bb);
2769 
2770     // Compute the usage within the block, and set the list of all nodes
2771     // in the block that have no uses within the block.
2772     ComputeUseCount(bb);
2773 
2774     // Schedule the remaining instructions in the block
2775     while ( _available.size() > 0 ) {
2776       Node *n = ChooseNodeToBundle();
2777       guarantee(n != NULL, "no nodes available");
2778       AddNodeToBundle(n,bb);
2779     }
2780 
2781     assert( _scheduled.size() == _bb_end - _bb_start, "wrong number of instructions" );
2782 #ifdef ASSERT
2783     for( uint l = _bb_start; l < _bb_end; l++ ) {
2784       Node *n = bb->get_node(l);
2785       uint m;
2786       for( m = 0; m < _bb_end-_bb_start; m++ )
2787         if( _scheduled[m] == n )
2788           break;
2789       assert( m < _bb_end-_bb_start, "instruction missing in schedule" );
2790     }
2791 #endif
2792 
2793     // Now copy the instructions (in reverse order) back to the block
2794     for ( uint k = _bb_start; k < _bb_end; k++ )
2795       bb->map_node(_scheduled[_bb_end-k-1], k);
2796 
2797 #ifndef PRODUCT
2798     if (_cfg->C->trace_opto_output()) {
2799       tty->print("#  Schedule BB#%03d (final)\n", i);
2800       uint current = 0;
2801       for (uint j = 0; j < bb->number_of_nodes(); j++) {
2802         Node *n = bb->get_node(j);
2803         if( valid_bundle_info(n) ) {
2804           Bundle *bundle = node_bundling(n);
2805           if (bundle->instr_count() > 0 || bundle->flags() > 0) {
2806             tty->print("*** Bundle: ");
2807             bundle->dump();
2808           }
2809           n->dump();
2810         }
2811       }
2812     }
2813 #endif
2814 #ifdef ASSERT
2815     verify_good_schedule(bb,"after block local scheduling");
2816 #endif
2817   }
2818 
2819 #ifndef PRODUCT
2820   if (_cfg->C->trace_opto_output())
2821     tty->print("# <- DoScheduling\n");
2822 #endif
2823 
2824   // Record final node-bundling array location
2825   _regalloc->C->output()->set_node_bundling_base(_node_bundling_base);
2826 
2827 } // end DoScheduling
2828 
2829 // Verify that no live-range used in the block is killed in the block by a
2830 // wrong DEF.  This doesn't verify live-ranges that span blocks.
2831 
2832 // Check for edge existence.  Used to avoid adding redundant precedence edges.
2833 static bool edge_from_to( Node *from, Node *to ) {
2834   for( uint i=0; i<from->len(); i++ )
2835     if( from->in(i) == to )
2836       return true;
2837   return false;
2838 }
2839 
2840 #ifdef ASSERT
2841 void Scheduling::verify_do_def( Node *n, OptoReg::Name def, const char *msg ) {
2842   // Check for bad kills
2843   if( OptoReg::is_valid(def) ) { // Ignore stores & control flow
2844     Node *prior_use = _reg_node[def];
2845     if( prior_use && !edge_from_to(prior_use,n) ) {
2846       tty->print("%s = ",OptoReg::as_VMReg(def)->name());
2847       n->dump();
2848       tty->print_cr("...");
2849       prior_use->dump();
2850       assert(edge_from_to(prior_use,n), "%s", msg);
2851     }
2852     _reg_node.map(def,NULL); // Kill live USEs
2853   }
2854 }
2855 
2856 void Scheduling::verify_good_schedule( Block *b, const char *msg ) {
2857 
2858   // Zap to something reasonable for the verify code
2859   _reg_node.clear();
2860 
2861   // Walk over the block backwards.  Check to make sure each DEF doesn't
2862   // kill a live value (other than the one it's supposed to).  Add each
2863   // USE to the live set.
2864   for( uint i = b->number_of_nodes()-1; i >= _bb_start; i-- ) {
2865     Node *n = b->get_node(i);
2866     int n_op = n->Opcode();
2867     if( n_op == Op_MachProj && n->ideal_reg() == MachProjNode::fat_proj ) {
2868       // Fat-proj kills a slew of registers
2869       RegMaskIterator rmi(n->out_RegMask());
2870       while (rmi.has_next()) {
2871         OptoReg::Name kill = rmi.next();
2872         verify_do_def(n, kill, msg);
2873       }
2874     } else if( n_op != Op_Node ) { // Avoid brand new antidependence nodes
2875       // Get DEF'd registers the normal way
2876       verify_do_def( n, _regalloc->get_reg_first(n), msg );
2877       verify_do_def( n, _regalloc->get_reg_second(n), msg );
2878     }
2879 
2880     // Now make all USEs live
2881     for( uint i=1; i<n->req(); i++ ) {
2882       Node *def = n->in(i);
2883       assert(def != 0, "input edge required");
2884       OptoReg::Name reg_lo = _regalloc->get_reg_first(def);
2885       OptoReg::Name reg_hi = _regalloc->get_reg_second(def);
2886       if( OptoReg::is_valid(reg_lo) ) {
2887         assert(!_reg_node[reg_lo] || edge_from_to(_reg_node[reg_lo],def), "%s", msg);
2888         _reg_node.map(reg_lo,n);
2889       }
2890       if( OptoReg::is_valid(reg_hi) ) {
2891         assert(!_reg_node[reg_hi] || edge_from_to(_reg_node[reg_hi],def), "%s", msg);
2892         _reg_node.map(reg_hi,n);
2893       }
2894     }
2895 
2896   }
2897 
2898   // Zap to something reasonable for the Antidependence code
2899   _reg_node.clear();
2900 }
2901 #endif
2902 
2903 // Conditionally add precedence edges.  Avoid putting edges on Projs.
2904 static void add_prec_edge_from_to( Node *from, Node *to ) {
2905   if( from->is_Proj() ) {       // Put precedence edge on Proj's input
2906     assert( from->req() == 1 && (from->len() == 1 || from->in(1)==0), "no precedence edges on projections" );
2907     from = from->in(0);
2908   }
2909   if( from != to &&             // No cycles (for things like LD L0,[L0+4] )
2910       !edge_from_to( from, to ) ) // Avoid duplicate edge
2911     from->add_prec(to);
2912 }
2913 
2914 void Scheduling::anti_do_def( Block *b, Node *def, OptoReg::Name def_reg, int is_def ) {
2915   if( !OptoReg::is_valid(def_reg) ) // Ignore stores & control flow
2916     return;
2917 
2918   if (OptoReg::is_reg(def_reg)) {
2919     VMReg vmreg = OptoReg::as_VMReg(def_reg);
2920     if (vmreg->is_reg() && !vmreg->is_concrete() && !vmreg->prev()->is_concrete()) {
2921       // This is one of the high slots of a vector register.
2922       // ScheduleAndBundle already checked there are no live wide
2923       // vectors in this method so it can be safely ignored.
2924       return;
2925     }
2926   }
2927 
2928   Node *pinch = _reg_node[def_reg]; // Get pinch point
2929   if ((pinch == NULL) || _cfg->get_block_for_node(pinch) != b || // No pinch-point yet?
2930       is_def ) {    // Check for a true def (not a kill)
2931     _reg_node.map(def_reg,def); // Record def/kill as the optimistic pinch-point
2932     return;
2933   }
2934 
2935   Node *kill = def;             // Rename 'def' to more descriptive 'kill'
2936   debug_only( def = (Node*)((intptr_t)0xdeadbeef); )
2937 
2938   // After some number of kills there _may_ be a later def
2939   Node *later_def = NULL;
2940 
2941   Compile* C = Compile::current();
2942 
2943   // Finding a kill requires a real pinch-point.
2944   // Check for not already having a pinch-point.
2945   // Pinch points are Op_Node's.
2946   if( pinch->Opcode() != Op_Node ) { // Or later-def/kill as pinch-point?
2947     later_def = pinch;            // Must be def/kill as optimistic pinch-point
2948     if ( _pinch_free_list.size() > 0) {
2949       pinch = _pinch_free_list.pop();
2950     } else {
2951       pinch = new Node(1); // Pinch point to-be
2952     }
2953     if (pinch->_idx >= _regalloc->node_regs_max_index()) {
2954       _cfg->C->record_method_not_compilable("too many D-U pinch points");
2955       return;
2956     }
2957     _cfg->map_node_to_block(pinch, b);      // Pretend it's valid in this block (lazy init)
2958     _reg_node.map(def_reg,pinch); // Record pinch-point
2959     //regalloc()->set_bad(pinch->_idx); // Already initialized this way.
2960     if( later_def->outcnt() == 0 || later_def->ideal_reg() == MachProjNode::fat_proj ) { // Distinguish def from kill
2961       pinch->init_req(0, C->top());     // set not NULL for the next call
2962       add_prec_edge_from_to(later_def,pinch); // Add edge from kill to pinch
2963       later_def = NULL;           // and no later def
2964     }
2965     pinch->set_req(0,later_def);  // Hook later def so we can find it
2966   } else {                        // Else have valid pinch point
2967     if( pinch->in(0) )            // If there is a later-def
2968       later_def = pinch->in(0);   // Get it
2969   }
2970 
2971   // Add output-dependence edge from later def to kill
2972   if( later_def )               // If there is some original def
2973     add_prec_edge_from_to(later_def,kill); // Add edge from def to kill
2974 
2975   // See if current kill is also a use, and so is forced to be the pinch-point.
2976   if( pinch->Opcode() == Op_Node ) {
2977     Node *uses = kill->is_Proj() ? kill->in(0) : kill;
2978     for( uint i=1; i<uses->req(); i++ ) {
2979       if( _regalloc->get_reg_first(uses->in(i)) == def_reg ||
2980           _regalloc->get_reg_second(uses->in(i)) == def_reg ) {
2981         // Yes, found a use/kill pinch-point
2982         pinch->set_req(0,NULL);  //
2983         pinch->replace_by(kill); // Move anti-dep edges up
2984         pinch = kill;
2985         _reg_node.map(def_reg,pinch);
2986         return;
2987       }
2988     }
2989   }
2990 
2991   // Add edge from kill to pinch-point
2992   add_prec_edge_from_to(kill,pinch);
2993 }
2994 
2995 void Scheduling::anti_do_use( Block *b, Node *use, OptoReg::Name use_reg ) {
2996   if( !OptoReg::is_valid(use_reg) ) // Ignore stores & control flow
2997     return;
2998   Node *pinch = _reg_node[use_reg]; // Get pinch point
2999   // Check for no later def_reg/kill in block
3000   if ((pinch != NULL) && _cfg->get_block_for_node(pinch) == b &&
3001       // Use has to be block-local as well
3002       _cfg->get_block_for_node(use) == b) {
3003     if( pinch->Opcode() == Op_Node && // Real pinch-point (not optimistic?)
3004         pinch->req() == 1 ) {   // pinch not yet in block?
3005       pinch->del_req(0);        // yank pointer to later-def, also set flag
3006       // Insert the pinch-point in the block just after the last use
3007       b->insert_node(pinch, b->find_node(use) + 1);
3008       _bb_end++;                // Increase size scheduled region in block
3009     }
3010 
3011     add_prec_edge_from_to(pinch,use);
3012   }
3013 }
3014 
3015 // We insert antidependences between the reads and following write of
3016 // allocated registers to prevent illegal code motion. Hopefully, the
3017 // number of added references should be fairly small, especially as we
3018 // are only adding references within the current basic block.
3019 void Scheduling::ComputeRegisterAntidependencies(Block *b) {
3020 
3021 #ifdef ASSERT
3022   verify_good_schedule(b,"before block local scheduling");
3023 #endif
3024 
3025   // A valid schedule, for each register independently, is an endless cycle
3026   // of: a def, then some uses (connected to the def by true dependencies),
3027   // then some kills (defs with no uses), finally the cycle repeats with a new
3028   // def.  The uses are allowed to float relative to each other, as are the
3029   // kills.  No use is allowed to slide past a kill (or def).  This requires
3030   // antidependencies between all uses of a single def and all kills that
3031   // follow, up to the next def.  More edges are redundant, because later defs
3032   // & kills are already serialized with true or antidependencies.  To keep
3033   // the edge count down, we add a 'pinch point' node if there's more than
3034   // one use or more than one kill/def.
3035 
3036   // We add dependencies in one bottom-up pass.
3037 
3038   // For each instruction we handle it's DEFs/KILLs, then it's USEs.
3039 
3040   // For each DEF/KILL, we check to see if there's a prior DEF/KILL for this
3041   // register.  If not, we record the DEF/KILL in _reg_node, the
3042   // register-to-def mapping.  If there is a prior DEF/KILL, we insert a
3043   // "pinch point", a new Node that's in the graph but not in the block.
3044   // We put edges from the prior and current DEF/KILLs to the pinch point.
3045   // We put the pinch point in _reg_node.  If there's already a pinch point
3046   // we merely add an edge from the current DEF/KILL to the pinch point.
3047 
3048   // After doing the DEF/KILLs, we handle USEs.  For each used register, we
3049   // put an edge from the pinch point to the USE.
3050 
3051   // To be expedient, the _reg_node array is pre-allocated for the whole
3052   // compilation.  _reg_node is lazily initialized; it either contains a NULL,
3053   // or a valid def/kill/pinch-point, or a leftover node from some prior
3054   // block.  Leftover node from some prior block is treated like a NULL (no
3055   // prior def, so no anti-dependence needed).  Valid def is distinguished by
3056   // it being in the current block.
3057   bool fat_proj_seen = false;
3058   uint last_safept = _bb_end-1;
3059   Node* end_node         = (_bb_end-1 >= _bb_start) ? b->get_node(last_safept) : NULL;
3060   Node* last_safept_node = end_node;
3061   for( uint i = _bb_end-1; i >= _bb_start; i-- ) {
3062     Node *n = b->get_node(i);
3063     int is_def = n->outcnt();   // def if some uses prior to adding precedence edges
3064     if( n->is_MachProj() && n->ideal_reg() == MachProjNode::fat_proj ) {
3065       // Fat-proj kills a slew of registers
3066       // This can add edges to 'n' and obscure whether or not it was a def,
3067       // hence the is_def flag.
3068       fat_proj_seen = true;
3069       RegMaskIterator rmi(n->out_RegMask());
3070       while (rmi.has_next()) {
3071         OptoReg::Name kill = rmi.next();
3072         anti_do_def(b, n, kill, is_def);
3073       }
3074     } else {
3075       // Get DEF'd registers the normal way
3076       anti_do_def( b, n, _regalloc->get_reg_first(n), is_def );
3077       anti_do_def( b, n, _regalloc->get_reg_second(n), is_def );
3078     }
3079 
3080     // Kill projections on a branch should appear to occur on the
3081     // branch, not afterwards, so grab the masks from the projections
3082     // and process them.
3083     if (n->is_MachBranch() || (n->is_Mach() && n->as_Mach()->ideal_Opcode() == Op_Jump)) {
3084       for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) {
3085         Node* use = n->fast_out(i);
3086         if (use->is_Proj()) {
3087           RegMaskIterator rmi(use->out_RegMask());
3088           while (rmi.has_next()) {
3089             OptoReg::Name kill = rmi.next();
3090             anti_do_def(b, n, kill, false);
3091           }
3092         }
3093       }
3094     }
3095 
3096     // Check each register used by this instruction for a following DEF/KILL
3097     // that must occur afterward and requires an anti-dependence edge.
3098     for( uint j=0; j<n->req(); j++ ) {
3099       Node *def = n->in(j);
3100       if( def ) {
3101         assert( !def->is_MachProj() || def->ideal_reg() != MachProjNode::fat_proj, "" );
3102         anti_do_use( b, n, _regalloc->get_reg_first(def) );
3103         anti_do_use( b, n, _regalloc->get_reg_second(def) );
3104       }
3105     }
3106     // Do not allow defs of new derived values to float above GC
3107     // points unless the base is definitely available at the GC point.
3108 
3109     Node *m = b->get_node(i);
3110 
3111     // Add precedence edge from following safepoint to use of derived pointer
3112     if( last_safept_node != end_node &&
3113         m != last_safept_node) {
3114       for (uint k = 1; k < m->req(); k++) {
3115         const Type *t = m->in(k)->bottom_type();
3116         if( t->isa_oop_ptr() &&
3117             t->is_ptr()->offset() != 0 ) {
3118           last_safept_node->add_prec( m );
3119           break;
3120         }
3121       }
3122     }
3123 
3124     if( n->jvms() ) {           // Precedence edge from derived to safept
3125       // Check if last_safept_node was moved by pinch-point insertion in anti_do_use()
3126       if( b->get_node(last_safept) != last_safept_node ) {
3127         last_safept = b->find_node(last_safept_node);
3128       }
3129       for( uint j=last_safept; j > i; j-- ) {
3130         Node *mach = b->get_node(j);
3131         if( mach->is_Mach() && mach->as_Mach()->ideal_Opcode() == Op_AddP )
3132           mach->add_prec( n );
3133       }
3134       last_safept = i;
3135       last_safept_node = m;
3136     }
3137   }
3138 
3139   if (fat_proj_seen) {
3140     // Garbage collect pinch nodes that were not consumed.
3141     // They are usually created by a fat kill MachProj for a call.
3142     garbage_collect_pinch_nodes();
3143   }
3144 }
3145 
3146 // Garbage collect pinch nodes for reuse by other blocks.
3147 //
3148 // The block scheduler's insertion of anti-dependence
3149 // edges creates many pinch nodes when the block contains
3150 // 2 or more Calls.  A pinch node is used to prevent a
3151 // combinatorial explosion of edges.  If a set of kills for a
3152 // register is anti-dependent on a set of uses (or defs), rather
3153 // than adding an edge in the graph between each pair of kill
3154 // and use (or def), a pinch is inserted between them:
3155 //
3156 //            use1   use2  use3
3157 //                \   |   /
3158 //                 \  |  /
3159 //                  pinch
3160 //                 /  |  \
3161 //                /   |   \
3162 //            kill1 kill2 kill3
3163 //
3164 // One pinch node is created per register killed when
3165 // the second call is encountered during a backwards pass
3166 // over the block.  Most of these pinch nodes are never
3167 // wired into the graph because the register is never
3168 // used or def'ed in the block.
3169 //
3170 void Scheduling::garbage_collect_pinch_nodes() {
3171 #ifndef PRODUCT
3172   if (_cfg->C->trace_opto_output()) tty->print("Reclaimed pinch nodes:");
3173 #endif
3174   int trace_cnt = 0;
3175   for (uint k = 0; k < _reg_node.Size(); k++) {
3176     Node* pinch = _reg_node[k];
3177     if ((pinch != NULL) && pinch->Opcode() == Op_Node &&
3178         // no predecence input edges
3179         (pinch->req() == pinch->len() || pinch->in(pinch->req()) == NULL) ) {
3180       cleanup_pinch(pinch);
3181       _pinch_free_list.push(pinch);
3182       _reg_node.map(k, NULL);
3183 #ifndef PRODUCT
3184       if (_cfg->C->trace_opto_output()) {
3185         trace_cnt++;
3186         if (trace_cnt > 40) {
3187           tty->print("\n");
3188           trace_cnt = 0;
3189         }
3190         tty->print(" %d", pinch->_idx);
3191       }
3192 #endif
3193     }
3194   }
3195 #ifndef PRODUCT
3196   if (_cfg->C->trace_opto_output()) tty->print("\n");
3197 #endif
3198 }
3199 
3200 // Clean up a pinch node for reuse.
3201 void Scheduling::cleanup_pinch( Node *pinch ) {
3202   assert (pinch && pinch->Opcode() == Op_Node && pinch->req() == 1, "just checking");
3203 
3204   for (DUIterator_Last imin, i = pinch->last_outs(imin); i >= imin; ) {
3205     Node* use = pinch->last_out(i);
3206     uint uses_found = 0;
3207     for (uint j = use->req(); j < use->len(); j++) {
3208       if (use->in(j) == pinch) {
3209         use->rm_prec(j);
3210         uses_found++;
3211       }
3212     }
3213     assert(uses_found > 0, "must be a precedence edge");
3214     i -= uses_found;    // we deleted 1 or more copies of this edge
3215   }
3216   // May have a later_def entry
3217   pinch->set_req(0, NULL);
3218 }
3219 
3220 #ifndef PRODUCT
3221 
3222 void Scheduling::dump_available() const {
3223   tty->print("#Availist  ");
3224   for (uint i = 0; i < _available.size(); i++)
3225     tty->print(" N%d/l%d", _available[i]->_idx,_current_latency[_available[i]->_idx]);
3226   tty->cr();
3227 }
3228 
3229 // Print Scheduling Statistics
3230 void Scheduling::print_statistics() {
3231   // Print the size added by nops for bundling
3232   tty->print("Nops added %d bytes to total of %d bytes",
3233              _total_nop_size, _total_method_size);
3234   if (_total_method_size > 0)
3235     tty->print(", for %.2f%%",
3236                ((double)_total_nop_size) / ((double) _total_method_size) * 100.0);
3237   tty->print("\n");
3238 
3239   // Print the number of branch shadows filled
3240   if (Pipeline::_branch_has_delay_slot) {
3241     tty->print("Of %d branches, %d had unconditional delay slots filled",
3242                _total_branches, _total_unconditional_delays);
3243     if (_total_branches > 0)
3244       tty->print(", for %.2f%%",
3245                  ((double)_total_unconditional_delays) / ((double)_total_branches) * 100.0);
3246     tty->print("\n");
3247   }
3248 
3249   uint total_instructions = 0, total_bundles = 0;
3250 
3251   for (uint i = 1; i <= Pipeline::_max_instrs_per_cycle; i++) {
3252     uint bundle_count   = _total_instructions_per_bundle[i];
3253     total_instructions += bundle_count * i;
3254     total_bundles      += bundle_count;
3255   }
3256 
3257   if (total_bundles > 0)
3258     tty->print("Average ILP (excluding nops) is %.2f\n",
3259                ((double)total_instructions) / ((double)total_bundles));
3260 }
3261 #endif
3262 
3263 //-----------------------init_scratch_buffer_blob------------------------------
3264 // Construct a temporary BufferBlob and cache it for this compile.
3265 void PhaseOutput::init_scratch_buffer_blob(int const_size) {
3266   // If there is already a scratch buffer blob allocated and the
3267   // constant section is big enough, use it.  Otherwise free the
3268   // current and allocate a new one.
3269   BufferBlob* blob = scratch_buffer_blob();
3270   if ((blob != NULL) && (const_size <= _scratch_const_size)) {
3271     // Use the current blob.
3272   } else {
3273     if (blob != NULL) {
3274       BufferBlob::free(blob);
3275     }
3276 
3277     ResourceMark rm;
3278     _scratch_const_size = const_size;
3279     int size = C2Compiler::initial_code_buffer_size(const_size);
3280     blob = BufferBlob::create("Compile::scratch_buffer", size);
3281     // Record the buffer blob for next time.
3282     set_scratch_buffer_blob(blob);
3283     // Have we run out of code space?
3284     if (scratch_buffer_blob() == NULL) {
3285       // Let CompilerBroker disable further compilations.
3286       C->record_failure("Not enough space for scratch buffer in CodeCache");
3287       return;
3288     }
3289   }
3290 
3291   // Initialize the relocation buffers
3292   relocInfo* locs_buf = (relocInfo*) blob->content_end() - MAX_locs_size;
3293   set_scratch_locs_memory(locs_buf);
3294 }
3295 
3296 
3297 //-----------------------scratch_emit_size-------------------------------------
3298 // Helper function that computes size by emitting code
3299 uint PhaseOutput::scratch_emit_size(const Node* n) {
3300   // Start scratch_emit_size section.
3301   set_in_scratch_emit_size(true);
3302 
3303   // Emit into a trash buffer and count bytes emitted.
3304   // This is a pretty expensive way to compute a size,
3305   // but it works well enough if seldom used.
3306   // All common fixed-size instructions are given a size
3307   // method by the AD file.
3308   // Note that the scratch buffer blob and locs memory are
3309   // allocated at the beginning of the compile task, and
3310   // may be shared by several calls to scratch_emit_size.
3311   // The allocation of the scratch buffer blob is particularly
3312   // expensive, since it has to grab the code cache lock.
3313   BufferBlob* blob = this->scratch_buffer_blob();
3314   assert(blob != NULL, "Initialize BufferBlob at start");
3315   assert(blob->size() > MAX_inst_size, "sanity");
3316   relocInfo* locs_buf = scratch_locs_memory();
3317   address blob_begin = blob->content_begin();
3318   address blob_end   = (address)locs_buf;
3319   assert(blob->contains(blob_end), "sanity");
3320   CodeBuffer buf(blob_begin, blob_end - blob_begin);
3321   buf.initialize_consts_size(_scratch_const_size);
3322   buf.initialize_stubs_size(MAX_stubs_size);
3323   assert(locs_buf != NULL, "sanity");
3324   int lsize = MAX_locs_size / 3;
3325   buf.consts()->initialize_shared_locs(&locs_buf[lsize * 0], lsize);
3326   buf.insts()->initialize_shared_locs( &locs_buf[lsize * 1], lsize);
3327   buf.stubs()->initialize_shared_locs( &locs_buf[lsize * 2], lsize);
3328   // Mark as scratch buffer.
3329   buf.consts()->set_scratch_emit();
3330   buf.insts()->set_scratch_emit();
3331   buf.stubs()->set_scratch_emit();
3332 
3333   // Do the emission.
3334 
3335   Label fakeL; // Fake label for branch instructions.
3336   Label*   saveL = NULL;
3337   uint save_bnum = 0;
3338   bool is_branch = n->is_MachBranch();
3339   if (is_branch) {
3340     MacroAssembler masm(&buf);
3341     masm.bind(fakeL);
3342     n->as_MachBranch()->save_label(&saveL, &save_bnum);
3343     n->as_MachBranch()->label_set(&fakeL, 0);
3344   }
3345   n->emit(buf, C->regalloc());
3346 
3347   // Emitting into the scratch buffer should not fail
3348   assert (!C->failing(), "Must not have pending failure. Reason is: %s", C->failure_reason());
3349 
3350   if (is_branch) // Restore label.
3351     n->as_MachBranch()->label_set(saveL, save_bnum);
3352 
3353   // End scratch_emit_size section.
3354   set_in_scratch_emit_size(false);
3355 
3356   return buf.insts_size();
3357 }
3358 
3359 void PhaseOutput::install() {
3360   if (!C->should_install_code()) {
3361     return;
3362   } else if (C->stub_function() != NULL) {
3363     install_stub(C->stub_name());
3364   } else {
3365     install_code(C->method(),
3366                  C->entry_bci(),
3367                  CompileBroker::compiler2(),
3368                  C->has_unsafe_access(),
3369                  SharedRuntime::is_wide_vector(C->max_vector_size()),
3370                  C->rtm_state());
3371   }
3372 }
3373 
3374 void PhaseOutput::install_code(ciMethod*         target,
3375                                int               entry_bci,
3376                                AbstractCompiler* compiler,
3377                                bool              has_unsafe_access,
3378                                bool              has_wide_vectors,
3379                                RTMState          rtm_state) {
3380   // Check if we want to skip execution of all compiled code.
3381   {
3382 #ifndef PRODUCT
3383     if (OptoNoExecute) {
3384       C->record_method_not_compilable("+OptoNoExecute");  // Flag as failed
3385       return;
3386     }
3387 #endif
3388     Compile::TracePhase tp("install_code", &timers[_t_registerMethod]);
3389 
3390     if (C->is_osr_compilation()) {
3391       _code_offsets.set_value(CodeOffsets::Verified_Entry, 0);
3392       _code_offsets.set_value(CodeOffsets::OSR_Entry, _first_block_size);
3393     } else {
3394       _code_offsets.set_value(CodeOffsets::Verified_Entry, _first_block_size);
3395       _code_offsets.set_value(CodeOffsets::OSR_Entry, 0);
3396     }
3397 
3398     C->env()->register_method(target,
3399                                      entry_bci,
3400                                      &_code_offsets,
3401                                      _orig_pc_slot_offset_in_bytes,
3402                                      code_buffer(),
3403                                      frame_size_in_words(),
3404                                      oop_map_set(),
3405                                      &_handler_table,
3406                                      inc_table(),
3407                                      compiler,
3408                                      has_unsafe_access,
3409                                      SharedRuntime::is_wide_vector(C->max_vector_size()),
3410                                      C->rtm_state(),
3411                                      C->native_invokers());
3412 
3413     if (C->log() != NULL) { // Print code cache state into compiler log
3414       C->log()->code_cache_state();
3415     }
3416   }
3417 }
3418 void PhaseOutput::install_stub(const char* stub_name) {
3419   // Entry point will be accessed using stub_entry_point();
3420   if (code_buffer() == NULL) {
3421     Matcher::soft_match_failure();
3422   } else {
3423     if (PrintAssembly && (WizardMode || Verbose))
3424       tty->print_cr("### Stub::%s", stub_name);
3425 
3426     if (!C->failing()) {
3427       assert(C->fixed_slots() == 0, "no fixed slots used for runtime stubs");
3428 
3429       // Make the NMethod
3430       // For now we mark the frame as never safe for profile stackwalking
3431       RuntimeStub *rs = RuntimeStub::new_runtime_stub(stub_name,
3432                                                       code_buffer(),
3433                                                       CodeOffsets::frame_never_safe,
3434                                                       // _code_offsets.value(CodeOffsets::Frame_Complete),
3435                                                       frame_size_in_words(),
3436                                                       oop_map_set(),
3437                                                       false);
3438       assert(rs != NULL && rs->is_runtime_stub(), "sanity check");
3439 
3440       C->set_stub_entry_point(rs->entry_point());
3441     }
3442   }
3443 }
3444 
3445 // Support for bundling info
3446 Bundle* PhaseOutput::node_bundling(const Node *n) {
3447   assert(valid_bundle_info(n), "oob");
3448   return &_node_bundling_base[n->_idx];
3449 }
3450 
3451 bool PhaseOutput::valid_bundle_info(const Node *n) {
3452   return (_node_bundling_limit > n->_idx);
3453 }
3454 
3455 //------------------------------frame_size_in_words-----------------------------
3456 // frame_slots in units of words
3457 int PhaseOutput::frame_size_in_words() const {
3458   // shift is 0 in LP32 and 1 in LP64
3459   const int shift = (LogBytesPerWord - LogBytesPerInt);
3460   int words = _frame_slots >> shift;
3461   assert( words << shift == _frame_slots, "frame size must be properly aligned in LP64" );
3462   return words;
3463 }
3464 
3465 // To bang the stack of this compiled method we use the stack size
3466 // that the interpreter would need in case of a deoptimization. This
3467 // removes the need to bang the stack in the deoptimization blob which
3468 // in turn simplifies stack overflow handling.
3469 int PhaseOutput::bang_size_in_bytes() const {
3470   return MAX2(frame_size_in_bytes() + os::extra_bang_size_in_bytes(), C->interpreter_frame_size());
3471 }
3472 
3473 //------------------------------dump_asm---------------------------------------
3474 // Dump formatted assembly
3475 #if defined(SUPPORT_OPTO_ASSEMBLY)
3476 void PhaseOutput::dump_asm_on(outputStream* st, int* pcs, uint pc_limit) {
3477 
3478   int pc_digits = 3; // #chars required for pc
3479   int sb_chars  = 3; // #chars for "start bundle" indicator
3480   int tab_size  = 8;
3481   if (pcs != NULL) {
3482     int max_pc = 0;
3483     for (uint i = 0; i < pc_limit; i++) {
3484       max_pc = (max_pc < pcs[i]) ? pcs[i] : max_pc;
3485     }
3486     pc_digits  = ((max_pc < 4096) ? 3 : ((max_pc < 65536) ? 4 : ((max_pc < 65536*256) ? 6 : 8))); // #chars required for pc
3487   }
3488   int prefix_len = ((pc_digits + sb_chars + tab_size - 1)/tab_size)*tab_size;
3489 
3490   bool cut_short = false;
3491   st->print_cr("#");
3492   st->print("#  ");  C->tf()->dump_on(st);  st->cr();
3493   st->print_cr("#");
3494 
3495   // For all blocks
3496   int pc = 0x0;                 // Program counter
3497   char starts_bundle = ' ';
3498   C->regalloc()->dump_frame();
3499 
3500   Node *n = NULL;
3501   for (uint i = 0; i < C->cfg()->number_of_blocks(); i++) {
3502     if (VMThread::should_terminate()) {
3503       cut_short = true;
3504       break;
3505     }
3506     Block* block = C->cfg()->get_block(i);
3507     if (block->is_connector() && !Verbose) {
3508       continue;
3509     }
3510     n = block->head();
3511     if ((pcs != NULL) && (n->_idx < pc_limit)) {
3512       pc = pcs[n->_idx];
3513       st->print("%*.*x", pc_digits, pc_digits, pc);
3514     }
3515     st->fill_to(prefix_len);
3516     block->dump_head(C->cfg(), st);
3517     if (block->is_connector()) {
3518       st->fill_to(prefix_len);
3519       st->print_cr("# Empty connector block");
3520     } else if (block->num_preds() == 2 && block->pred(1)->is_CatchProj() && block->pred(1)->as_CatchProj()->_con == CatchProjNode::fall_through_index) {
3521       st->fill_to(prefix_len);
3522       st->print_cr("# Block is sole successor of call");
3523     }
3524 
3525     // For all instructions
3526     Node *delay = NULL;
3527     for (uint j = 0; j < block->number_of_nodes(); j++) {
3528       if (VMThread::should_terminate()) {
3529         cut_short = true;
3530         break;
3531       }
3532       n = block->get_node(j);
3533       if (valid_bundle_info(n)) {
3534         Bundle* bundle = node_bundling(n);
3535         if (bundle->used_in_unconditional_delay()) {
3536           delay = n;
3537           continue;
3538         }
3539         if (bundle->starts_bundle()) {
3540           starts_bundle = '+';
3541         }
3542       }
3543 
3544       if (WizardMode) {
3545         n->dump();
3546       }
3547 
3548       if( !n->is_Region() &&    // Dont print in the Assembly
3549           !n->is_Phi() &&       // a few noisely useless nodes
3550           !n->is_Proj() &&
3551           !n->is_MachTemp() &&
3552           !n->is_SafePointScalarObject() &&
3553           !n->is_Catch() &&     // Would be nice to print exception table targets
3554           !n->is_MergeMem() &&  // Not very interesting
3555           !n->is_top() &&       // Debug info table constants
3556           !(n->is_Con() && !n->is_Mach())// Debug info table constants
3557           ) {
3558         if ((pcs != NULL) && (n->_idx < pc_limit)) {
3559           pc = pcs[n->_idx];
3560           st->print("%*.*x", pc_digits, pc_digits, pc);
3561         } else {
3562           st->fill_to(pc_digits);
3563         }
3564         st->print(" %c ", starts_bundle);
3565         starts_bundle = ' ';
3566         st->fill_to(prefix_len);
3567         n->format(C->regalloc(), st);
3568         st->cr();
3569       }
3570 
3571       // If we have an instruction with a delay slot, and have seen a delay,
3572       // then back up and print it
3573       if (valid_bundle_info(n) && node_bundling(n)->use_unconditional_delay()) {
3574         // Coverity finding - Explicit null dereferenced.
3575         guarantee(delay != NULL, "no unconditional delay instruction");
3576         if (WizardMode) delay->dump();
3577 
3578         if (node_bundling(delay)->starts_bundle())
3579           starts_bundle = '+';
3580         if ((pcs != NULL) && (n->_idx < pc_limit)) {
3581           pc = pcs[n->_idx];
3582           st->print("%*.*x", pc_digits, pc_digits, pc);
3583         } else {
3584           st->fill_to(pc_digits);
3585         }
3586         st->print(" %c ", starts_bundle);
3587         starts_bundle = ' ';
3588         st->fill_to(prefix_len);
3589         delay->format(C->regalloc(), st);
3590         st->cr();
3591         delay = NULL;
3592       }
3593 
3594       // Dump the exception table as well
3595       if( n->is_Catch() && (Verbose || WizardMode) ) {
3596         // Print the exception table for this offset
3597         _handler_table.print_subtable_for(pc);
3598       }
3599       st->bol(); // Make sure we start on a new line
3600     }
3601     st->cr(); // one empty line between blocks
3602     assert(cut_short || delay == NULL, "no unconditional delay branch");
3603   } // End of per-block dump
3604 
3605   if (cut_short)  st->print_cr("*** disassembly is cut short ***");
3606 }
3607 #endif
3608 
3609 #ifndef PRODUCT
3610 void PhaseOutput::print_statistics() {
3611   Scheduling::print_statistics();
3612 }
3613 #endif