1 /* 2 * Copyright (c) 2005, 2023, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 * 23 */ 24 25 #include "precompiled.hpp" 26 #include "c1/c1_CFGPrinter.hpp" 27 #include "c1/c1_CodeStubs.hpp" 28 #include "c1/c1_Compilation.hpp" 29 #include "c1/c1_FrameMap.hpp" 30 #include "c1/c1_IR.hpp" 31 #include "c1/c1_LIRGenerator.hpp" 32 #include "c1/c1_LinearScan.hpp" 33 #include "c1/c1_ValueStack.hpp" 34 #include "code/vmreg.inline.hpp" 35 #include "runtime/timerTrace.hpp" 36 #include "utilities/bitMap.inline.hpp" 37 38 #ifndef PRODUCT 39 40 static LinearScanStatistic _stat_before_alloc; 41 static LinearScanStatistic _stat_after_asign; 42 static LinearScanStatistic _stat_final; 43 44 static LinearScanTimers _total_timer; 45 46 // helper macro for short definition of timer 47 #define TIME_LINEAR_SCAN(timer_name) TraceTime _block_timer("", _total_timer.timer(LinearScanTimers::timer_name), TimeLinearScan || TimeEachLinearScan, Verbose); 48 49 #else 50 #define TIME_LINEAR_SCAN(timer_name) 51 #endif 52 53 #ifdef ASSERT 54 55 // helper macro for short definition of trace-output inside code 56 #define TRACE_LINEAR_SCAN(level, code) \ 57 if (TraceLinearScanLevel >= level) { \ 58 code; \ 59 } 60 #else 61 #define TRACE_LINEAR_SCAN(level, code) 62 #endif 63 64 // Map BasicType to spill size in 32-bit words, matching VMReg's notion of words 65 #ifdef _LP64 66 static int type2spill_size[T_CONFLICT+1]={ -1, 0, 0, 0, 1, 1, 1, 2, 1, 1, 1, 2, 2, 2, 2, 0, 2, 1, 2, 1, -1}; 67 #else 68 static int type2spill_size[T_CONFLICT+1]={ -1, 0, 0, 0, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 1, 0, 1, -1, 1, 1, -1}; 69 #endif 70 71 72 // Implementation of LinearScan 73 74 LinearScan::LinearScan(IR* ir, LIRGenerator* gen, FrameMap* frame_map) 75 : _compilation(ir->compilation()) 76 , _ir(ir) 77 , _gen(gen) 78 , _frame_map(frame_map) 79 , _cached_blocks(*ir->linear_scan_order()) 80 , _num_virtual_regs(gen->max_virtual_register_number()) 81 , _has_fpu_registers(false) 82 , _num_calls(-1) 83 , _max_spills(0) 84 , _unused_spill_slot(-1) 85 , _intervals(0) // initialized later with correct length 86 , _new_intervals_from_allocation(nullptr) 87 , _sorted_intervals(nullptr) 88 , _needs_full_resort(false) 89 , _lir_ops(0) // initialized later with correct length 90 , _block_of_op(0) // initialized later with correct length 91 , _has_info(0) 92 , _has_call(0) 93 , _interval_in_loop(0) // initialized later with correct length 94 , _scope_value_cache(0) // initialized later with correct length 95 #ifdef IA32 96 , _fpu_stack_allocator(nullptr) 97 #endif 98 { 99 assert(this->ir() != nullptr, "check if valid"); 100 assert(this->compilation() != nullptr, "check if valid"); 101 assert(this->gen() != nullptr, "check if valid"); 102 assert(this->frame_map() != nullptr, "check if valid"); 103 } 104 105 106 // ********** functions for converting LIR-Operands to register numbers 107 // 108 // Emulate a flat register file comprising physical integer registers, 109 // physical floating-point registers and virtual registers, in that order. 110 // Virtual registers already have appropriate numbers, since V0 is 111 // the number of physical registers. 112 // Returns -1 for hi word if opr is a single word operand. 113 // 114 // Note: the inverse operation (calculating an operand for register numbers) 115 // is done in calc_operand_for_interval() 116 117 int LinearScan::reg_num(LIR_Opr opr) { 118 assert(opr->is_register(), "should not call this otherwise"); 119 120 if (opr->is_virtual_register()) { 121 assert(opr->vreg_number() >= nof_regs, "found a virtual register with a fixed-register number"); 122 return opr->vreg_number(); 123 } else if (opr->is_single_cpu()) { 124 return opr->cpu_regnr(); 125 } else if (opr->is_double_cpu()) { 126 return opr->cpu_regnrLo(); 127 #ifdef X86 128 } else if (opr->is_single_xmm()) { 129 return opr->fpu_regnr() + pd_first_xmm_reg; 130 } else if (opr->is_double_xmm()) { 131 return opr->fpu_regnrLo() + pd_first_xmm_reg; 132 #endif 133 } else if (opr->is_single_fpu()) { 134 return opr->fpu_regnr() + pd_first_fpu_reg; 135 } else if (opr->is_double_fpu()) { 136 return opr->fpu_regnrLo() + pd_first_fpu_reg; 137 } else { 138 ShouldNotReachHere(); 139 return -1; 140 } 141 } 142 143 int LinearScan::reg_numHi(LIR_Opr opr) { 144 assert(opr->is_register(), "should not call this otherwise"); 145 146 if (opr->is_virtual_register()) { 147 return -1; 148 } else if (opr->is_single_cpu()) { 149 return -1; 150 } else if (opr->is_double_cpu()) { 151 return opr->cpu_regnrHi(); 152 #ifdef X86 153 } else if (opr->is_single_xmm()) { 154 return -1; 155 } else if (opr->is_double_xmm()) { 156 return -1; 157 #endif 158 } else if (opr->is_single_fpu()) { 159 return -1; 160 } else if (opr->is_double_fpu()) { 161 return opr->fpu_regnrHi() + pd_first_fpu_reg; 162 } else { 163 ShouldNotReachHere(); 164 return -1; 165 } 166 } 167 168 169 // ********** functions for classification of intervals 170 171 bool LinearScan::is_precolored_interval(const Interval* i) { 172 return i->reg_num() < LinearScan::nof_regs; 173 } 174 175 bool LinearScan::is_virtual_interval(const Interval* i) { 176 return i->reg_num() >= LIR_Opr::vreg_base; 177 } 178 179 bool LinearScan::is_precolored_cpu_interval(const Interval* i) { 180 return i->reg_num() < LinearScan::nof_cpu_regs; 181 } 182 183 bool LinearScan::is_virtual_cpu_interval(const Interval* i) { 184 #if defined(__SOFTFP__) || defined(E500V2) 185 return i->reg_num() >= LIR_Opr::vreg_base; 186 #else 187 return i->reg_num() >= LIR_Opr::vreg_base && (i->type() != T_FLOAT && i->type() != T_DOUBLE); 188 #endif // __SOFTFP__ or E500V2 189 } 190 191 bool LinearScan::is_precolored_fpu_interval(const Interval* i) { 192 return i->reg_num() >= LinearScan::nof_cpu_regs && i->reg_num() < LinearScan::nof_regs; 193 } 194 195 bool LinearScan::is_virtual_fpu_interval(const Interval* i) { 196 #if defined(__SOFTFP__) || defined(E500V2) 197 return false; 198 #else 199 return i->reg_num() >= LIR_Opr::vreg_base && (i->type() == T_FLOAT || i->type() == T_DOUBLE); 200 #endif // __SOFTFP__ or E500V2 201 } 202 203 bool LinearScan::is_in_fpu_register(const Interval* i) { 204 // fixed intervals not needed for FPU stack allocation 205 return i->reg_num() >= nof_regs && pd_first_fpu_reg <= i->assigned_reg() && i->assigned_reg() <= pd_last_fpu_reg; 206 } 207 208 bool LinearScan::is_oop_interval(const Interval* i) { 209 // fixed intervals never contain oops 210 return i->reg_num() >= nof_regs && i->type() == T_OBJECT; 211 } 212 213 214 // ********** General helper functions 215 216 // compute next unused stack index that can be used for spilling 217 int LinearScan::allocate_spill_slot(bool double_word) { 218 int spill_slot; 219 if (double_word) { 220 if ((_max_spills & 1) == 1) { 221 // alignment of double-word values 222 // the hole because of the alignment is filled with the next single-word value 223 assert(_unused_spill_slot == -1, "wasting a spill slot"); 224 _unused_spill_slot = _max_spills; 225 _max_spills++; 226 } 227 spill_slot = _max_spills; 228 _max_spills += 2; 229 230 } else if (_unused_spill_slot != -1) { 231 // re-use hole that was the result of a previous double-word alignment 232 spill_slot = _unused_spill_slot; 233 _unused_spill_slot = -1; 234 235 } else { 236 spill_slot = _max_spills; 237 _max_spills++; 238 } 239 240 int result = spill_slot + LinearScan::nof_regs + frame_map()->argcount(); 241 242 // if too many slots used, bailout compilation. 243 if (result > 2000) { 244 bailout("too many stack slots used"); 245 } 246 247 return result; 248 } 249 250 void LinearScan::assign_spill_slot(Interval* it) { 251 // assign the canonical spill slot of the parent (if a part of the interval 252 // is already spilled) or allocate a new spill slot 253 if (it->canonical_spill_slot() >= 0) { 254 it->assign_reg(it->canonical_spill_slot()); 255 } else { 256 int spill = allocate_spill_slot(type2spill_size[it->type()] == 2); 257 it->set_canonical_spill_slot(spill); 258 it->assign_reg(spill); 259 } 260 } 261 262 void LinearScan::propagate_spill_slots() { 263 if (!frame_map()->finalize_frame(max_spills(), compilation()->needs_stack_repair())) { 264 bailout("frame too large"); 265 } 266 } 267 268 // create a new interval with a predefined reg_num 269 // (only used for parent intervals that are created during the building phase) 270 Interval* LinearScan::create_interval(int reg_num) { 271 assert(_intervals.at(reg_num) == nullptr, "overwriting existing interval"); 272 273 Interval* interval = new Interval(reg_num); 274 _intervals.at_put(reg_num, interval); 275 276 // assign register number for precolored intervals 277 if (reg_num < LIR_Opr::vreg_base) { 278 interval->assign_reg(reg_num); 279 } 280 return interval; 281 } 282 283 // assign a new reg_num to the interval and append it to the list of intervals 284 // (only used for child intervals that are created during register allocation) 285 void LinearScan::append_interval(Interval* it) { 286 it->set_reg_num(_intervals.length()); 287 _intervals.append(it); 288 IntervalList* new_intervals = _new_intervals_from_allocation; 289 if (new_intervals == nullptr) { 290 new_intervals = _new_intervals_from_allocation = new IntervalList(); 291 } 292 new_intervals->append(it); 293 } 294 295 // copy the vreg-flags if an interval is split 296 void LinearScan::copy_register_flags(Interval* from, Interval* to) { 297 if (gen()->is_vreg_flag_set(from->reg_num(), LIRGenerator::byte_reg)) { 298 gen()->set_vreg_flag(to->reg_num(), LIRGenerator::byte_reg); 299 } 300 if (gen()->is_vreg_flag_set(from->reg_num(), LIRGenerator::callee_saved)) { 301 gen()->set_vreg_flag(to->reg_num(), LIRGenerator::callee_saved); 302 } 303 304 // Note: do not copy the must_start_in_memory flag because it is not necessary for child 305 // intervals (only the very beginning of the interval must be in memory) 306 } 307 308 309 // ********** spill move optimization 310 // eliminate moves from register to stack if stack slot is known to be correct 311 312 // called during building of intervals 313 void LinearScan::change_spill_definition_pos(Interval* interval, int def_pos) { 314 assert(interval->is_split_parent(), "can only be called for split parents"); 315 316 switch (interval->spill_state()) { 317 case noDefinitionFound: 318 assert(interval->spill_definition_pos() == -1, "must no be set before"); 319 interval->set_spill_definition_pos(def_pos); 320 interval->set_spill_state(oneDefinitionFound); 321 break; 322 323 case oneDefinitionFound: 324 assert(def_pos <= interval->spill_definition_pos(), "positions are processed in reverse order when intervals are created"); 325 if (def_pos < interval->spill_definition_pos() - 2) { 326 // second definition found, so no spill optimization possible for this interval 327 interval->set_spill_state(noOptimization); 328 } else { 329 // two consecutive definitions (because of two-operand LIR form) 330 assert(block_of_op_with_id(def_pos) == block_of_op_with_id(interval->spill_definition_pos()), "block must be equal"); 331 } 332 break; 333 334 case noOptimization: 335 // nothing to do 336 break; 337 338 default: 339 assert(false, "other states not allowed at this time"); 340 } 341 } 342 343 // called during register allocation 344 void LinearScan::change_spill_state(Interval* interval, int spill_pos) { 345 switch (interval->spill_state()) { 346 case oneDefinitionFound: { 347 int def_loop_depth = block_of_op_with_id(interval->spill_definition_pos())->loop_depth(); 348 int spill_loop_depth = block_of_op_with_id(spill_pos)->loop_depth(); 349 350 if (def_loop_depth < spill_loop_depth) { 351 // the loop depth of the spilling position is higher then the loop depth 352 // at the definition of the interval -> move write to memory out of loop 353 // by storing at definitin of the interval 354 interval->set_spill_state(storeAtDefinition); 355 } else { 356 // the interval is currently spilled only once, so for now there is no 357 // reason to store the interval at the definition 358 interval->set_spill_state(oneMoveInserted); 359 } 360 break; 361 } 362 363 case oneMoveInserted: { 364 // the interval is spilled more then once, so it is better to store it to 365 // memory at the definition 366 interval->set_spill_state(storeAtDefinition); 367 break; 368 } 369 370 case storeAtDefinition: 371 case startInMemory: 372 case noOptimization: 373 case noDefinitionFound: 374 // nothing to do 375 break; 376 377 default: 378 assert(false, "other states not allowed at this time"); 379 } 380 } 381 382 383 bool LinearScan::must_store_at_definition(const Interval* i) { 384 return i->is_split_parent() && i->spill_state() == storeAtDefinition; 385 } 386 387 // called once before assignment of register numbers 388 void LinearScan::eliminate_spill_moves() { 389 TIME_LINEAR_SCAN(timer_eliminate_spill_moves); 390 TRACE_LINEAR_SCAN(3, tty->print_cr("***** Eliminating unnecessary spill moves")); 391 392 // collect all intervals that must be stored after their definion. 393 // the list is sorted by Interval::spill_definition_pos 394 Interval* interval; 395 Interval* temp_list; 396 create_unhandled_lists(&interval, &temp_list, must_store_at_definition, nullptr); 397 398 #ifdef ASSERT 399 Interval* prev = nullptr; 400 Interval* temp = interval; 401 while (temp != Interval::end()) { 402 assert(temp->spill_definition_pos() > 0, "invalid spill definition pos"); 403 if (prev != nullptr) { 404 assert(temp->from() >= prev->from(), "intervals not sorted"); 405 assert(temp->spill_definition_pos() >= prev->spill_definition_pos(), "when intervals are sorted by from, then they must also be sorted by spill_definition_pos"); 406 } 407 408 assert(temp->canonical_spill_slot() >= LinearScan::nof_regs, "interval has no spill slot assigned"); 409 assert(temp->spill_definition_pos() >= temp->from(), "invalid order"); 410 assert(temp->spill_definition_pos() <= temp->from() + 2, "only intervals defined once at their start-pos can be optimized"); 411 412 TRACE_LINEAR_SCAN(4, tty->print_cr("interval %d (from %d to %d) must be stored at %d", temp->reg_num(), temp->from(), temp->to(), temp->spill_definition_pos())); 413 414 temp = temp->next(); 415 } 416 #endif 417 418 LIR_InsertionBuffer insertion_buffer; 419 int num_blocks = block_count(); 420 for (int i = 0; i < num_blocks; i++) { 421 BlockBegin* block = block_at(i); 422 LIR_OpList* instructions = block->lir()->instructions_list(); 423 int num_inst = instructions->length(); 424 bool has_new = false; 425 426 // iterate all instructions of the block. skip the first because it is always a label 427 for (int j = 1; j < num_inst; j++) { 428 LIR_Op* op = instructions->at(j); 429 int op_id = op->id(); 430 431 if (op_id == -1) { 432 // remove move from register to stack if the stack slot is guaranteed to be correct. 433 // only moves that have been inserted by LinearScan can be removed. 434 assert(op->code() == lir_move, "only moves can have a op_id of -1"); 435 assert(op->as_Op1() != nullptr, "move must be LIR_Op1"); 436 assert(op->as_Op1()->result_opr()->is_virtual(), "LinearScan inserts only moves to virtual registers"); 437 438 LIR_Op1* op1 = (LIR_Op1*)op; 439 Interval* interval = interval_at(op1->result_opr()->vreg_number()); 440 441 if (interval->assigned_reg() >= LinearScan::nof_regs && interval->always_in_memory()) { 442 // move target is a stack slot that is always correct, so eliminate instruction 443 TRACE_LINEAR_SCAN(4, tty->print_cr("eliminating move from interval %d to %d", op1->in_opr()->vreg_number(), op1->result_opr()->vreg_number())); 444 instructions->at_put(j, nullptr); // null-instructions are deleted by assign_reg_num 445 } 446 447 } else { 448 // insert move from register to stack just after the beginning of the interval 449 assert(interval == Interval::end() || interval->spill_definition_pos() >= op_id, "invalid order"); 450 assert(interval == Interval::end() || (interval->is_split_parent() && interval->spill_state() == storeAtDefinition), "invalid interval"); 451 452 while (interval != Interval::end() && interval->spill_definition_pos() == op_id) { 453 if (!has_new) { 454 // prepare insertion buffer (appended when all instructions of the block are processed) 455 insertion_buffer.init(block->lir()); 456 has_new = true; 457 } 458 459 LIR_Opr from_opr = operand_for_interval(interval); 460 LIR_Opr to_opr = canonical_spill_opr(interval); 461 assert(from_opr->is_fixed_cpu() || from_opr->is_fixed_fpu(), "from operand must be a register"); 462 assert(to_opr->is_stack(), "to operand must be a stack slot"); 463 464 insertion_buffer.move(j, from_opr, to_opr); 465 TRACE_LINEAR_SCAN(4, tty->print_cr("inserting move after definition of interval %d to stack slot %d at op_id %d", interval->reg_num(), interval->canonical_spill_slot() - LinearScan::nof_regs, op_id)); 466 467 interval = interval->next(); 468 } 469 } 470 } // end of instruction iteration 471 472 if (has_new) { 473 block->lir()->append(&insertion_buffer); 474 } 475 } // end of block iteration 476 477 assert(interval == Interval::end(), "missed an interval"); 478 } 479 480 481 // ********** Phase 1: number all instructions in all blocks 482 // Compute depth-first and linear scan block orders, and number LIR_Op nodes for linear scan. 483 484 void LinearScan::number_instructions() { 485 { 486 // dummy-timer to measure the cost of the timer itself 487 // (this time is then subtracted from all other timers to get the real value) 488 TIME_LINEAR_SCAN(timer_do_nothing); 489 } 490 TIME_LINEAR_SCAN(timer_number_instructions); 491 492 // Assign IDs to LIR nodes and build a mapping, lir_ops, from ID to LIR_Op node. 493 int num_blocks = block_count(); 494 int num_instructions = 0; 495 int i; 496 for (i = 0; i < num_blocks; i++) { 497 num_instructions += block_at(i)->lir()->instructions_list()->length(); 498 } 499 500 // initialize with correct length 501 _lir_ops = LIR_OpArray(num_instructions, num_instructions, nullptr); 502 _block_of_op = BlockBeginArray(num_instructions, num_instructions, nullptr); 503 504 int op_id = 0; 505 int idx = 0; 506 507 for (i = 0; i < num_blocks; i++) { 508 BlockBegin* block = block_at(i); 509 block->set_first_lir_instruction_id(op_id); 510 LIR_OpList* instructions = block->lir()->instructions_list(); 511 512 int num_inst = instructions->length(); 513 for (int j = 0; j < num_inst; j++) { 514 LIR_Op* op = instructions->at(j); 515 op->set_id(op_id); 516 517 _lir_ops.at_put(idx, op); 518 _block_of_op.at_put(idx, block); 519 assert(lir_op_with_id(op_id) == op, "must match"); 520 521 idx++; 522 op_id += 2; // numbering of lir_ops by two 523 } 524 block->set_last_lir_instruction_id(op_id - 2); 525 } 526 assert(idx == num_instructions, "must match"); 527 assert(idx * 2 == op_id, "must match"); 528 529 _has_call.initialize(num_instructions); 530 _has_info.initialize(num_instructions); 531 } 532 533 534 // ********** Phase 2: compute local live sets separately for each block 535 // (sets live_gen and live_kill for each block) 536 537 void LinearScan::set_live_gen_kill(Value value, LIR_Op* op, BitMap& live_gen, BitMap& live_kill) { 538 LIR_Opr opr = value->operand(); 539 Constant* con = value->as_Constant(); 540 541 // check some asumptions about debug information 542 assert(!value->type()->is_illegal(), "if this local is used by the interpreter it shouldn't be of indeterminate type"); 543 assert(con == nullptr || opr->is_virtual() || opr->is_constant() || opr->is_illegal(), "assumption: Constant instructions have only constant operands"); 544 assert(con != nullptr || opr->is_virtual(), "assumption: non-Constant instructions have only virtual operands"); 545 546 if ((con == nullptr || con->is_pinned()) && opr->is_register()) { 547 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); 548 int reg = opr->vreg_number(); 549 if (!live_kill.at(reg)) { 550 live_gen.set_bit(reg); 551 TRACE_LINEAR_SCAN(4, tty->print_cr(" Setting live_gen for value %c%d, LIR op_id %d, register number %d", value->type()->tchar(), value->id(), op->id(), reg)); 552 } 553 } 554 } 555 556 557 void LinearScan::compute_local_live_sets() { 558 TIME_LINEAR_SCAN(timer_compute_local_live_sets); 559 560 int num_blocks = block_count(); 561 int live_size = live_set_size(); 562 bool local_has_fpu_registers = false; 563 int local_num_calls = 0; 564 LIR_OpVisitState visitor; 565 566 BitMap2D local_interval_in_loop = BitMap2D(_num_virtual_regs, num_loops()); 567 568 // iterate all blocks 569 for (int i = 0; i < num_blocks; i++) { 570 BlockBegin* block = block_at(i); 571 572 ResourceBitMap live_gen(live_size); 573 ResourceBitMap live_kill(live_size); 574 575 if (block->is_set(BlockBegin::exception_entry_flag)) { 576 // Phi functions at the begin of an exception handler are 577 // implicitly defined (= killed) at the beginning of the block. 578 for_each_phi_fun(block, phi, 579 if (!phi->is_illegal()) { live_kill.set_bit(phi->operand()->vreg_number()); } 580 ); 581 } 582 583 LIR_OpList* instructions = block->lir()->instructions_list(); 584 int num_inst = instructions->length(); 585 586 // iterate all instructions of the block. skip the first because it is always a label 587 assert(visitor.no_operands(instructions->at(0)), "first operation must always be a label"); 588 for (int j = 1; j < num_inst; j++) { 589 LIR_Op* op = instructions->at(j); 590 591 // visit operation to collect all operands 592 visitor.visit(op); 593 594 if (visitor.has_call()) { 595 _has_call.set_bit(op->id() >> 1); 596 local_num_calls++; 597 } 598 if (visitor.info_count() > 0) { 599 _has_info.set_bit(op->id() >> 1); 600 } 601 602 // iterate input operands of instruction 603 int k, n, reg; 604 n = visitor.opr_count(LIR_OpVisitState::inputMode); 605 for (k = 0; k < n; k++) { 606 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::inputMode, k); 607 assert(opr->is_register(), "visitor should only return register operands"); 608 609 if (opr->is_virtual_register()) { 610 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); 611 reg = opr->vreg_number(); 612 if (!live_kill.at(reg)) { 613 live_gen.set_bit(reg); 614 TRACE_LINEAR_SCAN(4, tty->print_cr(" Setting live_gen for register %d at instruction %d", reg, op->id())); 615 } 616 if (block->loop_index() >= 0) { 617 local_interval_in_loop.set_bit(reg, block->loop_index()); 618 } 619 local_has_fpu_registers = local_has_fpu_registers || opr->is_virtual_fpu(); 620 } 621 622 #ifdef ASSERT 623 // fixed intervals are never live at block boundaries, so 624 // they need not be processed in live sets. 625 // this is checked by these assertions to be sure about it. 626 // the entry block may have incoming values in registers, which is ok. 627 if (!opr->is_virtual_register() && block != ir()->start()) { 628 reg = reg_num(opr); 629 if (is_processed_reg_num(reg)) { 630 assert(live_kill.at(reg), "using fixed register that is not defined in this block"); 631 } 632 reg = reg_numHi(opr); 633 if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) { 634 assert(live_kill.at(reg), "using fixed register that is not defined in this block"); 635 } 636 } 637 #endif 638 } 639 640 // Add uses of live locals from interpreter's point of view for proper debug information generation 641 n = visitor.info_count(); 642 for (k = 0; k < n; k++) { 643 CodeEmitInfo* info = visitor.info_at(k); 644 ValueStack* stack = info->stack(); 645 for_each_state_value(stack, value, 646 set_live_gen_kill(value, op, live_gen, live_kill); 647 local_has_fpu_registers = local_has_fpu_registers || value->type()->is_float_kind(); 648 ); 649 } 650 651 // iterate temp operands of instruction 652 n = visitor.opr_count(LIR_OpVisitState::tempMode); 653 for (k = 0; k < n; k++) { 654 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::tempMode, k); 655 assert(opr->is_register(), "visitor should only return register operands"); 656 657 if (opr->is_virtual_register()) { 658 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); 659 reg = opr->vreg_number(); 660 live_kill.set_bit(reg); 661 if (block->loop_index() >= 0) { 662 local_interval_in_loop.set_bit(reg, block->loop_index()); 663 } 664 local_has_fpu_registers = local_has_fpu_registers || opr->is_virtual_fpu(); 665 } 666 667 #ifdef ASSERT 668 // fixed intervals are never live at block boundaries, so 669 // they need not be processed in live sets 670 // process them only in debug mode so that this can be checked 671 if (!opr->is_virtual_register()) { 672 reg = reg_num(opr); 673 if (is_processed_reg_num(reg)) { 674 live_kill.set_bit(reg_num(opr)); 675 } 676 reg = reg_numHi(opr); 677 if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) { 678 live_kill.set_bit(reg); 679 } 680 } 681 #endif 682 } 683 684 // iterate output operands of instruction 685 n = visitor.opr_count(LIR_OpVisitState::outputMode); 686 for (k = 0; k < n; k++) { 687 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::outputMode, k); 688 assert(opr->is_register(), "visitor should only return register operands"); 689 690 if (opr->is_virtual_register()) { 691 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); 692 reg = opr->vreg_number(); 693 live_kill.set_bit(reg); 694 if (block->loop_index() >= 0) { 695 local_interval_in_loop.set_bit(reg, block->loop_index()); 696 } 697 local_has_fpu_registers = local_has_fpu_registers || opr->is_virtual_fpu(); 698 } 699 700 #ifdef ASSERT 701 // fixed intervals are never live at block boundaries, so 702 // they need not be processed in live sets 703 // process them only in debug mode so that this can be checked 704 if (!opr->is_virtual_register()) { 705 reg = reg_num(opr); 706 if (is_processed_reg_num(reg)) { 707 live_kill.set_bit(reg_num(opr)); 708 } 709 reg = reg_numHi(opr); 710 if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) { 711 live_kill.set_bit(reg); 712 } 713 } 714 #endif 715 } 716 } // end of instruction iteration 717 718 block->set_live_gen (live_gen); 719 block->set_live_kill(live_kill); 720 block->set_live_in (ResourceBitMap(live_size)); 721 block->set_live_out (ResourceBitMap(live_size)); 722 723 TRACE_LINEAR_SCAN(4, tty->print("live_gen B%d ", block->block_id()); print_bitmap(block->live_gen())); 724 TRACE_LINEAR_SCAN(4, tty->print("live_kill B%d ", block->block_id()); print_bitmap(block->live_kill())); 725 } // end of block iteration 726 727 // propagate local calculated information into LinearScan object 728 _has_fpu_registers = local_has_fpu_registers; 729 compilation()->set_has_fpu_code(local_has_fpu_registers); 730 731 _num_calls = local_num_calls; 732 _interval_in_loop = local_interval_in_loop; 733 } 734 735 736 // ********** Phase 3: perform a backward dataflow analysis to compute global live sets 737 // (sets live_in and live_out for each block) 738 739 void LinearScan::compute_global_live_sets() { 740 TIME_LINEAR_SCAN(timer_compute_global_live_sets); 741 742 int num_blocks = block_count(); 743 bool change_occurred; 744 bool change_occurred_in_block; 745 int iteration_count = 0; 746 ResourceBitMap live_out(live_set_size()); // scratch set for calculations 747 748 // Perform a backward dataflow analysis to compute live_out and live_in for each block. 749 // The loop is executed until a fixpoint is reached (no changes in an iteration) 750 // Exception handlers must be processed because not all live values are 751 // present in the state array, e.g. because of global value numbering 752 do { 753 change_occurred = false; 754 755 // iterate all blocks in reverse order 756 for (int i = num_blocks - 1; i >= 0; i--) { 757 BlockBegin* block = block_at(i); 758 759 change_occurred_in_block = false; 760 761 // live_out(block) is the union of live_in(sux), for successors sux of block 762 int n = block->number_of_sux(); 763 int e = block->number_of_exception_handlers(); 764 if (n + e > 0) { 765 // block has successors 766 if (n > 0) { 767 live_out.set_from(block->sux_at(0)->live_in()); 768 for (int j = 1; j < n; j++) { 769 live_out.set_union(block->sux_at(j)->live_in()); 770 } 771 } else { 772 live_out.clear(); 773 } 774 for (int j = 0; j < e; j++) { 775 live_out.set_union(block->exception_handler_at(j)->live_in()); 776 } 777 778 if (!block->live_out().is_same(live_out)) { 779 // A change occurred. Swap the old and new live out sets to avoid copying. 780 ResourceBitMap temp = block->live_out(); 781 block->set_live_out(live_out); 782 live_out = temp; 783 784 change_occurred = true; 785 change_occurred_in_block = true; 786 } 787 } 788 789 if (iteration_count == 0 || change_occurred_in_block) { 790 // live_in(block) is the union of live_gen(block) with (live_out(block) & !live_kill(block)) 791 // note: live_in has to be computed only in first iteration or if live_out has changed! 792 ResourceBitMap live_in = block->live_in(); 793 live_in.set_from(block->live_out()); 794 live_in.set_difference(block->live_kill()); 795 live_in.set_union(block->live_gen()); 796 } 797 798 #ifdef ASSERT 799 if (TraceLinearScanLevel >= 4) { 800 char c = ' '; 801 if (iteration_count == 0 || change_occurred_in_block) { 802 c = '*'; 803 } 804 tty->print("(%d) live_in%c B%d ", iteration_count, c, block->block_id()); print_bitmap(block->live_in()); 805 tty->print("(%d) live_out%c B%d ", iteration_count, c, block->block_id()); print_bitmap(block->live_out()); 806 } 807 #endif 808 } 809 iteration_count++; 810 811 if (change_occurred && iteration_count > 50) { 812 BAILOUT("too many iterations in compute_global_live_sets"); 813 } 814 } while (change_occurred); 815 816 817 #ifdef ASSERT 818 // check that fixed intervals are not live at block boundaries 819 // (live set must be empty at fixed intervals) 820 for (int i = 0; i < num_blocks; i++) { 821 BlockBegin* block = block_at(i); 822 for (int j = 0; j < LIR_Opr::vreg_base; j++) { 823 assert(block->live_in().at(j) == false, "live_in set of fixed register must be empty"); 824 assert(block->live_out().at(j) == false, "live_out set of fixed register must be empty"); 825 assert(block->live_gen().at(j) == false, "live_gen set of fixed register must be empty"); 826 } 827 } 828 #endif 829 830 // check that the live_in set of the first block is empty 831 ResourceBitMap live_in_args(ir()->start()->live_in().size()); 832 if (!ir()->start()->live_in().is_same(live_in_args)) { 833 #ifdef ASSERT 834 tty->print_cr("Error: live_in set of first block must be empty (when this fails, virtual registers are used before they are defined)"); 835 tty->print_cr("affected registers:"); 836 print_bitmap(ir()->start()->live_in()); 837 838 // print some additional information to simplify debugging 839 for (unsigned int i = 0; i < ir()->start()->live_in().size(); i++) { 840 if (ir()->start()->live_in().at(i)) { 841 Instruction* instr = gen()->instruction_for_vreg(i); 842 tty->print_cr("* vreg %d (HIR instruction %c%d)", i, instr == nullptr ? ' ' : instr->type()->tchar(), instr == nullptr ? 0 : instr->id()); 843 844 for (int j = 0; j < num_blocks; j++) { 845 BlockBegin* block = block_at(j); 846 if (block->live_gen().at(i)) { 847 tty->print_cr(" used in block B%d", block->block_id()); 848 } 849 if (block->live_kill().at(i)) { 850 tty->print_cr(" defined in block B%d", block->block_id()); 851 } 852 } 853 } 854 } 855 856 #endif 857 // when this fails, virtual registers are used before they are defined. 858 assert(false, "live_in set of first block must be empty"); 859 // bailout of if this occurs in product mode. 860 bailout("live_in set of first block not empty"); 861 } 862 } 863 864 865 // ********** Phase 4: build intervals 866 // (fills the list _intervals) 867 868 void LinearScan::add_use(Value value, int from, int to, IntervalUseKind use_kind) { 869 assert(!value->type()->is_illegal(), "if this value is used by the interpreter it shouldn't be of indeterminate type"); 870 LIR_Opr opr = value->operand(); 871 Constant* con = value->as_Constant(); 872 873 if ((con == nullptr || con->is_pinned()) && opr->is_register()) { 874 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); 875 add_use(opr, from, to, use_kind); 876 } 877 } 878 879 880 void LinearScan::add_def(LIR_Opr opr, int def_pos, IntervalUseKind use_kind) { 881 TRACE_LINEAR_SCAN(2, tty->print(" def "); opr->print(tty); tty->print_cr(" def_pos %d (%d)", def_pos, use_kind)); 882 assert(opr->is_register(), "should not be called otherwise"); 883 884 if (opr->is_virtual_register()) { 885 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); 886 add_def(opr->vreg_number(), def_pos, use_kind, opr->type_register()); 887 888 } else { 889 int reg = reg_num(opr); 890 if (is_processed_reg_num(reg)) { 891 add_def(reg, def_pos, use_kind, opr->type_register()); 892 } 893 reg = reg_numHi(opr); 894 if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) { 895 add_def(reg, def_pos, use_kind, opr->type_register()); 896 } 897 } 898 } 899 900 void LinearScan::add_use(LIR_Opr opr, int from, int to, IntervalUseKind use_kind) { 901 TRACE_LINEAR_SCAN(2, tty->print(" use "); opr->print(tty); tty->print_cr(" from %d to %d (%d)", from, to, use_kind)); 902 assert(opr->is_register(), "should not be called otherwise"); 903 904 if (opr->is_virtual_register()) { 905 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); 906 add_use(opr->vreg_number(), from, to, use_kind, opr->type_register()); 907 908 } else { 909 int reg = reg_num(opr); 910 if (is_processed_reg_num(reg)) { 911 add_use(reg, from, to, use_kind, opr->type_register()); 912 } 913 reg = reg_numHi(opr); 914 if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) { 915 add_use(reg, from, to, use_kind, opr->type_register()); 916 } 917 } 918 } 919 920 void LinearScan::add_temp(LIR_Opr opr, int temp_pos, IntervalUseKind use_kind) { 921 TRACE_LINEAR_SCAN(2, tty->print(" temp "); opr->print(tty); tty->print_cr(" temp_pos %d (%d)", temp_pos, use_kind)); 922 assert(opr->is_register(), "should not be called otherwise"); 923 924 if (opr->is_virtual_register()) { 925 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); 926 add_temp(opr->vreg_number(), temp_pos, use_kind, opr->type_register()); 927 928 } else { 929 int reg = reg_num(opr); 930 if (is_processed_reg_num(reg)) { 931 add_temp(reg, temp_pos, use_kind, opr->type_register()); 932 } 933 reg = reg_numHi(opr); 934 if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) { 935 add_temp(reg, temp_pos, use_kind, opr->type_register()); 936 } 937 } 938 } 939 940 941 void LinearScan::add_def(int reg_num, int def_pos, IntervalUseKind use_kind, BasicType type) { 942 Interval* interval = interval_at(reg_num); 943 if (interval != nullptr) { 944 assert(interval->reg_num() == reg_num, "wrong interval"); 945 946 if (type != T_ILLEGAL) { 947 interval->set_type(type); 948 } 949 950 Range* r = interval->first(); 951 if (r->from() <= def_pos) { 952 // Update the starting point (when a range is first created for a use, its 953 // start is the beginning of the current block until a def is encountered.) 954 r->set_from(def_pos); 955 interval->add_use_pos(def_pos, use_kind); 956 957 } else { 958 // Dead value - make vacuous interval 959 // also add use_kind for dead intervals 960 interval->add_range(def_pos, def_pos + 1); 961 interval->add_use_pos(def_pos, use_kind); 962 TRACE_LINEAR_SCAN(2, tty->print_cr("Warning: def of reg %d at %d occurs without use", reg_num, def_pos)); 963 } 964 965 } else { 966 // Dead value - make vacuous interval 967 // also add use_kind for dead intervals 968 interval = create_interval(reg_num); 969 if (type != T_ILLEGAL) { 970 interval->set_type(type); 971 } 972 973 interval->add_range(def_pos, def_pos + 1); 974 interval->add_use_pos(def_pos, use_kind); 975 TRACE_LINEAR_SCAN(2, tty->print_cr("Warning: dead value %d at %d in live intervals", reg_num, def_pos)); 976 } 977 978 change_spill_definition_pos(interval, def_pos); 979 if (use_kind == noUse && interval->spill_state() <= startInMemory) { 980 // detection of method-parameters and roundfp-results 981 // TODO: move this directly to position where use-kind is computed 982 interval->set_spill_state(startInMemory); 983 } 984 } 985 986 void LinearScan::add_use(int reg_num, int from, int to, IntervalUseKind use_kind, BasicType type) { 987 Interval* interval = interval_at(reg_num); 988 if (interval == nullptr) { 989 interval = create_interval(reg_num); 990 } 991 assert(interval->reg_num() == reg_num, "wrong interval"); 992 993 if (type != T_ILLEGAL) { 994 interval->set_type(type); 995 } 996 997 interval->add_range(from, to); 998 interval->add_use_pos(to, use_kind); 999 } 1000 1001 void LinearScan::add_temp(int reg_num, int temp_pos, IntervalUseKind use_kind, BasicType type) { 1002 Interval* interval = interval_at(reg_num); 1003 if (interval == nullptr) { 1004 interval = create_interval(reg_num); 1005 } 1006 assert(interval->reg_num() == reg_num, "wrong interval"); 1007 1008 if (type != T_ILLEGAL) { 1009 interval->set_type(type); 1010 } 1011 1012 interval->add_range(temp_pos, temp_pos + 1); 1013 interval->add_use_pos(temp_pos, use_kind); 1014 } 1015 1016 1017 // the results of this functions are used for optimizing spilling and reloading 1018 // if the functions return shouldHaveRegister and the interval is spilled, 1019 // it is not reloaded to a register. 1020 IntervalUseKind LinearScan::use_kind_of_output_operand(LIR_Op* op, LIR_Opr opr) { 1021 if (op->code() == lir_move) { 1022 assert(op->as_Op1() != nullptr, "lir_move must be LIR_Op1"); 1023 LIR_Op1* move = (LIR_Op1*)op; 1024 LIR_Opr res = move->result_opr(); 1025 bool result_in_memory = res->is_virtual() && gen()->is_vreg_flag_set(res->vreg_number(), LIRGenerator::must_start_in_memory); 1026 1027 if (result_in_memory) { 1028 // Begin of an interval with must_start_in_memory set. 1029 // This interval will always get a stack slot first, so return noUse. 1030 return noUse; 1031 1032 } else if (move->in_opr()->is_stack()) { 1033 // method argument (condition must be equal to handle_method_arguments) 1034 return noUse; 1035 1036 } else if (move->in_opr()->is_register() && move->result_opr()->is_register()) { 1037 // Move from register to register 1038 if (block_of_op_with_id(op->id())->is_set(BlockBegin::osr_entry_flag)) { 1039 // special handling of phi-function moves inside osr-entry blocks 1040 // input operand must have a register instead of output operand (leads to better register allocation) 1041 return shouldHaveRegister; 1042 } 1043 } 1044 } 1045 1046 if (opr->is_virtual() && 1047 gen()->is_vreg_flag_set(opr->vreg_number(), LIRGenerator::must_start_in_memory)) { 1048 // result is a stack-slot, so prevent immediate reloading 1049 return noUse; 1050 } 1051 1052 // all other operands require a register 1053 return mustHaveRegister; 1054 } 1055 1056 IntervalUseKind LinearScan::use_kind_of_input_operand(LIR_Op* op, LIR_Opr opr) { 1057 if (op->code() == lir_move) { 1058 assert(op->as_Op1() != nullptr, "lir_move must be LIR_Op1"); 1059 LIR_Op1* move = (LIR_Op1*)op; 1060 LIR_Opr res = move->result_opr(); 1061 bool result_in_memory = res->is_virtual() && gen()->is_vreg_flag_set(res->vreg_number(), LIRGenerator::must_start_in_memory); 1062 1063 if (result_in_memory) { 1064 // Move to an interval with must_start_in_memory set. 1065 // To avoid moves from stack to stack (not allowed) force the input operand to a register 1066 return mustHaveRegister; 1067 1068 } else if (move->in_opr()->is_register() && move->result_opr()->is_register()) { 1069 // Move from register to register 1070 if (block_of_op_with_id(op->id())->is_set(BlockBegin::osr_entry_flag)) { 1071 // special handling of phi-function moves inside osr-entry blocks 1072 // input operand must have a register instead of output operand (leads to better register allocation) 1073 return mustHaveRegister; 1074 } 1075 1076 // The input operand is not forced to a register (moves from stack to register are allowed), 1077 // but it is faster if the input operand is in a register 1078 return shouldHaveRegister; 1079 } 1080 } 1081 1082 1083 #if defined(X86) || defined(S390) 1084 if (op->code() == lir_cmove) { 1085 // conditional moves can handle stack operands 1086 assert(op->result_opr()->is_register(), "result must always be in a register"); 1087 return shouldHaveRegister; 1088 } 1089 1090 // optimizations for second input operand of arithmehtic operations on Intel 1091 // this operand is allowed to be on the stack in some cases 1092 BasicType opr_type = opr->type_register(); 1093 if (opr_type == T_FLOAT || opr_type == T_DOUBLE) { 1094 if (IA32_ONLY( (UseSSE == 1 && opr_type == T_FLOAT) || UseSSE >= 2 ) NOT_IA32( true )) { 1095 // SSE float instruction (T_DOUBLE only supported with SSE2) 1096 switch (op->code()) { 1097 case lir_cmp: 1098 case lir_add: 1099 case lir_sub: 1100 case lir_mul: 1101 case lir_div: 1102 { 1103 assert(op->as_Op2() != nullptr, "must be LIR_Op2"); 1104 LIR_Op2* op2 = (LIR_Op2*)op; 1105 if (op2->in_opr1() != op2->in_opr2() && op2->in_opr2() == opr) { 1106 assert((op2->result_opr()->is_register() || op->code() == lir_cmp) && op2->in_opr1()->is_register(), "cannot mark second operand as stack if others are not in register"); 1107 return shouldHaveRegister; 1108 } 1109 } 1110 default: 1111 break; 1112 } 1113 } else { 1114 // FPU stack float instruction 1115 switch (op->code()) { 1116 case lir_add: 1117 case lir_sub: 1118 case lir_mul: 1119 case lir_div: 1120 { 1121 assert(op->as_Op2() != nullptr, "must be LIR_Op2"); 1122 LIR_Op2* op2 = (LIR_Op2*)op; 1123 if (op2->in_opr1() != op2->in_opr2() && op2->in_opr2() == opr) { 1124 assert((op2->result_opr()->is_register() || op->code() == lir_cmp) && op2->in_opr1()->is_register(), "cannot mark second operand as stack if others are not in register"); 1125 return shouldHaveRegister; 1126 } 1127 } 1128 default: 1129 break; 1130 } 1131 } 1132 // We want to sometimes use logical operations on pointers, in particular in GC barriers. 1133 // Since 64bit logical operations do not current support operands on stack, we have to make sure 1134 // T_OBJECT doesn't get spilled along with T_LONG. 1135 } else if (opr_type != T_LONG LP64_ONLY(&& opr_type != T_OBJECT)) { 1136 // integer instruction (note: long operands must always be in register) 1137 switch (op->code()) { 1138 case lir_cmp: 1139 case lir_add: 1140 case lir_sub: 1141 case lir_logic_and: 1142 case lir_logic_or: 1143 case lir_logic_xor: 1144 { 1145 assert(op->as_Op2() != nullptr, "must be LIR_Op2"); 1146 LIR_Op2* op2 = (LIR_Op2*)op; 1147 if (op2->in_opr1() != op2->in_opr2() && op2->in_opr2() == opr) { 1148 assert((op2->result_opr()->is_register() || op->code() == lir_cmp) && op2->in_opr1()->is_register(), "cannot mark second operand as stack if others are not in register"); 1149 return shouldHaveRegister; 1150 } 1151 } 1152 default: 1153 break; 1154 } 1155 } 1156 #endif // X86 || S390 1157 1158 // all other operands require a register 1159 return mustHaveRegister; 1160 } 1161 1162 1163 void LinearScan::handle_method_arguments(LIR_Op* op) { 1164 // special handling for method arguments (moves from stack to virtual register): 1165 // the interval gets no register assigned, but the stack slot. 1166 // it is split before the first use by the register allocator. 1167 1168 if (op->code() == lir_move) { 1169 assert(op->as_Op1() != nullptr, "must be LIR_Op1"); 1170 LIR_Op1* move = (LIR_Op1*)op; 1171 1172 if (move->in_opr()->is_stack()) { 1173 #ifdef ASSERT 1174 int arg_size = compilation()->method()->arg_size(); 1175 LIR_Opr o = move->in_opr(); 1176 if (o->is_single_stack()) { 1177 assert(o->single_stack_ix() >= 0 && o->single_stack_ix() < arg_size, "out of range"); 1178 } else if (o->is_double_stack()) { 1179 assert(o->double_stack_ix() >= 0 && o->double_stack_ix() < arg_size, "out of range"); 1180 } else { 1181 ShouldNotReachHere(); 1182 } 1183 1184 assert(move->id() > 0, "invalid id"); 1185 assert(block_of_op_with_id(move->id())->number_of_preds() == 0, "move from stack must be in first block"); 1186 assert(move->result_opr()->is_virtual(), "result of move must be a virtual register"); 1187 1188 TRACE_LINEAR_SCAN(4, tty->print_cr("found move from stack slot %d to vreg %d", o->is_single_stack() ? o->single_stack_ix() : o->double_stack_ix(), reg_num(move->result_opr()))); 1189 #endif 1190 1191 Interval* interval = interval_at(reg_num(move->result_opr())); 1192 1193 int stack_slot = LinearScan::nof_regs + (move->in_opr()->is_single_stack() ? move->in_opr()->single_stack_ix() : move->in_opr()->double_stack_ix()); 1194 interval->set_canonical_spill_slot(stack_slot); 1195 interval->assign_reg(stack_slot); 1196 } 1197 } 1198 } 1199 1200 void LinearScan::handle_doubleword_moves(LIR_Op* op) { 1201 // special handling for doubleword move from memory to register: 1202 // in this case the registers of the input address and the result 1203 // registers must not overlap -> add a temp range for the input registers 1204 if (op->code() == lir_move) { 1205 assert(op->as_Op1() != nullptr, "must be LIR_Op1"); 1206 LIR_Op1* move = (LIR_Op1*)op; 1207 1208 if (move->result_opr()->is_double_cpu() && move->in_opr()->is_pointer()) { 1209 LIR_Address* address = move->in_opr()->as_address_ptr(); 1210 if (address != nullptr) { 1211 if (address->base()->is_valid()) { 1212 add_temp(address->base(), op->id(), noUse); 1213 } 1214 if (address->index()->is_valid()) { 1215 add_temp(address->index(), op->id(), noUse); 1216 } 1217 } 1218 } 1219 } 1220 } 1221 1222 void LinearScan::add_register_hints(LIR_Op* op) { 1223 switch (op->code()) { 1224 case lir_move: // fall through 1225 case lir_convert: { 1226 assert(op->as_Op1() != nullptr, "lir_move, lir_convert must be LIR_Op1"); 1227 LIR_Op1* move = (LIR_Op1*)op; 1228 1229 LIR_Opr move_from = move->in_opr(); 1230 LIR_Opr move_to = move->result_opr(); 1231 1232 if (move_to->is_register() && move_from->is_register()) { 1233 Interval* from = interval_at(reg_num(move_from)); 1234 Interval* to = interval_at(reg_num(move_to)); 1235 if (from != nullptr && to != nullptr) { 1236 to->set_register_hint(from); 1237 TRACE_LINEAR_SCAN(4, tty->print_cr("operation at op_id %d: added hint from interval %d to %d", move->id(), from->reg_num(), to->reg_num())); 1238 } 1239 } 1240 break; 1241 } 1242 case lir_cmove: { 1243 assert(op->as_Op4() != nullptr, "lir_cmove must be LIR_Op4"); 1244 LIR_Op4* cmove = (LIR_Op4*)op; 1245 1246 LIR_Opr move_from = cmove->in_opr1(); 1247 LIR_Opr move_to = cmove->result_opr(); 1248 1249 if (move_to->is_register() && move_from->is_register()) { 1250 Interval* from = interval_at(reg_num(move_from)); 1251 Interval* to = interval_at(reg_num(move_to)); 1252 if (from != nullptr && to != nullptr) { 1253 to->set_register_hint(from); 1254 TRACE_LINEAR_SCAN(4, tty->print_cr("operation at op_id %d: added hint from interval %d to %d", cmove->id(), from->reg_num(), to->reg_num())); 1255 } 1256 } 1257 break; 1258 } 1259 default: 1260 break; 1261 } 1262 } 1263 1264 1265 void LinearScan::build_intervals() { 1266 TIME_LINEAR_SCAN(timer_build_intervals); 1267 1268 // initialize interval list with expected number of intervals 1269 // (32 is added to have some space for split children without having to resize the list) 1270 _intervals = IntervalList(num_virtual_regs() + 32); 1271 // initialize all slots that are used by build_intervals 1272 _intervals.at_put_grow(num_virtual_regs() - 1, nullptr, nullptr); 1273 1274 // create a list with all caller-save registers (cpu, fpu, xmm) 1275 // when an instruction is a call, a temp range is created for all these registers 1276 int num_caller_save_registers = 0; 1277 int caller_save_registers[LinearScan::nof_regs]; 1278 1279 int i; 1280 for (i = 0; i < FrameMap::nof_caller_save_cpu_regs(); i++) { 1281 LIR_Opr opr = FrameMap::caller_save_cpu_reg_at(i); 1282 assert(opr->is_valid() && opr->is_register(), "FrameMap should not return invalid operands"); 1283 assert(reg_numHi(opr) == -1, "missing addition of range for hi-register"); 1284 caller_save_registers[num_caller_save_registers++] = reg_num(opr); 1285 } 1286 1287 // temp ranges for fpu registers are only created when the method has 1288 // virtual fpu operands. Otherwise no allocation for fpu registers is 1289 // performed and so the temp ranges would be useless 1290 if (has_fpu_registers()) { 1291 #ifdef X86 1292 if (UseSSE < 2) { 1293 #endif // X86 1294 for (i = 0; i < FrameMap::nof_caller_save_fpu_regs; i++) { 1295 LIR_Opr opr = FrameMap::caller_save_fpu_reg_at(i); 1296 assert(opr->is_valid() && opr->is_register(), "FrameMap should not return invalid operands"); 1297 assert(reg_numHi(opr) == -1, "missing addition of range for hi-register"); 1298 caller_save_registers[num_caller_save_registers++] = reg_num(opr); 1299 } 1300 #ifdef X86 1301 } 1302 #endif // X86 1303 1304 #ifdef X86 1305 if (UseSSE > 0) { 1306 int num_caller_save_xmm_regs = FrameMap::get_num_caller_save_xmms(); 1307 for (i = 0; i < num_caller_save_xmm_regs; i ++) { 1308 LIR_Opr opr = FrameMap::caller_save_xmm_reg_at(i); 1309 assert(opr->is_valid() && opr->is_register(), "FrameMap should not return invalid operands"); 1310 assert(reg_numHi(opr) == -1, "missing addition of range for hi-register"); 1311 caller_save_registers[num_caller_save_registers++] = reg_num(opr); 1312 } 1313 } 1314 #endif // X86 1315 } 1316 assert(num_caller_save_registers <= LinearScan::nof_regs, "out of bounds"); 1317 1318 1319 LIR_OpVisitState visitor; 1320 1321 // iterate all blocks in reverse order 1322 for (i = block_count() - 1; i >= 0; i--) { 1323 BlockBegin* block = block_at(i); 1324 LIR_OpList* instructions = block->lir()->instructions_list(); 1325 int block_from = block->first_lir_instruction_id(); 1326 int block_to = block->last_lir_instruction_id(); 1327 1328 assert(block_from == instructions->at(0)->id(), "must be"); 1329 assert(block_to == instructions->at(instructions->length() - 1)->id(), "must be"); 1330 1331 // Update intervals for registers live at the end of this block; 1332 ResourceBitMap& live = block->live_out(); 1333 auto updater = [&](BitMap::idx_t index) { 1334 int number = static_cast<int>(index); 1335 assert(number >= LIR_Opr::vreg_base, "fixed intervals must not be live on block bounds"); 1336 TRACE_LINEAR_SCAN(2, tty->print_cr("live in %d to %d", number, block_to + 2)); 1337 1338 add_use(number, block_from, block_to + 2, noUse, T_ILLEGAL); 1339 1340 // add special use positions for loop-end blocks when the 1341 // interval is used anywhere inside this loop. It's possible 1342 // that the block was part of a non-natural loop, so it might 1343 // have an invalid loop index. 1344 if (block->is_set(BlockBegin::linear_scan_loop_end_flag) && 1345 block->loop_index() != -1 && 1346 is_interval_in_loop(number, block->loop_index())) { 1347 interval_at(number)->add_use_pos(block_to + 1, loopEndMarker); 1348 } 1349 }; 1350 live.iterate(updater); 1351 1352 // iterate all instructions of the block in reverse order. 1353 // skip the first instruction because it is always a label 1354 // definitions of intervals are processed before uses 1355 assert(visitor.no_operands(instructions->at(0)), "first operation must always be a label"); 1356 for (int j = instructions->length() - 1; j >= 1; j--) { 1357 LIR_Op* op = instructions->at(j); 1358 int op_id = op->id(); 1359 1360 // visit operation to collect all operands 1361 visitor.visit(op); 1362 1363 // add a temp range for each register if operation destroys caller-save registers 1364 if (visitor.has_call()) { 1365 for (int k = 0; k < num_caller_save_registers; k++) { 1366 add_temp(caller_save_registers[k], op_id, noUse, T_ILLEGAL); 1367 } 1368 TRACE_LINEAR_SCAN(4, tty->print_cr("operation destroys all caller-save registers")); 1369 } 1370 1371 // Add any platform dependent temps 1372 pd_add_temps(op); 1373 1374 // visit definitions (output and temp operands) 1375 int k, n; 1376 n = visitor.opr_count(LIR_OpVisitState::outputMode); 1377 for (k = 0; k < n; k++) { 1378 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::outputMode, k); 1379 assert(opr->is_register(), "visitor should only return register operands"); 1380 add_def(opr, op_id, use_kind_of_output_operand(op, opr)); 1381 } 1382 1383 n = visitor.opr_count(LIR_OpVisitState::tempMode); 1384 for (k = 0; k < n; k++) { 1385 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::tempMode, k); 1386 assert(opr->is_register(), "visitor should only return register operands"); 1387 add_temp(opr, op_id, mustHaveRegister); 1388 } 1389 1390 // visit uses (input operands) 1391 n = visitor.opr_count(LIR_OpVisitState::inputMode); 1392 for (k = 0; k < n; k++) { 1393 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::inputMode, k); 1394 assert(opr->is_register(), "visitor should only return register operands"); 1395 add_use(opr, block_from, op_id, use_kind_of_input_operand(op, opr)); 1396 } 1397 1398 // Add uses of live locals from interpreter's point of view for proper 1399 // debug information generation 1400 // Treat these operands as temp values (if the life range is extended 1401 // to a call site, the value would be in a register at the call otherwise) 1402 n = visitor.info_count(); 1403 for (k = 0; k < n; k++) { 1404 CodeEmitInfo* info = visitor.info_at(k); 1405 ValueStack* stack = info->stack(); 1406 for_each_state_value(stack, value, 1407 add_use(value, block_from, op_id + 1, noUse); 1408 ); 1409 } 1410 1411 // special steps for some instructions (especially moves) 1412 handle_method_arguments(op); 1413 handle_doubleword_moves(op); 1414 add_register_hints(op); 1415 1416 } // end of instruction iteration 1417 } // end of block iteration 1418 1419 1420 // add the range [0, 1[ to all fixed intervals 1421 // -> the register allocator need not handle unhandled fixed intervals 1422 for (int n = 0; n < LinearScan::nof_regs; n++) { 1423 Interval* interval = interval_at(n); 1424 if (interval != nullptr) { 1425 interval->add_range(0, 1); 1426 } 1427 } 1428 } 1429 1430 1431 // ********** Phase 5: actual register allocation 1432 1433 int LinearScan::interval_cmp(Interval** a, Interval** b) { 1434 if (*a != nullptr) { 1435 if (*b != nullptr) { 1436 return (*a)->from() - (*b)->from(); 1437 } else { 1438 return -1; 1439 } 1440 } else { 1441 if (*b != nullptr) { 1442 return 1; 1443 } else { 1444 return 0; 1445 } 1446 } 1447 } 1448 1449 #ifndef PRODUCT 1450 int interval_cmp(Interval* const& l, Interval* const& r) { 1451 return l->from() - r->from(); 1452 } 1453 1454 bool find_interval(Interval* interval, IntervalArray* intervals) { 1455 bool found; 1456 int idx = intervals->find_sorted<Interval*, interval_cmp>(interval, found); 1457 1458 if (!found) { 1459 return false; 1460 } 1461 1462 int from = interval->from(); 1463 1464 // The index we've found using binary search is pointing to an interval 1465 // that is defined in the same place as the interval we were looking for. 1466 // So now we have to look around that index and find exact interval. 1467 for (int i = idx; i >= 0; i--) { 1468 if (intervals->at(i) == interval) { 1469 return true; 1470 } 1471 if (intervals->at(i)->from() != from) { 1472 break; 1473 } 1474 } 1475 1476 for (int i = idx + 1; i < intervals->length(); i++) { 1477 if (intervals->at(i) == interval) { 1478 return true; 1479 } 1480 if (intervals->at(i)->from() != from) { 1481 break; 1482 } 1483 } 1484 1485 return false; 1486 } 1487 1488 bool LinearScan::is_sorted(IntervalArray* intervals) { 1489 int from = -1; 1490 int null_count = 0; 1491 1492 for (int i = 0; i < intervals->length(); i++) { 1493 Interval* it = intervals->at(i); 1494 if (it != nullptr) { 1495 assert(from <= it->from(), "Intervals are unordered"); 1496 from = it->from(); 1497 } else { 1498 null_count++; 1499 } 1500 } 1501 1502 assert(null_count == 0, "Sorted intervals should not contain nulls"); 1503 1504 null_count = 0; 1505 1506 for (int i = 0; i < interval_count(); i++) { 1507 Interval* interval = interval_at(i); 1508 if (interval != nullptr) { 1509 assert(find_interval(interval, intervals), "Lists do not contain same intervals"); 1510 } else { 1511 null_count++; 1512 } 1513 } 1514 1515 assert(interval_count() - null_count == intervals->length(), 1516 "Sorted list should contain the same amount of non-null intervals as unsorted list"); 1517 1518 return true; 1519 } 1520 #endif 1521 1522 void LinearScan::add_to_list(Interval** first, Interval** prev, Interval* interval) { 1523 if (*prev != nullptr) { 1524 (*prev)->set_next(interval); 1525 } else { 1526 *first = interval; 1527 } 1528 *prev = interval; 1529 } 1530 1531 void LinearScan::create_unhandled_lists(Interval** list1, Interval** list2, bool (is_list1)(const Interval* i), bool (is_list2)(const Interval* i)) { 1532 assert(is_sorted(_sorted_intervals), "interval list is not sorted"); 1533 1534 *list1 = *list2 = Interval::end(); 1535 1536 Interval* list1_prev = nullptr; 1537 Interval* list2_prev = nullptr; 1538 Interval* v; 1539 1540 const int n = _sorted_intervals->length(); 1541 for (int i = 0; i < n; i++) { 1542 v = _sorted_intervals->at(i); 1543 if (v == nullptr) continue; 1544 1545 if (is_list1(v)) { 1546 add_to_list(list1, &list1_prev, v); 1547 } else if (is_list2 == nullptr || is_list2(v)) { 1548 add_to_list(list2, &list2_prev, v); 1549 } 1550 } 1551 1552 if (list1_prev != nullptr) list1_prev->set_next(Interval::end()); 1553 if (list2_prev != nullptr) list2_prev->set_next(Interval::end()); 1554 1555 assert(list1_prev == nullptr || list1_prev->next() == Interval::end(), "linear list ends not with sentinel"); 1556 assert(list2_prev == nullptr || list2_prev->next() == Interval::end(), "linear list ends not with sentinel"); 1557 } 1558 1559 1560 void LinearScan::sort_intervals_before_allocation() { 1561 TIME_LINEAR_SCAN(timer_sort_intervals_before); 1562 1563 if (_needs_full_resort) { 1564 // There is no known reason why this should occur but just in case... 1565 assert(false, "should never occur"); 1566 // Re-sort existing interval list because an Interval::from() has changed 1567 _sorted_intervals->sort(interval_cmp); 1568 _needs_full_resort = false; 1569 } 1570 1571 IntervalList* unsorted_list = &_intervals; 1572 int unsorted_len = unsorted_list->length(); 1573 int sorted_len = 0; 1574 int unsorted_idx; 1575 int sorted_idx = 0; 1576 int sorted_from_max = -1; 1577 1578 // calc number of items for sorted list (sorted list must not contain null values) 1579 for (unsorted_idx = 0; unsorted_idx < unsorted_len; unsorted_idx++) { 1580 if (unsorted_list->at(unsorted_idx) != nullptr) { 1581 sorted_len++; 1582 } 1583 } 1584 IntervalArray* sorted_list = new IntervalArray(sorted_len, sorted_len, nullptr); 1585 1586 // special sorting algorithm: the original interval-list is almost sorted, 1587 // only some intervals are swapped. So this is much faster than a complete QuickSort 1588 for (unsorted_idx = 0; unsorted_idx < unsorted_len; unsorted_idx++) { 1589 Interval* cur_interval = unsorted_list->at(unsorted_idx); 1590 1591 if (cur_interval != nullptr) { 1592 int cur_from = cur_interval->from(); 1593 1594 if (sorted_from_max <= cur_from) { 1595 sorted_list->at_put(sorted_idx++, cur_interval); 1596 sorted_from_max = cur_interval->from(); 1597 } else { 1598 // the assumption that the intervals are already sorted failed, 1599 // so this interval must be sorted in manually 1600 int j; 1601 for (j = sorted_idx - 1; j >= 0 && cur_from < sorted_list->at(j)->from(); j--) { 1602 sorted_list->at_put(j + 1, sorted_list->at(j)); 1603 } 1604 sorted_list->at_put(j + 1, cur_interval); 1605 sorted_idx++; 1606 } 1607 } 1608 } 1609 _sorted_intervals = sorted_list; 1610 assert(is_sorted(_sorted_intervals), "intervals unsorted"); 1611 } 1612 1613 void LinearScan::sort_intervals_after_allocation() { 1614 TIME_LINEAR_SCAN(timer_sort_intervals_after); 1615 1616 if (_needs_full_resort) { 1617 // Re-sort existing interval list because an Interval::from() has changed 1618 _sorted_intervals->sort(interval_cmp); 1619 _needs_full_resort = false; 1620 } 1621 1622 IntervalArray* old_list = _sorted_intervals; 1623 IntervalList* new_list = _new_intervals_from_allocation; 1624 int old_len = old_list->length(); 1625 int new_len = new_list == nullptr ? 0 : new_list->length(); 1626 1627 if (new_len == 0) { 1628 // no intervals have been added during allocation, so sorted list is already up to date 1629 assert(is_sorted(_sorted_intervals), "intervals unsorted"); 1630 return; 1631 } 1632 1633 // conventional sort-algorithm for new intervals 1634 new_list->sort(interval_cmp); 1635 1636 // merge old and new list (both already sorted) into one combined list 1637 int combined_list_len = old_len + new_len; 1638 IntervalArray* combined_list = new IntervalArray(combined_list_len, combined_list_len, nullptr); 1639 int old_idx = 0; 1640 int new_idx = 0; 1641 1642 while (old_idx + new_idx < old_len + new_len) { 1643 if (new_idx >= new_len || (old_idx < old_len && old_list->at(old_idx)->from() <= new_list->at(new_idx)->from())) { 1644 combined_list->at_put(old_idx + new_idx, old_list->at(old_idx)); 1645 old_idx++; 1646 } else { 1647 combined_list->at_put(old_idx + new_idx, new_list->at(new_idx)); 1648 new_idx++; 1649 } 1650 } 1651 1652 _sorted_intervals = combined_list; 1653 assert(is_sorted(_sorted_intervals), "intervals unsorted"); 1654 } 1655 1656 1657 void LinearScan::allocate_registers() { 1658 TIME_LINEAR_SCAN(timer_allocate_registers); 1659 1660 Interval* precolored_cpu_intervals, *not_precolored_cpu_intervals; 1661 Interval* precolored_fpu_intervals, *not_precolored_fpu_intervals; 1662 1663 // collect cpu intervals 1664 create_unhandled_lists(&precolored_cpu_intervals, ¬_precolored_cpu_intervals, 1665 is_precolored_cpu_interval, is_virtual_cpu_interval); 1666 1667 // collect fpu intervals 1668 create_unhandled_lists(&precolored_fpu_intervals, ¬_precolored_fpu_intervals, 1669 is_precolored_fpu_interval, is_virtual_fpu_interval); 1670 // this fpu interval collection cannot be moved down below with the allocation section as 1671 // the cpu_lsw.walk() changes interval positions. 1672 1673 if (!has_fpu_registers()) { 1674 #ifdef ASSERT 1675 assert(not_precolored_fpu_intervals == Interval::end(), "missed an uncolored fpu interval"); 1676 #else 1677 if (not_precolored_fpu_intervals != Interval::end()) { 1678 BAILOUT("missed an uncolored fpu interval"); 1679 } 1680 #endif 1681 } 1682 1683 // allocate cpu registers 1684 LinearScanWalker cpu_lsw(this, precolored_cpu_intervals, not_precolored_cpu_intervals); 1685 cpu_lsw.walk(); 1686 cpu_lsw.finish_allocation(); 1687 1688 if (has_fpu_registers()) { 1689 // allocate fpu registers 1690 LinearScanWalker fpu_lsw(this, precolored_fpu_intervals, not_precolored_fpu_intervals); 1691 fpu_lsw.walk(); 1692 fpu_lsw.finish_allocation(); 1693 } 1694 } 1695 1696 1697 // ********** Phase 6: resolve data flow 1698 // (insert moves at edges between blocks if intervals have been split) 1699 1700 // wrapper for Interval::split_child_at_op_id that performs a bailout in product mode 1701 // instead of returning null 1702 Interval* LinearScan::split_child_at_op_id(Interval* interval, int op_id, LIR_OpVisitState::OprMode mode) { 1703 Interval* result = interval->split_child_at_op_id(op_id, mode); 1704 if (result != nullptr) { 1705 return result; 1706 } 1707 1708 assert(false, "must find an interval, but do a clean bailout in product mode"); 1709 result = new Interval(LIR_Opr::vreg_base); 1710 result->assign_reg(0); 1711 result->set_type(T_INT); 1712 BAILOUT_("LinearScan: interval is null", result); 1713 } 1714 1715 1716 Interval* LinearScan::interval_at_block_begin(BlockBegin* block, int reg_num) { 1717 assert(LinearScan::nof_regs <= reg_num && reg_num < num_virtual_regs(), "register number out of bounds"); 1718 assert(interval_at(reg_num) != nullptr, "no interval found"); 1719 1720 return split_child_at_op_id(interval_at(reg_num), block->first_lir_instruction_id(), LIR_OpVisitState::outputMode); 1721 } 1722 1723 Interval* LinearScan::interval_at_block_end(BlockBegin* block, int reg_num) { 1724 assert(LinearScan::nof_regs <= reg_num && reg_num < num_virtual_regs(), "register number out of bounds"); 1725 assert(interval_at(reg_num) != nullptr, "no interval found"); 1726 1727 return split_child_at_op_id(interval_at(reg_num), block->last_lir_instruction_id() + 1, LIR_OpVisitState::outputMode); 1728 } 1729 1730 Interval* LinearScan::interval_at_op_id(int reg_num, int op_id) { 1731 assert(LinearScan::nof_regs <= reg_num && reg_num < num_virtual_regs(), "register number out of bounds"); 1732 assert(interval_at(reg_num) != nullptr, "no interval found"); 1733 1734 return split_child_at_op_id(interval_at(reg_num), op_id, LIR_OpVisitState::inputMode); 1735 } 1736 1737 1738 void LinearScan::resolve_collect_mappings(BlockBegin* from_block, BlockBegin* to_block, MoveResolver &move_resolver) { 1739 DEBUG_ONLY(move_resolver.check_empty()); 1740 1741 // visit all registers where the live_at_edge bit is set 1742 const ResourceBitMap& live_at_edge = to_block->live_in(); 1743 auto visitor = [&](BitMap::idx_t index) { 1744 int r = static_cast<int>(index); 1745 assert(r < num_virtual_regs(), "live information set for not existing interval"); 1746 assert(from_block->live_out().at(r) && to_block->live_in().at(r), "interval not live at this edge"); 1747 1748 Interval* from_interval = interval_at_block_end(from_block, r); 1749 Interval* to_interval = interval_at_block_begin(to_block, r); 1750 1751 if (from_interval != to_interval && (from_interval->assigned_reg() != to_interval->assigned_reg() || from_interval->assigned_regHi() != to_interval->assigned_regHi())) { 1752 // need to insert move instruction 1753 move_resolver.add_mapping(from_interval, to_interval); 1754 } 1755 }; 1756 live_at_edge.iterate(visitor, 0, live_set_size()); 1757 } 1758 1759 1760 void LinearScan::resolve_find_insert_pos(BlockBegin* from_block, BlockBegin* to_block, MoveResolver &move_resolver) { 1761 if (from_block->number_of_sux() <= 1) { 1762 TRACE_LINEAR_SCAN(4, tty->print_cr("inserting moves at end of from_block B%d", from_block->block_id())); 1763 1764 LIR_OpList* instructions = from_block->lir()->instructions_list(); 1765 LIR_OpBranch* branch = instructions->last()->as_OpBranch(); 1766 if (branch != nullptr) { 1767 // insert moves before branch 1768 assert(branch->cond() == lir_cond_always, "block does not end with an unconditional jump"); 1769 move_resolver.set_insert_position(from_block->lir(), instructions->length() - 2); 1770 } else { 1771 move_resolver.set_insert_position(from_block->lir(), instructions->length() - 1); 1772 } 1773 1774 } else { 1775 TRACE_LINEAR_SCAN(4, tty->print_cr("inserting moves at beginning of to_block B%d", to_block->block_id())); 1776 #ifdef ASSERT 1777 assert(from_block->lir()->instructions_list()->at(0)->as_OpLabel() != nullptr, "block does not start with a label"); 1778 1779 // because the number of predecessor edges matches the number of 1780 // successor edges, blocks which are reached by switch statements 1781 // may have be more than one predecessor but it will be guaranteed 1782 // that all predecessors will be the same. 1783 for (int i = 0; i < to_block->number_of_preds(); i++) { 1784 assert(from_block == to_block->pred_at(i), "all critical edges must be broken"); 1785 } 1786 #endif 1787 1788 move_resolver.set_insert_position(to_block->lir(), 0); 1789 } 1790 } 1791 1792 1793 // insert necessary moves (spilling or reloading) at edges between blocks if interval has been split 1794 void LinearScan::resolve_data_flow() { 1795 TIME_LINEAR_SCAN(timer_resolve_data_flow); 1796 1797 int num_blocks = block_count(); 1798 MoveResolver move_resolver(this); 1799 ResourceBitMap block_completed(num_blocks); 1800 ResourceBitMap already_resolved(num_blocks); 1801 1802 int i; 1803 for (i = 0; i < num_blocks; i++) { 1804 BlockBegin* block = block_at(i); 1805 1806 // check if block has only one predecessor and only one successor 1807 if (block->number_of_preds() == 1 && block->number_of_sux() == 1 && block->number_of_exception_handlers() == 0) { 1808 LIR_OpList* instructions = block->lir()->instructions_list(); 1809 assert(instructions->at(0)->code() == lir_label, "block must start with label"); 1810 assert(instructions->last()->code() == lir_branch, "block with successors must end with branch"); 1811 assert(instructions->last()->as_OpBranch()->cond() == lir_cond_always, "block with successor must end with unconditional branch"); 1812 1813 // check if block is empty (only label and branch) 1814 if (instructions->length() == 2) { 1815 BlockBegin* pred = block->pred_at(0); 1816 BlockBegin* sux = block->sux_at(0); 1817 1818 // prevent optimization of two consecutive blocks 1819 if (!block_completed.at(pred->linear_scan_number()) && !block_completed.at(sux->linear_scan_number())) { 1820 TRACE_LINEAR_SCAN(3, tty->print_cr("**** optimizing empty block B%d (pred: B%d, sux: B%d)", block->block_id(), pred->block_id(), sux->block_id())); 1821 block_completed.set_bit(block->linear_scan_number()); 1822 1823 // directly resolve between pred and sux (without looking at the empty block between) 1824 resolve_collect_mappings(pred, sux, move_resolver); 1825 if (move_resolver.has_mappings()) { 1826 move_resolver.set_insert_position(block->lir(), 0); 1827 move_resolver.resolve_and_append_moves(); 1828 } 1829 } 1830 } 1831 } 1832 } 1833 1834 1835 for (i = 0; i < num_blocks; i++) { 1836 if (!block_completed.at(i)) { 1837 BlockBegin* from_block = block_at(i); 1838 already_resolved.set_from(block_completed); 1839 1840 int num_sux = from_block->number_of_sux(); 1841 for (int s = 0; s < num_sux; s++) { 1842 BlockBegin* to_block = from_block->sux_at(s); 1843 1844 // check for duplicate edges between the same blocks (can happen with switch blocks) 1845 if (!already_resolved.at(to_block->linear_scan_number())) { 1846 TRACE_LINEAR_SCAN(3, tty->print_cr("**** processing edge between B%d and B%d", from_block->block_id(), to_block->block_id())); 1847 already_resolved.set_bit(to_block->linear_scan_number()); 1848 1849 // collect all intervals that have been split between from_block and to_block 1850 resolve_collect_mappings(from_block, to_block, move_resolver); 1851 if (move_resolver.has_mappings()) { 1852 resolve_find_insert_pos(from_block, to_block, move_resolver); 1853 move_resolver.resolve_and_append_moves(); 1854 } 1855 } 1856 } 1857 } 1858 } 1859 } 1860 1861 1862 void LinearScan::resolve_exception_entry(BlockBegin* block, int reg_num, MoveResolver &move_resolver) { 1863 if (interval_at(reg_num) == nullptr) { 1864 // if a phi function is never used, no interval is created -> ignore this 1865 return; 1866 } 1867 1868 Interval* interval = interval_at_block_begin(block, reg_num); 1869 int reg = interval->assigned_reg(); 1870 int regHi = interval->assigned_regHi(); 1871 1872 if ((reg < nof_regs && interval->always_in_memory()) || 1873 (use_fpu_stack_allocation() && reg >= pd_first_fpu_reg && reg <= pd_last_fpu_reg)) { 1874 // the interval is split to get a short range that is located on the stack 1875 // in the following two cases: 1876 // * the interval started in memory (e.g. method parameter), but is currently in a register 1877 // this is an optimization for exception handling that reduces the number of moves that 1878 // are necessary for resolving the states when an exception uses this exception handler 1879 // * the interval would be on the fpu stack at the begin of the exception handler 1880 // this is not allowed because of the complicated fpu stack handling on Intel 1881 1882 // range that will be spilled to memory 1883 int from_op_id = block->first_lir_instruction_id(); 1884 int to_op_id = from_op_id + 1; // short live range of length 1 1885 assert(interval->from() <= from_op_id && interval->to() >= to_op_id, 1886 "no split allowed between exception entry and first instruction"); 1887 1888 if (interval->from() != from_op_id) { 1889 // the part before from_op_id is unchanged 1890 interval = interval->split(from_op_id); 1891 interval->assign_reg(reg, regHi); 1892 append_interval(interval); 1893 } else { 1894 _needs_full_resort = true; 1895 } 1896 assert(interval->from() == from_op_id, "must be true now"); 1897 1898 Interval* spilled_part = interval; 1899 if (interval->to() != to_op_id) { 1900 // the part after to_op_id is unchanged 1901 spilled_part = interval->split_from_start(to_op_id); 1902 append_interval(spilled_part); 1903 move_resolver.add_mapping(spilled_part, interval); 1904 } 1905 assign_spill_slot(spilled_part); 1906 1907 assert(spilled_part->from() == from_op_id && spilled_part->to() == to_op_id, "just checking"); 1908 } 1909 } 1910 1911 void LinearScan::resolve_exception_entry(BlockBegin* block, MoveResolver &move_resolver) { 1912 assert(block->is_set(BlockBegin::exception_entry_flag), "should not call otherwise"); 1913 DEBUG_ONLY(move_resolver.check_empty()); 1914 1915 // visit all registers where the live_in bit is set 1916 auto resolver = [&](BitMap::idx_t index) { 1917 int r = static_cast<int>(index); 1918 resolve_exception_entry(block, r, move_resolver); 1919 }; 1920 block->live_in().iterate(resolver, 0, live_set_size()); 1921 1922 // the live_in bits are not set for phi functions of the xhandler entry, so iterate them separately 1923 for_each_phi_fun(block, phi, 1924 if (!phi->is_illegal()) { resolve_exception_entry(block, phi->operand()->vreg_number(), move_resolver); } 1925 ); 1926 1927 if (move_resolver.has_mappings()) { 1928 // insert moves after first instruction 1929 move_resolver.set_insert_position(block->lir(), 0); 1930 move_resolver.resolve_and_append_moves(); 1931 } 1932 } 1933 1934 1935 void LinearScan::resolve_exception_edge(XHandler* handler, int throwing_op_id, int reg_num, Phi* phi, MoveResolver &move_resolver) { 1936 if (interval_at(reg_num) == nullptr) { 1937 // if a phi function is never used, no interval is created -> ignore this 1938 return; 1939 } 1940 1941 // the computation of to_interval is equal to resolve_collect_mappings, 1942 // but from_interval is more complicated because of phi functions 1943 BlockBegin* to_block = handler->entry_block(); 1944 Interval* to_interval = interval_at_block_begin(to_block, reg_num); 1945 1946 if (phi != nullptr) { 1947 // phi function of the exception entry block 1948 // no moves are created for this phi function in the LIR_Generator, so the 1949 // interval at the throwing instruction must be searched using the operands 1950 // of the phi function 1951 Value from_value = phi->operand_at(handler->phi_operand()); 1952 1953 // with phi functions it can happen that the same from_value is used in 1954 // multiple mappings, so notify move-resolver that this is allowed 1955 move_resolver.set_multiple_reads_allowed(); 1956 1957 Constant* con = from_value->as_Constant(); 1958 if (con != nullptr && (!con->is_pinned() || con->operand()->is_constant())) { 1959 // Need a mapping from constant to interval if unpinned (may have no register) or if the operand is a constant (no register). 1960 move_resolver.add_mapping(LIR_OprFact::value_type(con->type()), to_interval); 1961 } else { 1962 // search split child at the throwing op_id 1963 Interval* from_interval = interval_at_op_id(from_value->operand()->vreg_number(), throwing_op_id); 1964 move_resolver.add_mapping(from_interval, to_interval); 1965 } 1966 } else { 1967 // no phi function, so use reg_num also for from_interval 1968 // search split child at the throwing op_id 1969 Interval* from_interval = interval_at_op_id(reg_num, throwing_op_id); 1970 if (from_interval != to_interval) { 1971 // optimization to reduce number of moves: when to_interval is on stack and 1972 // the stack slot is known to be always correct, then no move is necessary 1973 if (!from_interval->always_in_memory() || from_interval->canonical_spill_slot() != to_interval->assigned_reg()) { 1974 move_resolver.add_mapping(from_interval, to_interval); 1975 } 1976 } 1977 } 1978 } 1979 1980 void LinearScan::resolve_exception_edge(XHandler* handler, int throwing_op_id, MoveResolver &move_resolver) { 1981 TRACE_LINEAR_SCAN(4, tty->print_cr("resolving exception handler B%d: throwing_op_id=%d", handler->entry_block()->block_id(), throwing_op_id)); 1982 1983 DEBUG_ONLY(move_resolver.check_empty()); 1984 assert(handler->lir_op_id() == -1, "already processed this xhandler"); 1985 DEBUG_ONLY(handler->set_lir_op_id(throwing_op_id)); 1986 assert(handler->entry_code() == nullptr, "code already present"); 1987 1988 // visit all registers where the live_in bit is set 1989 BlockBegin* block = handler->entry_block(); 1990 auto resolver = [&](BitMap::idx_t index) { 1991 int r = static_cast<int>(index); 1992 resolve_exception_edge(handler, throwing_op_id, r, nullptr, move_resolver); 1993 }; 1994 block->live_in().iterate(resolver, 0, live_set_size()); 1995 1996 // the live_in bits are not set for phi functions of the xhandler entry, so iterate them separately 1997 for_each_phi_fun(block, phi, 1998 if (!phi->is_illegal()) { resolve_exception_edge(handler, throwing_op_id, phi->operand()->vreg_number(), phi, move_resolver); } 1999 ); 2000 2001 if (move_resolver.has_mappings()) { 2002 LIR_List* entry_code = new LIR_List(compilation()); 2003 move_resolver.set_insert_position(entry_code, 0); 2004 move_resolver.resolve_and_append_moves(); 2005 2006 entry_code->jump(handler->entry_block()); 2007 handler->set_entry_code(entry_code); 2008 } 2009 } 2010 2011 2012 void LinearScan::resolve_exception_handlers() { 2013 MoveResolver move_resolver(this); 2014 LIR_OpVisitState visitor; 2015 int num_blocks = block_count(); 2016 2017 int i; 2018 for (i = 0; i < num_blocks; i++) { 2019 BlockBegin* block = block_at(i); 2020 if (block->is_set(BlockBegin::exception_entry_flag)) { 2021 resolve_exception_entry(block, move_resolver); 2022 } 2023 } 2024 2025 for (i = 0; i < num_blocks; i++) { 2026 BlockBegin* block = block_at(i); 2027 LIR_List* ops = block->lir(); 2028 int num_ops = ops->length(); 2029 2030 // iterate all instructions of the block. skip the first because it is always a label 2031 assert(visitor.no_operands(ops->at(0)), "first operation must always be a label"); 2032 for (int j = 1; j < num_ops; j++) { 2033 LIR_Op* op = ops->at(j); 2034 int op_id = op->id(); 2035 2036 if (op_id != -1 && has_info(op_id)) { 2037 // visit operation to collect all operands 2038 visitor.visit(op); 2039 assert(visitor.info_count() > 0, "should not visit otherwise"); 2040 2041 XHandlers* xhandlers = visitor.all_xhandler(); 2042 int n = xhandlers->length(); 2043 for (int k = 0; k < n; k++) { 2044 resolve_exception_edge(xhandlers->handler_at(k), op_id, move_resolver); 2045 } 2046 2047 #ifdef ASSERT 2048 } else { 2049 visitor.visit(op); 2050 assert(visitor.all_xhandler()->length() == 0, "missed exception handler"); 2051 #endif 2052 } 2053 } 2054 } 2055 } 2056 2057 2058 // ********** Phase 7: assign register numbers back to LIR 2059 // (includes computation of debug information and oop maps) 2060 2061 VMReg LinearScan::vm_reg_for_interval(Interval* interval) { 2062 VMReg reg = interval->cached_vm_reg(); 2063 if (!reg->is_valid() ) { 2064 reg = vm_reg_for_operand(operand_for_interval(interval)); 2065 interval->set_cached_vm_reg(reg); 2066 } 2067 assert(reg == vm_reg_for_operand(operand_for_interval(interval)), "wrong cached value"); 2068 return reg; 2069 } 2070 2071 VMReg LinearScan::vm_reg_for_operand(LIR_Opr opr) { 2072 assert(opr->is_oop(), "currently only implemented for oop operands"); 2073 return frame_map()->regname(opr); 2074 } 2075 2076 2077 LIR_Opr LinearScan::operand_for_interval(Interval* interval) { 2078 LIR_Opr opr = interval->cached_opr(); 2079 if (opr->is_illegal()) { 2080 opr = calc_operand_for_interval(interval); 2081 interval->set_cached_opr(opr); 2082 } 2083 2084 assert(opr == calc_operand_for_interval(interval), "wrong cached value"); 2085 return opr; 2086 } 2087 2088 LIR_Opr LinearScan::calc_operand_for_interval(const Interval* interval) { 2089 int assigned_reg = interval->assigned_reg(); 2090 BasicType type = interval->type(); 2091 2092 if (assigned_reg >= nof_regs) { 2093 // stack slot 2094 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); 2095 return LIR_OprFact::stack(assigned_reg - nof_regs, type); 2096 2097 } else { 2098 // register 2099 switch (type) { 2100 case T_OBJECT: { 2101 assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register"); 2102 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); 2103 return LIR_OprFact::single_cpu_oop(assigned_reg); 2104 } 2105 2106 case T_ADDRESS: { 2107 assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register"); 2108 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); 2109 return LIR_OprFact::single_cpu_address(assigned_reg); 2110 } 2111 2112 case T_METADATA: { 2113 assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register"); 2114 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); 2115 return LIR_OprFact::single_cpu_metadata(assigned_reg); 2116 } 2117 2118 #ifdef __SOFTFP__ 2119 case T_FLOAT: // fall through 2120 #endif // __SOFTFP__ 2121 case T_INT: { 2122 assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register"); 2123 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); 2124 return LIR_OprFact::single_cpu(assigned_reg); 2125 } 2126 2127 #ifdef __SOFTFP__ 2128 case T_DOUBLE: // fall through 2129 #endif // __SOFTFP__ 2130 case T_LONG: { 2131 int assigned_regHi = interval->assigned_regHi(); 2132 assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register"); 2133 assert(num_physical_regs(T_LONG) == 1 || 2134 (assigned_regHi >= pd_first_cpu_reg && assigned_regHi <= pd_last_cpu_reg), "no cpu register"); 2135 2136 assert(assigned_reg != assigned_regHi, "invalid allocation"); 2137 assert(num_physical_regs(T_LONG) == 1 || assigned_reg < assigned_regHi, 2138 "register numbers must be sorted (ensure that e.g. a move from eax,ebx to ebx,eax can not occur)"); 2139 assert((assigned_regHi != any_reg) ^ (num_physical_regs(T_LONG) == 1), "must be match"); 2140 if (requires_adjacent_regs(T_LONG)) { 2141 assert(assigned_reg % 2 == 0 && assigned_reg + 1 == assigned_regHi, "must be sequential and even"); 2142 } 2143 2144 #ifdef _LP64 2145 return LIR_OprFact::double_cpu(assigned_reg, assigned_reg); 2146 #else 2147 return LIR_OprFact::double_cpu(assigned_reg, assigned_regHi); 2148 #endif // LP64 2149 } 2150 2151 #ifndef __SOFTFP__ 2152 case T_FLOAT: { 2153 #ifdef X86 2154 if (UseSSE >= 1) { 2155 int last_xmm_reg = pd_last_xmm_reg; 2156 #ifdef _LP64 2157 if (UseAVX < 3) { 2158 last_xmm_reg = pd_first_xmm_reg + (pd_nof_xmm_regs_frame_map / 2) - 1; 2159 } 2160 #endif // LP64 2161 assert(assigned_reg >= pd_first_xmm_reg && assigned_reg <= last_xmm_reg, "no xmm register"); 2162 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); 2163 return LIR_OprFact::single_xmm(assigned_reg - pd_first_xmm_reg); 2164 } 2165 #endif // X86 2166 2167 assert(assigned_reg >= pd_first_fpu_reg && assigned_reg <= pd_last_fpu_reg, "no fpu register"); 2168 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); 2169 return LIR_OprFact::single_fpu(assigned_reg - pd_first_fpu_reg); 2170 } 2171 2172 case T_DOUBLE: { 2173 #ifdef X86 2174 if (UseSSE >= 2) { 2175 int last_xmm_reg = pd_last_xmm_reg; 2176 #ifdef _LP64 2177 if (UseAVX < 3) { 2178 last_xmm_reg = pd_first_xmm_reg + (pd_nof_xmm_regs_frame_map / 2) - 1; 2179 } 2180 #endif // LP64 2181 assert(assigned_reg >= pd_first_xmm_reg && assigned_reg <= last_xmm_reg, "no xmm register"); 2182 assert(interval->assigned_regHi() == any_reg, "must not have hi register (double xmm values are stored in one register)"); 2183 return LIR_OprFact::double_xmm(assigned_reg - pd_first_xmm_reg); 2184 } 2185 #endif // X86 2186 2187 #if defined(ARM32) 2188 assert(assigned_reg >= pd_first_fpu_reg && assigned_reg <= pd_last_fpu_reg, "no fpu register"); 2189 assert(interval->assigned_regHi() >= pd_first_fpu_reg && interval->assigned_regHi() <= pd_last_fpu_reg, "no fpu register"); 2190 assert(assigned_reg % 2 == 0 && assigned_reg + 1 == interval->assigned_regHi(), "must be sequential and even"); 2191 LIR_Opr result = LIR_OprFact::double_fpu(assigned_reg - pd_first_fpu_reg, interval->assigned_regHi() - pd_first_fpu_reg); 2192 #else 2193 assert(assigned_reg >= pd_first_fpu_reg && assigned_reg <= pd_last_fpu_reg, "no fpu register"); 2194 assert(interval->assigned_regHi() == any_reg, "must not have hi register (double fpu values are stored in one register on Intel)"); 2195 LIR_Opr result = LIR_OprFact::double_fpu(assigned_reg - pd_first_fpu_reg); 2196 #endif 2197 return result; 2198 } 2199 #endif // __SOFTFP__ 2200 2201 default: { 2202 ShouldNotReachHere(); 2203 return LIR_OprFact::illegalOpr; 2204 } 2205 } 2206 } 2207 } 2208 2209 LIR_Opr LinearScan::canonical_spill_opr(Interval* interval) { 2210 assert(interval->canonical_spill_slot() >= nof_regs, "canonical spill slot not set"); 2211 return LIR_OprFact::stack(interval->canonical_spill_slot() - nof_regs, interval->type()); 2212 } 2213 2214 LIR_Opr LinearScan::color_lir_opr(LIR_Opr opr, int op_id, LIR_OpVisitState::OprMode mode) { 2215 assert(opr->is_virtual(), "should not call this otherwise"); 2216 2217 Interval* interval = interval_at(opr->vreg_number()); 2218 assert(interval != nullptr, "interval must exist"); 2219 2220 if (op_id != -1) { 2221 #ifdef ASSERT 2222 BlockBegin* block = block_of_op_with_id(op_id); 2223 if (block->number_of_sux() <= 1 && op_id == block->last_lir_instruction_id()) { 2224 // check if spill moves could have been appended at the end of this block, but 2225 // before the branch instruction. So the split child information for this branch would 2226 // be incorrect. 2227 LIR_OpBranch* branch = block->lir()->instructions_list()->last()->as_OpBranch(); 2228 if (branch != nullptr) { 2229 if (block->live_out().at(opr->vreg_number())) { 2230 assert(branch->cond() == lir_cond_always, "block does not end with an unconditional jump"); 2231 assert(false, "can't get split child for the last branch of a block because the information would be incorrect (moves are inserted before the branch in resolve_data_flow)"); 2232 } 2233 } 2234 } 2235 #endif 2236 2237 // operands are not changed when an interval is split during allocation, 2238 // so search the right interval here 2239 interval = split_child_at_op_id(interval, op_id, mode); 2240 } 2241 2242 LIR_Opr res = operand_for_interval(interval); 2243 2244 #ifdef X86 2245 // new semantic for is_last_use: not only set on definite end of interval, 2246 // but also before hole 2247 // This may still miss some cases (e.g. for dead values), but it is not necessary that the 2248 // last use information is completely correct 2249 // information is only needed for fpu stack allocation 2250 if (res->is_fpu_register()) { 2251 if (opr->is_last_use() || op_id == interval->to() || (op_id != -1 && interval->has_hole_between(op_id, op_id + 1))) { 2252 assert(op_id == -1 || !is_block_begin(op_id), "holes at begin of block may also result from control flow"); 2253 res = res->make_last_use(); 2254 } 2255 } 2256 #endif 2257 2258 assert(!gen()->is_vreg_flag_set(opr->vreg_number(), LIRGenerator::callee_saved) || !FrameMap::is_caller_save_register(res), "bad allocation"); 2259 2260 return res; 2261 } 2262 2263 2264 #ifdef ASSERT 2265 // some methods used to check correctness of debug information 2266 2267 void assert_no_register_values(GrowableArray<ScopeValue*>* values) { 2268 if (values == nullptr) { 2269 return; 2270 } 2271 2272 for (int i = 0; i < values->length(); i++) { 2273 ScopeValue* value = values->at(i); 2274 2275 if (value->is_location()) { 2276 Location location = ((LocationValue*)value)->location(); 2277 assert(location.where() == Location::on_stack, "value is in register"); 2278 } 2279 } 2280 } 2281 2282 void assert_no_register_values(GrowableArray<MonitorValue*>* values) { 2283 if (values == nullptr) { 2284 return; 2285 } 2286 2287 for (int i = 0; i < values->length(); i++) { 2288 MonitorValue* value = values->at(i); 2289 2290 if (value->owner()->is_location()) { 2291 Location location = ((LocationValue*)value->owner())->location(); 2292 assert(location.where() == Location::on_stack, "owner is in register"); 2293 } 2294 assert(value->basic_lock().where() == Location::on_stack, "basic_lock is in register"); 2295 } 2296 } 2297 2298 void assert_equal(Location l1, Location l2) { 2299 assert(l1.where() == l2.where() && l1.type() == l2.type() && l1.offset() == l2.offset(), ""); 2300 } 2301 2302 void assert_equal(ScopeValue* v1, ScopeValue* v2) { 2303 if (v1->is_location()) { 2304 assert(v2->is_location(), ""); 2305 assert_equal(((LocationValue*)v1)->location(), ((LocationValue*)v2)->location()); 2306 } else if (v1->is_constant_int()) { 2307 assert(v2->is_constant_int(), ""); 2308 assert(((ConstantIntValue*)v1)->value() == ((ConstantIntValue*)v2)->value(), ""); 2309 } else if (v1->is_constant_double()) { 2310 assert(v2->is_constant_double(), ""); 2311 assert(((ConstantDoubleValue*)v1)->value() == ((ConstantDoubleValue*)v2)->value(), ""); 2312 } else if (v1->is_constant_long()) { 2313 assert(v2->is_constant_long(), ""); 2314 assert(((ConstantLongValue*)v1)->value() == ((ConstantLongValue*)v2)->value(), ""); 2315 } else if (v1->is_constant_oop()) { 2316 assert(v2->is_constant_oop(), ""); 2317 assert(((ConstantOopWriteValue*)v1)->value() == ((ConstantOopWriteValue*)v2)->value(), ""); 2318 } else { 2319 ShouldNotReachHere(); 2320 } 2321 } 2322 2323 void assert_equal(MonitorValue* m1, MonitorValue* m2) { 2324 assert_equal(m1->owner(), m2->owner()); 2325 assert_equal(m1->basic_lock(), m2->basic_lock()); 2326 } 2327 2328 void assert_equal(IRScopeDebugInfo* d1, IRScopeDebugInfo* d2) { 2329 assert(d1->scope() == d2->scope(), "not equal"); 2330 assert(d1->bci() == d2->bci(), "not equal"); 2331 2332 if (d1->locals() != nullptr) { 2333 assert(d1->locals() != nullptr && d2->locals() != nullptr, "not equal"); 2334 assert(d1->locals()->length() == d2->locals()->length(), "not equal"); 2335 for (int i = 0; i < d1->locals()->length(); i++) { 2336 assert_equal(d1->locals()->at(i), d2->locals()->at(i)); 2337 } 2338 } else { 2339 assert(d1->locals() == nullptr && d2->locals() == nullptr, "not equal"); 2340 } 2341 2342 if (d1->expressions() != nullptr) { 2343 assert(d1->expressions() != nullptr && d2->expressions() != nullptr, "not equal"); 2344 assert(d1->expressions()->length() == d2->expressions()->length(), "not equal"); 2345 for (int i = 0; i < d1->expressions()->length(); i++) { 2346 assert_equal(d1->expressions()->at(i), d2->expressions()->at(i)); 2347 } 2348 } else { 2349 assert(d1->expressions() == nullptr && d2->expressions() == nullptr, "not equal"); 2350 } 2351 2352 if (d1->monitors() != nullptr) { 2353 assert(d1->monitors() != nullptr && d2->monitors() != nullptr, "not equal"); 2354 assert(d1->monitors()->length() == d2->monitors()->length(), "not equal"); 2355 for (int i = 0; i < d1->monitors()->length(); i++) { 2356 assert_equal(d1->monitors()->at(i), d2->monitors()->at(i)); 2357 } 2358 } else { 2359 assert(d1->monitors() == nullptr && d2->monitors() == nullptr, "not equal"); 2360 } 2361 2362 if (d1->caller() != nullptr) { 2363 assert(d1->caller() != nullptr && d2->caller() != nullptr, "not equal"); 2364 assert_equal(d1->caller(), d2->caller()); 2365 } else { 2366 assert(d1->caller() == nullptr && d2->caller() == nullptr, "not equal"); 2367 } 2368 } 2369 2370 void check_stack_depth(CodeEmitInfo* info, int stack_end) { 2371 if (info->stack()->bci() != SynchronizationEntryBCI && !info->scope()->method()->is_native()) { 2372 Bytecodes::Code code = info->scope()->method()->java_code_at_bci(info->stack()->bci()); 2373 switch (code) { 2374 case Bytecodes::_ifnull : // fall through 2375 case Bytecodes::_ifnonnull : // fall through 2376 case Bytecodes::_ifeq : // fall through 2377 case Bytecodes::_ifne : // fall through 2378 case Bytecodes::_iflt : // fall through 2379 case Bytecodes::_ifge : // fall through 2380 case Bytecodes::_ifgt : // fall through 2381 case Bytecodes::_ifle : // fall through 2382 case Bytecodes::_if_icmpeq : // fall through 2383 case Bytecodes::_if_icmpne : // fall through 2384 case Bytecodes::_if_icmplt : // fall through 2385 case Bytecodes::_if_icmpge : // fall through 2386 case Bytecodes::_if_icmpgt : // fall through 2387 case Bytecodes::_if_icmple : // fall through 2388 case Bytecodes::_if_acmpeq : // fall through 2389 case Bytecodes::_if_acmpne : 2390 assert(stack_end >= -Bytecodes::depth(code), "must have non-empty expression stack at if bytecode"); 2391 break; 2392 default: 2393 break; 2394 } 2395 } 2396 } 2397 2398 #endif // ASSERT 2399 2400 2401 IntervalWalker* LinearScan::init_compute_oop_maps() { 2402 // setup lists of potential oops for walking 2403 Interval* oop_intervals; 2404 Interval* non_oop_intervals; 2405 2406 create_unhandled_lists(&oop_intervals, &non_oop_intervals, is_oop_interval, nullptr); 2407 2408 // intervals that have no oops inside need not to be processed 2409 // to ensure a walking until the last instruction id, add a dummy interval 2410 // with a high operation id 2411 non_oop_intervals = new Interval(any_reg); 2412 non_oop_intervals->add_range(max_jint - 2, max_jint - 1); 2413 2414 return new IntervalWalker(this, oop_intervals, non_oop_intervals); 2415 } 2416 2417 2418 OopMap* LinearScan::compute_oop_map(IntervalWalker* iw, LIR_Op* op, CodeEmitInfo* info, bool is_call_site) { 2419 TRACE_LINEAR_SCAN(3, tty->print_cr("creating oop map at op_id %d", op->id())); 2420 2421 // walk before the current operation -> intervals that start at 2422 // the operation (= output operands of the operation) are not 2423 // included in the oop map 2424 iw->walk_before(op->id()); 2425 2426 int frame_size = frame_map()->framesize(); 2427 int arg_count = frame_map()->oop_map_arg_count(); 2428 OopMap* map = new OopMap(frame_size, arg_count); 2429 2430 // Iterate through active intervals 2431 for (Interval* interval = iw->active_first(fixedKind); interval != Interval::end(); interval = interval->next()) { 2432 int assigned_reg = interval->assigned_reg(); 2433 2434 assert(interval->current_from() <= op->id() && op->id() <= interval->current_to(), "interval should not be active otherwise"); 2435 assert(interval->assigned_regHi() == any_reg, "oop must be single word"); 2436 assert(interval->reg_num() >= LIR_Opr::vreg_base, "fixed interval found"); 2437 2438 // Check if this range covers the instruction. Intervals that 2439 // start or end at the current operation are not included in the 2440 // oop map, except in the case of patching moves. For patching 2441 // moves, any intervals which end at this instruction are included 2442 // in the oop map since we may safepoint while doing the patch 2443 // before we've consumed the inputs. 2444 if (op->is_patching() || op->id() < interval->current_to()) { 2445 2446 // caller-save registers must not be included into oop-maps at calls 2447 assert(!is_call_site || assigned_reg >= nof_regs || !is_caller_save(assigned_reg), "interval is in a caller-save register at a call -> register will be overwritten"); 2448 2449 VMReg name = vm_reg_for_interval(interval); 2450 set_oop(map, name); 2451 2452 // Spill optimization: when the stack value is guaranteed to be always correct, 2453 // then it must be added to the oop map even if the interval is currently in a register 2454 if (interval->always_in_memory() && 2455 op->id() > interval->spill_definition_pos() && 2456 interval->assigned_reg() != interval->canonical_spill_slot()) { 2457 assert(interval->spill_definition_pos() > 0, "position not set correctly"); 2458 assert(interval->canonical_spill_slot() >= LinearScan::nof_regs, "no spill slot assigned"); 2459 assert(interval->assigned_reg() < LinearScan::nof_regs, "interval is on stack, so stack slot is registered twice"); 2460 2461 set_oop(map, frame_map()->slot_regname(interval->canonical_spill_slot() - LinearScan::nof_regs)); 2462 } 2463 } 2464 } 2465 2466 // add oops from lock stack 2467 assert(info->stack() != nullptr, "CodeEmitInfo must always have a stack"); 2468 int locks_count = info->stack()->total_locks_size(); 2469 for (int i = 0; i < locks_count; i++) { 2470 set_oop(map, frame_map()->monitor_object_regname(i)); 2471 } 2472 2473 return map; 2474 } 2475 2476 2477 void LinearScan::compute_oop_map(IntervalWalker* iw, const LIR_OpVisitState &visitor, LIR_Op* op) { 2478 assert(visitor.info_count() > 0, "no oop map needed"); 2479 2480 // compute oop_map only for first CodeEmitInfo 2481 // because it is (in most cases) equal for all other infos of the same operation 2482 CodeEmitInfo* first_info = visitor.info_at(0); 2483 OopMap* first_oop_map = compute_oop_map(iw, op, first_info, visitor.has_call()); 2484 2485 for (int i = 0; i < visitor.info_count(); i++) { 2486 CodeEmitInfo* info = visitor.info_at(i); 2487 OopMap* oop_map = first_oop_map; 2488 2489 // compute worst case interpreter size in case of a deoptimization 2490 _compilation->update_interpreter_frame_size(info->interpreter_frame_size()); 2491 2492 if (info->stack()->locks_size() != first_info->stack()->locks_size()) { 2493 // this info has a different number of locks then the precomputed oop map 2494 // (possible for lock and unlock instructions) -> compute oop map with 2495 // correct lock information 2496 oop_map = compute_oop_map(iw, op, info, visitor.has_call()); 2497 } 2498 2499 if (info->_oop_map == nullptr) { 2500 info->_oop_map = oop_map; 2501 } else { 2502 // a CodeEmitInfo can not be shared between different LIR-instructions 2503 // because interval splitting can occur anywhere between two instructions 2504 // and so the oop maps must be different 2505 // -> check if the already set oop_map is exactly the one calculated for this operation 2506 assert(info->_oop_map == oop_map, "same CodeEmitInfo used for multiple LIR instructions"); 2507 } 2508 } 2509 } 2510 2511 2512 // frequently used constants 2513 // Allocate them with new so they are never destroyed (otherwise, a 2514 // forced exit could destroy these objects while they are still in 2515 // use). 2516 ConstantOopWriteValue* LinearScan::_oop_null_scope_value = new (mtCompiler) ConstantOopWriteValue(nullptr); 2517 ConstantIntValue* LinearScan::_int_m1_scope_value = new (mtCompiler) ConstantIntValue(-1); 2518 ConstantIntValue* LinearScan::_int_0_scope_value = new (mtCompiler) ConstantIntValue((jint)0); 2519 ConstantIntValue* LinearScan::_int_1_scope_value = new (mtCompiler) ConstantIntValue(1); 2520 ConstantIntValue* LinearScan::_int_2_scope_value = new (mtCompiler) ConstantIntValue(2); 2521 LocationValue* _illegal_value = new (mtCompiler) LocationValue(Location()); 2522 2523 void LinearScan::init_compute_debug_info() { 2524 // cache for frequently used scope values 2525 // (cpu registers and stack slots) 2526 int cache_size = (LinearScan::nof_cpu_regs + frame_map()->argcount() + max_spills()) * 2; 2527 _scope_value_cache = ScopeValueArray(cache_size, cache_size, nullptr); 2528 } 2529 2530 MonitorValue* LinearScan::location_for_monitor_index(int monitor_index) { 2531 Location loc; 2532 if (!frame_map()->location_for_monitor_object(monitor_index, &loc)) { 2533 bailout("too large frame"); 2534 } 2535 ScopeValue* object_scope_value = new LocationValue(loc); 2536 2537 if (!frame_map()->location_for_monitor_lock(monitor_index, &loc)) { 2538 bailout("too large frame"); 2539 } 2540 return new MonitorValue(object_scope_value, loc); 2541 } 2542 2543 LocationValue* LinearScan::location_for_name(int name, Location::Type loc_type) { 2544 Location loc; 2545 if (!frame_map()->locations_for_slot(name, loc_type, &loc)) { 2546 bailout("too large frame"); 2547 } 2548 return new LocationValue(loc); 2549 } 2550 2551 2552 int LinearScan::append_scope_value_for_constant(LIR_Opr opr, GrowableArray<ScopeValue*>* scope_values) { 2553 assert(opr->is_constant(), "should not be called otherwise"); 2554 2555 LIR_Const* c = opr->as_constant_ptr(); 2556 BasicType t = c->type(); 2557 switch (t) { 2558 case T_OBJECT: { 2559 jobject value = c->as_jobject(); 2560 if (value == nullptr) { 2561 scope_values->append(_oop_null_scope_value); 2562 } else { 2563 scope_values->append(new ConstantOopWriteValue(c->as_jobject())); 2564 } 2565 return 1; 2566 } 2567 2568 case T_INT: // fall through 2569 case T_FLOAT: { 2570 int value = c->as_jint_bits(); 2571 switch (value) { 2572 case -1: scope_values->append(_int_m1_scope_value); break; 2573 case 0: scope_values->append(_int_0_scope_value); break; 2574 case 1: scope_values->append(_int_1_scope_value); break; 2575 case 2: scope_values->append(_int_2_scope_value); break; 2576 default: scope_values->append(new ConstantIntValue(c->as_jint_bits())); break; 2577 } 2578 return 1; 2579 } 2580 2581 case T_LONG: // fall through 2582 case T_DOUBLE: { 2583 #ifdef _LP64 2584 scope_values->append(_int_0_scope_value); 2585 scope_values->append(new ConstantLongValue(c->as_jlong_bits())); 2586 #else 2587 if (hi_word_offset_in_bytes > lo_word_offset_in_bytes) { 2588 scope_values->append(new ConstantIntValue(c->as_jint_hi_bits())); 2589 scope_values->append(new ConstantIntValue(c->as_jint_lo_bits())); 2590 } else { 2591 scope_values->append(new ConstantIntValue(c->as_jint_lo_bits())); 2592 scope_values->append(new ConstantIntValue(c->as_jint_hi_bits())); 2593 } 2594 #endif 2595 return 2; 2596 } 2597 2598 case T_ADDRESS: { 2599 #ifdef _LP64 2600 scope_values->append(new ConstantLongValue(c->as_jint())); 2601 #else 2602 scope_values->append(new ConstantIntValue(c->as_jint())); 2603 #endif 2604 return 1; 2605 } 2606 2607 default: 2608 ShouldNotReachHere(); 2609 return -1; 2610 } 2611 } 2612 2613 int LinearScan::append_scope_value_for_operand(LIR_Opr opr, GrowableArray<ScopeValue*>* scope_values) { 2614 if (opr->is_single_stack()) { 2615 int stack_idx = opr->single_stack_ix(); 2616 bool is_oop = opr->is_oop_register(); 2617 int cache_idx = (stack_idx + LinearScan::nof_cpu_regs) * 2 + (is_oop ? 1 : 0); 2618 2619 ScopeValue* sv = _scope_value_cache.at(cache_idx); 2620 if (sv == nullptr) { 2621 Location::Type loc_type = is_oop ? Location::oop : Location::normal; 2622 sv = location_for_name(stack_idx, loc_type); 2623 _scope_value_cache.at_put(cache_idx, sv); 2624 } 2625 2626 // check if cached value is correct 2627 DEBUG_ONLY(assert_equal(sv, location_for_name(stack_idx, is_oop ? Location::oop : Location::normal))); 2628 2629 scope_values->append(sv); 2630 return 1; 2631 2632 } else if (opr->is_single_cpu()) { 2633 bool is_oop = opr->is_oop_register(); 2634 int cache_idx = opr->cpu_regnr() * 2 + (is_oop ? 1 : 0); 2635 Location::Type int_loc_type = NOT_LP64(Location::normal) LP64_ONLY(Location::int_in_long); 2636 2637 ScopeValue* sv = _scope_value_cache.at(cache_idx); 2638 if (sv == nullptr) { 2639 Location::Type loc_type = is_oop ? Location::oop : int_loc_type; 2640 VMReg rname = frame_map()->regname(opr); 2641 sv = new LocationValue(Location::new_reg_loc(loc_type, rname)); 2642 _scope_value_cache.at_put(cache_idx, sv); 2643 } 2644 2645 // check if cached value is correct 2646 DEBUG_ONLY(assert_equal(sv, new LocationValue(Location::new_reg_loc(is_oop ? Location::oop : int_loc_type, frame_map()->regname(opr))))); 2647 2648 scope_values->append(sv); 2649 return 1; 2650 2651 #ifdef X86 2652 } else if (opr->is_single_xmm()) { 2653 VMReg rname = opr->as_xmm_float_reg()->as_VMReg(); 2654 LocationValue* sv = new LocationValue(Location::new_reg_loc(Location::normal, rname)); 2655 2656 scope_values->append(sv); 2657 return 1; 2658 #endif 2659 2660 } else if (opr->is_single_fpu()) { 2661 #ifdef IA32 2662 // the exact location of fpu stack values is only known 2663 // during fpu stack allocation, so the stack allocator object 2664 // must be present 2665 assert(use_fpu_stack_allocation(), "should not have float stack values without fpu stack allocation (all floats must be SSE2)"); 2666 assert(_fpu_stack_allocator != nullptr, "must be present"); 2667 opr = _fpu_stack_allocator->to_fpu_stack(opr); 2668 #elif defined(AMD64) 2669 assert(false, "FPU not used on x86-64"); 2670 #endif 2671 2672 Location::Type loc_type = float_saved_as_double ? Location::float_in_dbl : Location::normal; 2673 VMReg rname = frame_map()->fpu_regname(opr->fpu_regnr()); 2674 #ifndef __SOFTFP__ 2675 #ifndef VM_LITTLE_ENDIAN 2676 // On S390 a (single precision) float value occupies only the high 2677 // word of the full double register. So when the double register is 2678 // stored to memory (e.g. by the RegisterSaver), then the float value 2679 // is found at offset 0. I.e. the code below is not needed on S390. 2680 #ifndef S390 2681 if (! float_saved_as_double) { 2682 // On big endian system, we may have an issue if float registers use only 2683 // the low half of the (same) double registers. 2684 // Both the float and the double could have the same regnr but would correspond 2685 // to two different addresses once saved. 2686 2687 // get next safely (no assertion checks) 2688 VMReg next = VMRegImpl::as_VMReg(1+rname->value()); 2689 if (next->is_reg() && 2690 (next->as_FloatRegister() == rname->as_FloatRegister())) { 2691 // the back-end does use the same numbering for the double and the float 2692 rname = next; // VMReg for the low bits, e.g. the real VMReg for the float 2693 } 2694 } 2695 #endif // !S390 2696 #endif 2697 #endif 2698 LocationValue* sv = new LocationValue(Location::new_reg_loc(loc_type, rname)); 2699 2700 scope_values->append(sv); 2701 return 1; 2702 2703 } else { 2704 // double-size operands 2705 2706 ScopeValue* first; 2707 ScopeValue* second; 2708 2709 if (opr->is_double_stack()) { 2710 #ifdef _LP64 2711 Location loc1; 2712 Location::Type loc_type = opr->type() == T_LONG ? Location::lng : Location::dbl; 2713 if (!frame_map()->locations_for_slot(opr->double_stack_ix(), loc_type, &loc1, nullptr)) { 2714 bailout("too large frame"); 2715 } 2716 2717 first = new LocationValue(loc1); 2718 second = _int_0_scope_value; 2719 #else 2720 Location loc1, loc2; 2721 if (!frame_map()->locations_for_slot(opr->double_stack_ix(), Location::normal, &loc1, &loc2)) { 2722 bailout("too large frame"); 2723 } 2724 first = new LocationValue(loc1); 2725 second = new LocationValue(loc2); 2726 #endif // _LP64 2727 2728 } else if (opr->is_double_cpu()) { 2729 #ifdef _LP64 2730 VMReg rname_first = opr->as_register_lo()->as_VMReg(); 2731 first = new LocationValue(Location::new_reg_loc(Location::lng, rname_first)); 2732 second = _int_0_scope_value; 2733 #else 2734 VMReg rname_first = opr->as_register_lo()->as_VMReg(); 2735 VMReg rname_second = opr->as_register_hi()->as_VMReg(); 2736 2737 if (hi_word_offset_in_bytes < lo_word_offset_in_bytes) { 2738 // lo/hi and swapped relative to first and second, so swap them 2739 VMReg tmp = rname_first; 2740 rname_first = rname_second; 2741 rname_second = tmp; 2742 } 2743 2744 first = new LocationValue(Location::new_reg_loc(Location::normal, rname_first)); 2745 second = new LocationValue(Location::new_reg_loc(Location::normal, rname_second)); 2746 #endif //_LP64 2747 2748 2749 #ifdef X86 2750 } else if (opr->is_double_xmm()) { 2751 assert(opr->fpu_regnrLo() == opr->fpu_regnrHi(), "assumed in calculation"); 2752 VMReg rname_first = opr->as_xmm_double_reg()->as_VMReg(); 2753 # ifdef _LP64 2754 first = new LocationValue(Location::new_reg_loc(Location::dbl, rname_first)); 2755 second = _int_0_scope_value; 2756 # else 2757 first = new LocationValue(Location::new_reg_loc(Location::normal, rname_first)); 2758 // %%% This is probably a waste but we'll keep things as they were for now 2759 if (true) { 2760 VMReg rname_second = rname_first->next(); 2761 second = new LocationValue(Location::new_reg_loc(Location::normal, rname_second)); 2762 } 2763 # endif 2764 #endif 2765 2766 } else if (opr->is_double_fpu()) { 2767 // On SPARC, fpu_regnrLo/fpu_regnrHi represents the two halves of 2768 // the double as float registers in the native ordering. On X86, 2769 // fpu_regnrLo is a FPU stack slot whose VMReg represents 2770 // the low-order word of the double and fpu_regnrLo + 1 is the 2771 // name for the other half. *first and *second must represent the 2772 // least and most significant words, respectively. 2773 2774 #ifdef IA32 2775 // the exact location of fpu stack values is only known 2776 // during fpu stack allocation, so the stack allocator object 2777 // must be present 2778 assert(use_fpu_stack_allocation(), "should not have float stack values without fpu stack allocation (all floats must be SSE2)"); 2779 assert(_fpu_stack_allocator != nullptr, "must be present"); 2780 opr = _fpu_stack_allocator->to_fpu_stack(opr); 2781 2782 assert(opr->fpu_regnrLo() == opr->fpu_regnrHi(), "assumed in calculation (only fpu_regnrLo is used)"); 2783 #endif 2784 #ifdef AMD64 2785 assert(false, "FPU not used on x86-64"); 2786 #endif 2787 #ifdef ARM32 2788 assert(opr->fpu_regnrHi() == opr->fpu_regnrLo() + 1, "assumed in calculation (only fpu_regnrLo is used)"); 2789 #endif 2790 2791 #ifdef VM_LITTLE_ENDIAN 2792 VMReg rname_first = frame_map()->fpu_regname(opr->fpu_regnrLo()); 2793 #else 2794 VMReg rname_first = frame_map()->fpu_regname(opr->fpu_regnrHi()); 2795 #endif 2796 2797 #ifdef _LP64 2798 first = new LocationValue(Location::new_reg_loc(Location::dbl, rname_first)); 2799 second = _int_0_scope_value; 2800 #else 2801 first = new LocationValue(Location::new_reg_loc(Location::normal, rname_first)); 2802 // %%% This is probably a waste but we'll keep things as they were for now 2803 if (true) { 2804 VMReg rname_second = rname_first->next(); 2805 second = new LocationValue(Location::new_reg_loc(Location::normal, rname_second)); 2806 } 2807 #endif 2808 2809 } else { 2810 ShouldNotReachHere(); 2811 first = nullptr; 2812 second = nullptr; 2813 } 2814 2815 assert(first != nullptr && second != nullptr, "must be set"); 2816 // The convention the interpreter uses is that the second local 2817 // holds the first raw word of the native double representation. 2818 // This is actually reasonable, since locals and stack arrays 2819 // grow downwards in all implementations. 2820 // (If, on some machine, the interpreter's Java locals or stack 2821 // were to grow upwards, the embedded doubles would be word-swapped.) 2822 scope_values->append(second); 2823 scope_values->append(first); 2824 return 2; 2825 } 2826 } 2827 2828 2829 int LinearScan::append_scope_value(int op_id, Value value, GrowableArray<ScopeValue*>* scope_values) { 2830 if (value != nullptr) { 2831 LIR_Opr opr = value->operand(); 2832 Constant* con = value->as_Constant(); 2833 2834 assert(con == nullptr || opr->is_virtual() || opr->is_constant() || opr->is_illegal(), "assumption: Constant instructions have only constant operands (or illegal if constant is optimized away)"); 2835 assert(con != nullptr || opr->is_virtual(), "assumption: non-Constant instructions have only virtual operands"); 2836 2837 if (con != nullptr && !con->is_pinned() && !opr->is_constant()) { 2838 // Unpinned constants may have a virtual operand for a part of the lifetime 2839 // or may be illegal when it was optimized away, 2840 // so always use a constant operand 2841 opr = LIR_OprFact::value_type(con->type()); 2842 } 2843 assert(opr->is_virtual() || opr->is_constant(), "other cases not allowed here"); 2844 2845 if (opr->is_virtual()) { 2846 LIR_OpVisitState::OprMode mode = LIR_OpVisitState::inputMode; 2847 2848 BlockBegin* block = block_of_op_with_id(op_id); 2849 if (block->number_of_sux() == 1 && op_id == block->last_lir_instruction_id()) { 2850 // generating debug information for the last instruction of a block. 2851 // if this instruction is a branch, spill moves are inserted before this branch 2852 // and so the wrong operand would be returned (spill moves at block boundaries are not 2853 // considered in the live ranges of intervals) 2854 // Solution: use the first op_id of the branch target block instead. 2855 if (block->lir()->instructions_list()->last()->as_OpBranch() != nullptr) { 2856 if (block->live_out().at(opr->vreg_number())) { 2857 op_id = block->sux_at(0)->first_lir_instruction_id(); 2858 mode = LIR_OpVisitState::outputMode; 2859 } 2860 } 2861 } 2862 2863 // Get current location of operand 2864 // The operand must be live because debug information is considered when building the intervals 2865 // if the interval is not live, color_lir_opr will cause an assertion failure 2866 opr = color_lir_opr(opr, op_id, mode); 2867 assert(!has_call(op_id) || opr->is_stack() || !is_caller_save(reg_num(opr)), "can not have caller-save register operands at calls"); 2868 2869 // Append to ScopeValue array 2870 return append_scope_value_for_operand(opr, scope_values); 2871 2872 } else { 2873 assert(value->as_Constant() != nullptr, "all other instructions have only virtual operands"); 2874 assert(opr->is_constant(), "operand must be constant"); 2875 2876 return append_scope_value_for_constant(opr, scope_values); 2877 } 2878 } else { 2879 // append a dummy value because real value not needed 2880 scope_values->append(_illegal_value); 2881 return 1; 2882 } 2883 } 2884 2885 2886 IRScopeDebugInfo* LinearScan::compute_debug_info_for_scope(int op_id, IRScope* cur_scope, ValueStack* cur_state, ValueStack* innermost_state) { 2887 IRScopeDebugInfo* caller_debug_info = nullptr; 2888 2889 ValueStack* caller_state = cur_state->caller_state(); 2890 if (caller_state != nullptr) { 2891 // process recursively to compute outermost scope first 2892 caller_debug_info = compute_debug_info_for_scope(op_id, cur_scope->caller(), caller_state, innermost_state); 2893 } 2894 2895 // initialize these to null. 2896 // If we don't need deopt info or there are no locals, expressions or monitors, 2897 // then these get recorded as no information and avoids the allocation of 0 length arrays. 2898 GrowableArray<ScopeValue*>* locals = nullptr; 2899 GrowableArray<ScopeValue*>* expressions = nullptr; 2900 GrowableArray<MonitorValue*>* monitors = nullptr; 2901 2902 // describe local variable values 2903 int nof_locals = cur_state->locals_size(); 2904 if (nof_locals > 0) { 2905 locals = new GrowableArray<ScopeValue*>(nof_locals); 2906 2907 int pos = 0; 2908 while (pos < nof_locals) { 2909 assert(pos < cur_state->locals_size(), "why not?"); 2910 2911 Value local = cur_state->local_at(pos); 2912 pos += append_scope_value(op_id, local, locals); 2913 2914 assert(locals->length() == pos, "must match"); 2915 } 2916 assert(locals->length() == cur_scope->method()->max_locals(), "wrong number of locals"); 2917 assert(locals->length() == cur_state->locals_size(), "wrong number of locals"); 2918 } else if (cur_scope->method()->max_locals() > 0) { 2919 assert(cur_state->kind() == ValueStack::EmptyExceptionState, "should be"); 2920 nof_locals = cur_scope->method()->max_locals(); 2921 locals = new GrowableArray<ScopeValue*>(nof_locals); 2922 for(int i = 0; i < nof_locals; i++) { 2923 locals->append(_illegal_value); 2924 } 2925 } 2926 2927 // describe expression stack 2928 int nof_stack = cur_state->stack_size(); 2929 if (nof_stack > 0) { 2930 expressions = new GrowableArray<ScopeValue*>(nof_stack); 2931 2932 int pos = 0; 2933 while (pos < nof_stack) { 2934 Value expression = cur_state->stack_at_inc(pos); 2935 append_scope_value(op_id, expression, expressions); 2936 2937 assert(expressions->length() == pos, "must match"); 2938 } 2939 assert(expressions->length() == cur_state->stack_size(), "wrong number of stack entries"); 2940 } 2941 2942 // describe monitors 2943 int nof_locks = cur_state->locks_size(); 2944 if (nof_locks > 0) { 2945 int lock_offset = cur_state->caller_state() != nullptr ? cur_state->caller_state()->total_locks_size() : 0; 2946 monitors = new GrowableArray<MonitorValue*>(nof_locks); 2947 for (int i = 0; i < nof_locks; i++) { 2948 monitors->append(location_for_monitor_index(lock_offset + i)); 2949 } 2950 } 2951 2952 return new IRScopeDebugInfo(cur_scope, cur_state->bci(), locals, expressions, monitors, caller_debug_info, cur_state->should_reexecute()); 2953 } 2954 2955 2956 void LinearScan::compute_debug_info(CodeEmitInfo* info, int op_id) { 2957 TRACE_LINEAR_SCAN(3, tty->print_cr("creating debug information at op_id %d", op_id)); 2958 2959 IRScope* innermost_scope = info->scope(); 2960 ValueStack* innermost_state = info->stack(); 2961 2962 assert(innermost_scope != nullptr && innermost_state != nullptr, "why is it missing?"); 2963 2964 DEBUG_ONLY(check_stack_depth(info, innermost_state->stack_size())); 2965 2966 if (info->_scope_debug_info == nullptr) { 2967 // compute debug information 2968 info->_scope_debug_info = compute_debug_info_for_scope(op_id, innermost_scope, innermost_state, innermost_state); 2969 } else { 2970 // debug information already set. Check that it is correct from the current point of view 2971 DEBUG_ONLY(assert_equal(info->_scope_debug_info, compute_debug_info_for_scope(op_id, innermost_scope, innermost_state, innermost_state))); 2972 } 2973 } 2974 2975 2976 void LinearScan::assign_reg_num(LIR_OpList* instructions, IntervalWalker* iw) { 2977 LIR_OpVisitState visitor; 2978 int num_inst = instructions->length(); 2979 bool has_dead = false; 2980 2981 for (int j = 0; j < num_inst; j++) { 2982 LIR_Op* op = instructions->at(j); 2983 if (op == nullptr) { // this can happen when spill-moves are removed in eliminate_spill_moves 2984 has_dead = true; 2985 continue; 2986 } 2987 int op_id = op->id(); 2988 2989 // visit instruction to get list of operands 2990 visitor.visit(op); 2991 2992 // iterate all modes of the visitor and process all virtual operands 2993 for_each_visitor_mode(mode) { 2994 int n = visitor.opr_count(mode); 2995 for (int k = 0; k < n; k++) { 2996 LIR_Opr opr = visitor.opr_at(mode, k); 2997 if (opr->is_virtual_register()) { 2998 visitor.set_opr_at(mode, k, color_lir_opr(opr, op_id, mode)); 2999 } 3000 } 3001 } 3002 3003 if (visitor.info_count() > 0) { 3004 // exception handling 3005 if (compilation()->has_exception_handlers()) { 3006 XHandlers* xhandlers = visitor.all_xhandler(); 3007 int n = xhandlers->length(); 3008 for (int k = 0; k < n; k++) { 3009 XHandler* handler = xhandlers->handler_at(k); 3010 if (handler->entry_code() != nullptr) { 3011 assign_reg_num(handler->entry_code()->instructions_list(), nullptr); 3012 } 3013 } 3014 } else { 3015 assert(visitor.all_xhandler()->length() == 0, "missed exception handler"); 3016 } 3017 3018 // compute oop map 3019 assert(iw != nullptr, "needed for compute_oop_map"); 3020 compute_oop_map(iw, visitor, op); 3021 3022 // compute debug information 3023 if (!use_fpu_stack_allocation()) { 3024 // compute debug information if fpu stack allocation is not needed. 3025 // when fpu stack allocation is needed, the debug information can not 3026 // be computed here because the exact location of fpu operands is not known 3027 // -> debug information is created inside the fpu stack allocator 3028 int n = visitor.info_count(); 3029 for (int k = 0; k < n; k++) { 3030 compute_debug_info(visitor.info_at(k), op_id); 3031 } 3032 } 3033 } 3034 3035 #ifdef ASSERT 3036 // make sure we haven't made the op invalid. 3037 op->verify(); 3038 #endif 3039 3040 // remove useless moves 3041 if (op->code() == lir_move) { 3042 assert(op->as_Op1() != nullptr, "move must be LIR_Op1"); 3043 LIR_Op1* move = (LIR_Op1*)op; 3044 LIR_Opr src = move->in_opr(); 3045 LIR_Opr dst = move->result_opr(); 3046 if (dst == src || 3047 (!dst->is_pointer() && !src->is_pointer() && 3048 src->is_same_register(dst))) { 3049 instructions->at_put(j, nullptr); 3050 has_dead = true; 3051 } 3052 } 3053 } 3054 3055 if (has_dead) { 3056 // iterate all instructions of the block and remove all null-values. 3057 int insert_point = 0; 3058 for (int j = 0; j < num_inst; j++) { 3059 LIR_Op* op = instructions->at(j); 3060 if (op != nullptr) { 3061 if (insert_point != j) { 3062 instructions->at_put(insert_point, op); 3063 } 3064 insert_point++; 3065 } 3066 } 3067 instructions->trunc_to(insert_point); 3068 } 3069 } 3070 3071 void LinearScan::assign_reg_num() { 3072 TIME_LINEAR_SCAN(timer_assign_reg_num); 3073 3074 init_compute_debug_info(); 3075 IntervalWalker* iw = init_compute_oop_maps(); 3076 3077 int num_blocks = block_count(); 3078 for (int i = 0; i < num_blocks; i++) { 3079 BlockBegin* block = block_at(i); 3080 assign_reg_num(block->lir()->instructions_list(), iw); 3081 } 3082 } 3083 3084 3085 void LinearScan::do_linear_scan() { 3086 NOT_PRODUCT(_total_timer.begin_method()); 3087 3088 number_instructions(); 3089 3090 NOT_PRODUCT(print_lir(1, "Before Register Allocation")); 3091 3092 compute_local_live_sets(); 3093 compute_global_live_sets(); 3094 CHECK_BAILOUT(); 3095 3096 build_intervals(); 3097 CHECK_BAILOUT(); 3098 sort_intervals_before_allocation(); 3099 3100 NOT_PRODUCT(print_intervals("Before Register Allocation")); 3101 NOT_PRODUCT(LinearScanStatistic::compute(this, _stat_before_alloc)); 3102 3103 allocate_registers(); 3104 CHECK_BAILOUT(); 3105 3106 resolve_data_flow(); 3107 if (compilation()->has_exception_handlers()) { 3108 resolve_exception_handlers(); 3109 } 3110 // fill in number of spill slots into frame_map 3111 propagate_spill_slots(); 3112 CHECK_BAILOUT(); 3113 3114 NOT_PRODUCT(print_intervals("After Register Allocation")); 3115 NOT_PRODUCT(print_lir(2, "LIR after register allocation:")); 3116 3117 sort_intervals_after_allocation(); 3118 3119 DEBUG_ONLY(verify()); 3120 3121 eliminate_spill_moves(); 3122 assign_reg_num(); 3123 CHECK_BAILOUT(); 3124 3125 NOT_PRODUCT(print_lir(2, "LIR after assignment of register numbers:")); 3126 NOT_PRODUCT(LinearScanStatistic::compute(this, _stat_after_asign)); 3127 3128 { TIME_LINEAR_SCAN(timer_allocate_fpu_stack); 3129 3130 if (use_fpu_stack_allocation()) { 3131 allocate_fpu_stack(); // Only has effect on Intel 3132 NOT_PRODUCT(print_lir(2, "LIR after FPU stack allocation:")); 3133 } 3134 } 3135 3136 #ifndef RISCV 3137 // Disable these optimizations on riscv temporarily, because it does not 3138 // work when the comparison operands are bound to branches or cmoves. 3139 { TIME_LINEAR_SCAN(timer_optimize_lir); 3140 3141 EdgeMoveOptimizer::optimize(ir()->code()); 3142 ControlFlowOptimizer::optimize(ir()->code()); 3143 // check that cfg is still correct after optimizations 3144 ir()->verify(); 3145 } 3146 #endif 3147 3148 NOT_PRODUCT(print_lir(1, "Before Code Generation", false)); 3149 NOT_PRODUCT(LinearScanStatistic::compute(this, _stat_final)); 3150 NOT_PRODUCT(_total_timer.end_method(this)); 3151 } 3152 3153 3154 // ********** Printing functions 3155 3156 #ifndef PRODUCT 3157 3158 void LinearScan::print_timers(double total) { 3159 _total_timer.print(total); 3160 } 3161 3162 void LinearScan::print_statistics() { 3163 _stat_before_alloc.print("before allocation"); 3164 _stat_after_asign.print("after assignment of register"); 3165 _stat_final.print("after optimization"); 3166 } 3167 3168 void LinearScan::print_bitmap(BitMap& b) { 3169 for (unsigned int i = 0; i < b.size(); i++) { 3170 if (b.at(i)) tty->print("%d ", i); 3171 } 3172 tty->cr(); 3173 } 3174 3175 void LinearScan::print_intervals(const char* label) { 3176 if (TraceLinearScanLevel >= 1) { 3177 int i; 3178 tty->cr(); 3179 tty->print_cr("%s", label); 3180 3181 for (i = 0; i < interval_count(); i++) { 3182 Interval* interval = interval_at(i); 3183 if (interval != nullptr) { 3184 interval->print(); 3185 } 3186 } 3187 3188 tty->cr(); 3189 tty->print_cr("--- Basic Blocks ---"); 3190 for (i = 0; i < block_count(); i++) { 3191 BlockBegin* block = block_at(i); 3192 tty->print("B%d [%d, %d, %d, %d] ", block->block_id(), block->first_lir_instruction_id(), block->last_lir_instruction_id(), block->loop_index(), block->loop_depth()); 3193 } 3194 tty->cr(); 3195 tty->cr(); 3196 } 3197 3198 if (PrintCFGToFile) { 3199 CFGPrinter::print_intervals(&_intervals, label); 3200 } 3201 } 3202 3203 void LinearScan::print_lir(int level, const char* label, bool hir_valid) { 3204 if (TraceLinearScanLevel >= level) { 3205 tty->cr(); 3206 tty->print_cr("%s", label); 3207 print_LIR(ir()->linear_scan_order()); 3208 tty->cr(); 3209 } 3210 3211 if (level == 1 && PrintCFGToFile) { 3212 CFGPrinter::print_cfg(ir()->linear_scan_order(), label, hir_valid, true); 3213 } 3214 } 3215 3216 void LinearScan::print_reg_num(outputStream* out, int reg_num) { 3217 if (reg_num == -1) { 3218 out->print("[ANY]"); 3219 return; 3220 } else if (reg_num >= LIR_Opr::vreg_base) { 3221 out->print("[VREG %d]", reg_num); 3222 return; 3223 } 3224 3225 LIR_Opr opr = get_operand(reg_num); 3226 assert(opr->is_valid(), "unknown register"); 3227 opr->print(out); 3228 } 3229 3230 LIR_Opr LinearScan::get_operand(int reg_num) { 3231 LIR_Opr opr = LIR_OprFact::illegal(); 3232 3233 #ifdef X86 3234 int last_xmm_reg = pd_last_xmm_reg; 3235 #ifdef _LP64 3236 if (UseAVX < 3) { 3237 last_xmm_reg = pd_first_xmm_reg + (pd_nof_xmm_regs_frame_map / 2) - 1; 3238 } 3239 #endif 3240 #endif 3241 if (reg_num >= pd_first_cpu_reg && reg_num <= pd_last_cpu_reg) { 3242 opr = LIR_OprFact::single_cpu(reg_num); 3243 } else if (reg_num >= pd_first_fpu_reg && reg_num <= pd_last_fpu_reg) { 3244 opr = LIR_OprFact::single_fpu(reg_num - pd_first_fpu_reg); 3245 #ifdef X86 3246 } else if (reg_num >= pd_first_xmm_reg && reg_num <= last_xmm_reg) { 3247 opr = LIR_OprFact::single_xmm(reg_num - pd_first_xmm_reg); 3248 #endif 3249 } else { 3250 // reg_num == -1 or a virtual register, return the illegal operand 3251 } 3252 return opr; 3253 } 3254 3255 Interval* LinearScan::find_interval_at(int reg_num) const { 3256 if (reg_num < 0 || reg_num >= _intervals.length()) { 3257 return nullptr; 3258 } 3259 return interval_at(reg_num); 3260 } 3261 3262 #endif // PRODUCT 3263 3264 3265 // ********** verification functions for allocation 3266 // (check that all intervals have a correct register and that no registers are overwritten) 3267 #ifdef ASSERT 3268 3269 void LinearScan::verify() { 3270 TRACE_LINEAR_SCAN(2, tty->print_cr("********* verifying intervals ******************************************")); 3271 verify_intervals(); 3272 3273 TRACE_LINEAR_SCAN(2, tty->print_cr("********* verifying that no oops are in fixed intervals ****************")); 3274 verify_no_oops_in_fixed_intervals(); 3275 3276 TRACE_LINEAR_SCAN(2, tty->print_cr("********* verifying that unpinned constants are not alive across block boundaries")); 3277 verify_constants(); 3278 3279 TRACE_LINEAR_SCAN(2, tty->print_cr("********* verifying register allocation ********************************")); 3280 verify_registers(); 3281 3282 TRACE_LINEAR_SCAN(2, tty->print_cr("********* no errors found **********************************************")); 3283 } 3284 3285 void LinearScan::verify_intervals() { 3286 int len = interval_count(); 3287 bool has_error = false; 3288 3289 for (int i = 0; i < len; i++) { 3290 Interval* i1 = interval_at(i); 3291 if (i1 == nullptr) continue; 3292 3293 i1->check_split_children(); 3294 3295 if (i1->reg_num() != i) { 3296 tty->print_cr("Interval %d is on position %d in list", i1->reg_num(), i); i1->print(); tty->cr(); 3297 has_error = true; 3298 } 3299 3300 if (i1->reg_num() >= LIR_Opr::vreg_base && i1->type() == T_ILLEGAL) { 3301 tty->print_cr("Interval %d has no type assigned", i1->reg_num()); i1->print(); tty->cr(); 3302 has_error = true; 3303 } 3304 3305 if (i1->assigned_reg() == any_reg) { 3306 tty->print_cr("Interval %d has no register assigned", i1->reg_num()); i1->print(); tty->cr(); 3307 has_error = true; 3308 } 3309 3310 if (i1->assigned_reg() == i1->assigned_regHi()) { 3311 tty->print_cr("Interval %d: low and high register equal", i1->reg_num()); i1->print(); tty->cr(); 3312 has_error = true; 3313 } 3314 3315 if (!is_processed_reg_num(i1->assigned_reg())) { 3316 tty->print_cr("Can not have an Interval for an ignored register"); i1->print(); tty->cr(); 3317 has_error = true; 3318 } 3319 3320 // special intervals that are created in MoveResolver 3321 // -> ignore them because the range information has no meaning there 3322 if (i1->from() == 1 && i1->to() == 2) continue; 3323 3324 if (i1->first() == Range::end()) { 3325 tty->print_cr("Interval %d has no Range", i1->reg_num()); i1->print(); tty->cr(); 3326 has_error = true; 3327 } 3328 3329 for (Range* r = i1->first(); r != Range::end(); r = r->next()) { 3330 if (r->from() >= r->to()) { 3331 tty->print_cr("Interval %d has zero length range", i1->reg_num()); i1->print(); tty->cr(); 3332 has_error = true; 3333 } 3334 } 3335 3336 for (int j = i + 1; j < len; j++) { 3337 Interval* i2 = interval_at(j); 3338 if (i2 == nullptr || (i2->from() == 1 && i2->to() == 2)) continue; 3339 3340 int r1 = i1->assigned_reg(); 3341 int r1Hi = i1->assigned_regHi(); 3342 int r2 = i2->assigned_reg(); 3343 int r2Hi = i2->assigned_regHi(); 3344 if ((r1 == r2 || r1 == r2Hi || (r1Hi != any_reg && (r1Hi == r2 || r1Hi == r2Hi))) && i1->intersects(i2)) { 3345 tty->print_cr("Intervals %d and %d overlap and have the same register assigned", i1->reg_num(), i2->reg_num()); 3346 i1->print(); tty->cr(); 3347 i2->print(); tty->cr(); 3348 has_error = true; 3349 } 3350 } 3351 } 3352 3353 assert(has_error == false, "register allocation invalid"); 3354 } 3355 3356 3357 void LinearScan::verify_no_oops_in_fixed_intervals() { 3358 Interval* fixed_intervals; 3359 Interval* other_intervals; 3360 create_unhandled_lists(&fixed_intervals, &other_intervals, is_precolored_cpu_interval, nullptr); 3361 3362 // to ensure a walking until the last instruction id, add a dummy interval 3363 // with a high operation id 3364 other_intervals = new Interval(any_reg); 3365 other_intervals->add_range(max_jint - 2, max_jint - 1); 3366 IntervalWalker* iw = new IntervalWalker(this, fixed_intervals, other_intervals); 3367 3368 LIR_OpVisitState visitor; 3369 for (int i = 0; i < block_count(); i++) { 3370 BlockBegin* block = block_at(i); 3371 3372 LIR_OpList* instructions = block->lir()->instructions_list(); 3373 3374 for (int j = 0; j < instructions->length(); j++) { 3375 LIR_Op* op = instructions->at(j); 3376 int op_id = op->id(); 3377 3378 visitor.visit(op); 3379 3380 if (visitor.info_count() > 0) { 3381 iw->walk_before(op->id()); 3382 bool check_live = true; 3383 if (op->code() == lir_move) { 3384 LIR_Op1* move = (LIR_Op1*)op; 3385 check_live = (move->patch_code() == lir_patch_none); 3386 } 3387 LIR_OpBranch* branch = op->as_OpBranch(); 3388 if (branch != nullptr && branch->stub() != nullptr && branch->stub()->is_exception_throw_stub()) { 3389 // Don't bother checking the stub in this case since the 3390 // exception stub will never return to normal control flow. 3391 check_live = false; 3392 } 3393 3394 // Make sure none of the fixed registers is live across an 3395 // oopmap since we can't handle that correctly. 3396 if (check_live) { 3397 for (Interval* interval = iw->active_first(fixedKind); 3398 interval != Interval::end(); 3399 interval = interval->next()) { 3400 if (interval->current_to() > op->id() + 1) { 3401 // This interval is live out of this op so make sure 3402 // that this interval represents some value that's 3403 // referenced by this op either as an input or output. 3404 bool ok = false; 3405 for_each_visitor_mode(mode) { 3406 int n = visitor.opr_count(mode); 3407 for (int k = 0; k < n; k++) { 3408 LIR_Opr opr = visitor.opr_at(mode, k); 3409 if (opr->is_fixed_cpu()) { 3410 if (interval_at(reg_num(opr)) == interval) { 3411 ok = true; 3412 break; 3413 } 3414 int hi = reg_numHi(opr); 3415 if (hi != -1 && interval_at(hi) == interval) { 3416 ok = true; 3417 break; 3418 } 3419 } 3420 } 3421 } 3422 assert(ok, "fixed intervals should never be live across an oopmap point"); 3423 } 3424 } 3425 } 3426 } 3427 3428 // oop-maps at calls do not contain registers, so check is not needed 3429 if (!visitor.has_call()) { 3430 3431 for_each_visitor_mode(mode) { 3432 int n = visitor.opr_count(mode); 3433 for (int k = 0; k < n; k++) { 3434 LIR_Opr opr = visitor.opr_at(mode, k); 3435 3436 if (opr->is_fixed_cpu() && opr->is_oop()) { 3437 // operand is a non-virtual cpu register and contains an oop 3438 TRACE_LINEAR_SCAN(4, op->print_on(tty); tty->print("checking operand "); opr->print(); tty->cr()); 3439 3440 Interval* interval = interval_at(reg_num(opr)); 3441 assert(interval != nullptr, "no interval"); 3442 3443 if (mode == LIR_OpVisitState::inputMode) { 3444 if (interval->to() >= op_id + 1) { 3445 assert(interval->to() < op_id + 2 || 3446 interval->has_hole_between(op_id, op_id + 2), 3447 "oop input operand live after instruction"); 3448 } 3449 } else if (mode == LIR_OpVisitState::outputMode) { 3450 if (interval->from() <= op_id - 1) { 3451 assert(interval->has_hole_between(op_id - 1, op_id), 3452 "oop input operand live after instruction"); 3453 } 3454 } 3455 } 3456 } 3457 } 3458 } 3459 } 3460 } 3461 } 3462 3463 3464 void LinearScan::verify_constants() { 3465 int num_regs = num_virtual_regs(); 3466 int size = live_set_size(); 3467 int num_blocks = block_count(); 3468 3469 for (int i = 0; i < num_blocks; i++) { 3470 BlockBegin* block = block_at(i); 3471 ResourceBitMap& live_at_edge = block->live_in(); 3472 3473 // visit all registers where the live_at_edge bit is set 3474 auto visitor = [&](BitMap::idx_t index) { 3475 int r = static_cast<int>(index); 3476 TRACE_LINEAR_SCAN(4, tty->print("checking interval %d of block B%d", r, block->block_id())); 3477 3478 Value value = gen()->instruction_for_vreg(r); 3479 3480 assert(value != nullptr, "all intervals live across block boundaries must have Value"); 3481 assert(value->operand()->is_register() && value->operand()->is_virtual(), "value must have virtual operand"); 3482 assert(value->operand()->vreg_number() == r, "register number must match"); 3483 // TKR assert(value->as_Constant() == nullptr || value->is_pinned(), "only pinned constants can be alive across block boundaries"); 3484 }; 3485 live_at_edge.iterate(visitor, 0, size); 3486 } 3487 } 3488 3489 3490 class RegisterVerifier: public StackObj { 3491 private: 3492 LinearScan* _allocator; 3493 BlockList _work_list; // all blocks that must be processed 3494 IntervalsList _saved_states; // saved information of previous check 3495 3496 // simplified access to methods of LinearScan 3497 Compilation* compilation() const { return _allocator->compilation(); } 3498 Interval* interval_at(int reg_num) const { return _allocator->interval_at(reg_num); } 3499 int reg_num(LIR_Opr opr) const { return _allocator->reg_num(opr); } 3500 3501 // currently, only registers are processed 3502 int state_size() { return LinearScan::nof_regs; } 3503 3504 // accessors 3505 IntervalList* state_for_block(BlockBegin* block) { return _saved_states.at(block->block_id()); } 3506 void set_state_for_block(BlockBegin* block, IntervalList* saved_state) { _saved_states.at_put(block->block_id(), saved_state); } 3507 void add_to_work_list(BlockBegin* block) { if (!_work_list.contains(block)) _work_list.append(block); } 3508 3509 // helper functions 3510 IntervalList* copy(IntervalList* input_state); 3511 void state_put(IntervalList* input_state, int reg, Interval* interval); 3512 bool check_state(IntervalList* input_state, int reg, Interval* interval); 3513 3514 void process_block(BlockBegin* block); 3515 void process_xhandler(XHandler* xhandler, IntervalList* input_state); 3516 void process_successor(BlockBegin* block, IntervalList* input_state); 3517 void process_operations(LIR_List* ops, IntervalList* input_state); 3518 3519 public: 3520 RegisterVerifier(LinearScan* allocator) 3521 : _allocator(allocator) 3522 , _work_list(16) 3523 , _saved_states(BlockBegin::number_of_blocks(), BlockBegin::number_of_blocks(), nullptr) 3524 { } 3525 3526 void verify(BlockBegin* start); 3527 }; 3528 3529 3530 // entry function from LinearScan that starts the verification 3531 void LinearScan::verify_registers() { 3532 RegisterVerifier verifier(this); 3533 verifier.verify(block_at(0)); 3534 } 3535 3536 3537 void RegisterVerifier::verify(BlockBegin* start) { 3538 // setup input registers (method arguments) for first block 3539 int input_state_len = state_size(); 3540 IntervalList* input_state = new IntervalList(input_state_len, input_state_len, nullptr); 3541 CallingConvention* args = compilation()->frame_map()->incoming_arguments(); 3542 for (int n = 0; n < args->length(); n++) { 3543 LIR_Opr opr = args->at(n); 3544 if (opr->is_register()) { 3545 Interval* interval = interval_at(reg_num(opr)); 3546 3547 if (interval->assigned_reg() < state_size()) { 3548 input_state->at_put(interval->assigned_reg(), interval); 3549 } 3550 if (interval->assigned_regHi() != LinearScan::any_reg && interval->assigned_regHi() < state_size()) { 3551 input_state->at_put(interval->assigned_regHi(), interval); 3552 } 3553 } 3554 } 3555 3556 set_state_for_block(start, input_state); 3557 add_to_work_list(start); 3558 3559 // main loop for verification 3560 do { 3561 BlockBegin* block = _work_list.at(0); 3562 _work_list.remove_at(0); 3563 3564 process_block(block); 3565 } while (!_work_list.is_empty()); 3566 } 3567 3568 void RegisterVerifier::process_block(BlockBegin* block) { 3569 TRACE_LINEAR_SCAN(2, tty->cr(); tty->print_cr("process_block B%d", block->block_id())); 3570 3571 // must copy state because it is modified 3572 IntervalList* input_state = copy(state_for_block(block)); 3573 3574 if (TraceLinearScanLevel >= 4) { 3575 tty->print_cr("Input-State of intervals:"); 3576 tty->print(" "); 3577 for (int i = 0; i < state_size(); i++) { 3578 if (input_state->at(i) != nullptr) { 3579 tty->print(" %4d", input_state->at(i)->reg_num()); 3580 } else { 3581 tty->print(" __"); 3582 } 3583 } 3584 tty->cr(); 3585 tty->cr(); 3586 } 3587 3588 // process all operations of the block 3589 process_operations(block->lir(), input_state); 3590 3591 // iterate all successors 3592 for (int i = 0; i < block->number_of_sux(); i++) { 3593 process_successor(block->sux_at(i), input_state); 3594 } 3595 } 3596 3597 void RegisterVerifier::process_xhandler(XHandler* xhandler, IntervalList* input_state) { 3598 TRACE_LINEAR_SCAN(2, tty->print_cr("process_xhandler B%d", xhandler->entry_block()->block_id())); 3599 3600 // must copy state because it is modified 3601 input_state = copy(input_state); 3602 3603 if (xhandler->entry_code() != nullptr) { 3604 process_operations(xhandler->entry_code(), input_state); 3605 } 3606 process_successor(xhandler->entry_block(), input_state); 3607 } 3608 3609 void RegisterVerifier::process_successor(BlockBegin* block, IntervalList* input_state) { 3610 IntervalList* saved_state = state_for_block(block); 3611 3612 if (saved_state != nullptr) { 3613 // this block was already processed before. 3614 // check if new input_state is consistent with saved_state 3615 3616 bool saved_state_correct = true; 3617 for (int i = 0; i < state_size(); i++) { 3618 if (input_state->at(i) != saved_state->at(i)) { 3619 // current input_state and previous saved_state assume a different 3620 // interval in this register -> assume that this register is invalid 3621 if (saved_state->at(i) != nullptr) { 3622 // invalidate old calculation only if it assumed that 3623 // register was valid. when the register was already invalid, 3624 // then the old calculation was correct. 3625 saved_state_correct = false; 3626 saved_state->at_put(i, nullptr); 3627 3628 TRACE_LINEAR_SCAN(4, tty->print_cr("process_successor B%d: invalidating slot %d", block->block_id(), i)); 3629 } 3630 } 3631 } 3632 3633 if (saved_state_correct) { 3634 // already processed block with correct input_state 3635 TRACE_LINEAR_SCAN(2, tty->print_cr("process_successor B%d: previous visit already correct", block->block_id())); 3636 } else { 3637 // must re-visit this block 3638 TRACE_LINEAR_SCAN(2, tty->print_cr("process_successor B%d: must re-visit because input state changed", block->block_id())); 3639 add_to_work_list(block); 3640 } 3641 3642 } else { 3643 // block was not processed before, so set initial input_state 3644 TRACE_LINEAR_SCAN(2, tty->print_cr("process_successor B%d: initial visit", block->block_id())); 3645 3646 set_state_for_block(block, copy(input_state)); 3647 add_to_work_list(block); 3648 } 3649 } 3650 3651 3652 IntervalList* RegisterVerifier::copy(IntervalList* input_state) { 3653 IntervalList* copy_state = new IntervalList(input_state->length()); 3654 copy_state->appendAll(input_state); 3655 return copy_state; 3656 } 3657 3658 void RegisterVerifier::state_put(IntervalList* input_state, int reg, Interval* interval) { 3659 if (reg != LinearScan::any_reg && reg < state_size()) { 3660 if (interval != nullptr) { 3661 TRACE_LINEAR_SCAN(4, tty->print_cr(" reg[%d] = %d", reg, interval->reg_num())); 3662 } else if (input_state->at(reg) != nullptr) { 3663 TRACE_LINEAR_SCAN(4, tty->print_cr(" reg[%d] = null", reg)); 3664 } 3665 3666 input_state->at_put(reg, interval); 3667 } 3668 } 3669 3670 bool RegisterVerifier::check_state(IntervalList* input_state, int reg, Interval* interval) { 3671 if (reg != LinearScan::any_reg && reg < state_size()) { 3672 if (input_state->at(reg) != interval) { 3673 tty->print_cr("!! Error in register allocation: register %d does not contain interval %d", reg, interval->reg_num()); 3674 return true; 3675 } 3676 } 3677 return false; 3678 } 3679 3680 void RegisterVerifier::process_operations(LIR_List* ops, IntervalList* input_state) { 3681 // visit all instructions of the block 3682 LIR_OpVisitState visitor; 3683 bool has_error = false; 3684 3685 for (int i = 0; i < ops->length(); i++) { 3686 LIR_Op* op = ops->at(i); 3687 visitor.visit(op); 3688 3689 TRACE_LINEAR_SCAN(4, op->print_on(tty)); 3690 3691 // check if input operands are correct 3692 int j; 3693 int n = visitor.opr_count(LIR_OpVisitState::inputMode); 3694 for (j = 0; j < n; j++) { 3695 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::inputMode, j); 3696 if (opr->is_register() && LinearScan::is_processed_reg_num(reg_num(opr))) { 3697 Interval* interval = interval_at(reg_num(opr)); 3698 if (op->id() != -1) { 3699 interval = interval->split_child_at_op_id(op->id(), LIR_OpVisitState::inputMode); 3700 } 3701 3702 has_error |= check_state(input_state, interval->assigned_reg(), interval->split_parent()); 3703 has_error |= check_state(input_state, interval->assigned_regHi(), interval->split_parent()); 3704 3705 // When an operand is marked with is_last_use, then the fpu stack allocator 3706 // removes the register from the fpu stack -> the register contains no value 3707 if (opr->is_last_use()) { 3708 state_put(input_state, interval->assigned_reg(), nullptr); 3709 state_put(input_state, interval->assigned_regHi(), nullptr); 3710 } 3711 } 3712 } 3713 3714 // invalidate all caller save registers at calls 3715 if (visitor.has_call()) { 3716 for (j = 0; j < FrameMap::nof_caller_save_cpu_regs(); j++) { 3717 state_put(input_state, reg_num(FrameMap::caller_save_cpu_reg_at(j)), nullptr); 3718 } 3719 for (j = 0; j < FrameMap::nof_caller_save_fpu_regs; j++) { 3720 state_put(input_state, reg_num(FrameMap::caller_save_fpu_reg_at(j)), nullptr); 3721 } 3722 3723 #ifdef X86 3724 int num_caller_save_xmm_regs = FrameMap::get_num_caller_save_xmms(); 3725 for (j = 0; j < num_caller_save_xmm_regs; j++) { 3726 state_put(input_state, reg_num(FrameMap::caller_save_xmm_reg_at(j)), nullptr); 3727 } 3728 #endif 3729 } 3730 3731 // process xhandler before output and temp operands 3732 XHandlers* xhandlers = visitor.all_xhandler(); 3733 n = xhandlers->length(); 3734 for (int k = 0; k < n; k++) { 3735 process_xhandler(xhandlers->handler_at(k), input_state); 3736 } 3737 3738 // set temp operands (some operations use temp operands also as output operands, so can't set them null) 3739 n = visitor.opr_count(LIR_OpVisitState::tempMode); 3740 for (j = 0; j < n; j++) { 3741 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::tempMode, j); 3742 if (opr->is_register() && LinearScan::is_processed_reg_num(reg_num(opr))) { 3743 Interval* interval = interval_at(reg_num(opr)); 3744 if (op->id() != -1) { 3745 interval = interval->split_child_at_op_id(op->id(), LIR_OpVisitState::tempMode); 3746 } 3747 3748 state_put(input_state, interval->assigned_reg(), interval->split_parent()); 3749 state_put(input_state, interval->assigned_regHi(), interval->split_parent()); 3750 } 3751 } 3752 3753 // set output operands 3754 n = visitor.opr_count(LIR_OpVisitState::outputMode); 3755 for (j = 0; j < n; j++) { 3756 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::outputMode, j); 3757 if (opr->is_register() && LinearScan::is_processed_reg_num(reg_num(opr))) { 3758 Interval* interval = interval_at(reg_num(opr)); 3759 if (op->id() != -1) { 3760 interval = interval->split_child_at_op_id(op->id(), LIR_OpVisitState::outputMode); 3761 } 3762 3763 state_put(input_state, interval->assigned_reg(), interval->split_parent()); 3764 state_put(input_state, interval->assigned_regHi(), interval->split_parent()); 3765 } 3766 } 3767 } 3768 assert(has_error == false, "Error in register allocation"); 3769 } 3770 3771 #endif // ASSERT 3772 3773 3774 3775 // **** Implementation of MoveResolver ****************************** 3776 3777 MoveResolver::MoveResolver(LinearScan* allocator) : 3778 _allocator(allocator), 3779 _insert_list(nullptr), 3780 _insert_idx(-1), 3781 _insertion_buffer(), 3782 _mapping_from(8), 3783 _mapping_from_opr(8), 3784 _mapping_to(8), 3785 _multiple_reads_allowed(false) 3786 { 3787 for (int i = 0; i < LinearScan::nof_regs; i++) { 3788 _register_blocked[i] = 0; 3789 } 3790 DEBUG_ONLY(check_empty()); 3791 } 3792 3793 3794 #ifdef ASSERT 3795 3796 void MoveResolver::check_empty() { 3797 assert(_mapping_from.length() == 0 && _mapping_from_opr.length() == 0 && _mapping_to.length() == 0, "list must be empty before and after processing"); 3798 for (int i = 0; i < LinearScan::nof_regs; i++) { 3799 assert(register_blocked(i) == 0, "register map must be empty before and after processing"); 3800 } 3801 assert(_multiple_reads_allowed == false, "must have default value"); 3802 } 3803 3804 void MoveResolver::verify_before_resolve() { 3805 assert(_mapping_from.length() == _mapping_from_opr.length(), "length must be equal"); 3806 assert(_mapping_from.length() == _mapping_to.length(), "length must be equal"); 3807 assert(_insert_list != nullptr && _insert_idx != -1, "insert position not set"); 3808 3809 int i, j; 3810 if (!_multiple_reads_allowed) { 3811 for (i = 0; i < _mapping_from.length(); i++) { 3812 for (j = i + 1; j < _mapping_from.length(); j++) { 3813 assert(_mapping_from.at(i) == nullptr || _mapping_from.at(i) != _mapping_from.at(j), "cannot read from same interval twice"); 3814 } 3815 } 3816 } 3817 3818 for (i = 0; i < _mapping_to.length(); i++) { 3819 for (j = i + 1; j < _mapping_to.length(); j++) { 3820 assert(_mapping_to.at(i) != _mapping_to.at(j), "cannot write to same interval twice"); 3821 } 3822 } 3823 3824 3825 ResourceBitMap used_regs(LinearScan::nof_regs + allocator()->frame_map()->argcount() + allocator()->max_spills()); 3826 if (!_multiple_reads_allowed) { 3827 for (i = 0; i < _mapping_from.length(); i++) { 3828 Interval* it = _mapping_from.at(i); 3829 if (it != nullptr) { 3830 assert(!used_regs.at(it->assigned_reg()), "cannot read from same register twice"); 3831 used_regs.set_bit(it->assigned_reg()); 3832 3833 if (it->assigned_regHi() != LinearScan::any_reg) { 3834 assert(!used_regs.at(it->assigned_regHi()), "cannot read from same register twice"); 3835 used_regs.set_bit(it->assigned_regHi()); 3836 } 3837 } 3838 } 3839 } 3840 3841 used_regs.clear(); 3842 for (i = 0; i < _mapping_to.length(); i++) { 3843 Interval* it = _mapping_to.at(i); 3844 assert(!used_regs.at(it->assigned_reg()), "cannot write to same register twice"); 3845 used_regs.set_bit(it->assigned_reg()); 3846 3847 if (it->assigned_regHi() != LinearScan::any_reg) { 3848 assert(!used_regs.at(it->assigned_regHi()), "cannot write to same register twice"); 3849 used_regs.set_bit(it->assigned_regHi()); 3850 } 3851 } 3852 3853 used_regs.clear(); 3854 for (i = 0; i < _mapping_from.length(); i++) { 3855 Interval* it = _mapping_from.at(i); 3856 if (it != nullptr && it->assigned_reg() >= LinearScan::nof_regs) { 3857 used_regs.set_bit(it->assigned_reg()); 3858 } 3859 } 3860 for (i = 0; i < _mapping_to.length(); i++) { 3861 Interval* it = _mapping_to.at(i); 3862 assert(!used_regs.at(it->assigned_reg()) || it->assigned_reg() == _mapping_from.at(i)->assigned_reg(), "stack slots used in _mapping_from must be disjoint to _mapping_to"); 3863 } 3864 } 3865 3866 #endif // ASSERT 3867 3868 3869 // mark assigned_reg and assigned_regHi of the interval as blocked 3870 void MoveResolver::block_registers(Interval* it) { 3871 int reg = it->assigned_reg(); 3872 if (reg < LinearScan::nof_regs) { 3873 assert(_multiple_reads_allowed || register_blocked(reg) == 0, "register already marked as used"); 3874 set_register_blocked(reg, 1); 3875 } 3876 reg = it->assigned_regHi(); 3877 if (reg != LinearScan::any_reg && reg < LinearScan::nof_regs) { 3878 assert(_multiple_reads_allowed || register_blocked(reg) == 0, "register already marked as used"); 3879 set_register_blocked(reg, 1); 3880 } 3881 } 3882 3883 // mark assigned_reg and assigned_regHi of the interval as unblocked 3884 void MoveResolver::unblock_registers(Interval* it) { 3885 int reg = it->assigned_reg(); 3886 if (reg < LinearScan::nof_regs) { 3887 assert(register_blocked(reg) > 0, "register already marked as unused"); 3888 set_register_blocked(reg, -1); 3889 } 3890 reg = it->assigned_regHi(); 3891 if (reg != LinearScan::any_reg && reg < LinearScan::nof_regs) { 3892 assert(register_blocked(reg) > 0, "register already marked as unused"); 3893 set_register_blocked(reg, -1); 3894 } 3895 } 3896 3897 // check if assigned_reg and assigned_regHi of the to-interval are not blocked (or only blocked by from) 3898 bool MoveResolver::save_to_process_move(Interval* from, Interval* to) { 3899 int from_reg = -1; 3900 int from_regHi = -1; 3901 if (from != nullptr) { 3902 from_reg = from->assigned_reg(); 3903 from_regHi = from->assigned_regHi(); 3904 } 3905 3906 int reg = to->assigned_reg(); 3907 if (reg < LinearScan::nof_regs) { 3908 if (register_blocked(reg) > 1 || (register_blocked(reg) == 1 && reg != from_reg && reg != from_regHi)) { 3909 return false; 3910 } 3911 } 3912 reg = to->assigned_regHi(); 3913 if (reg != LinearScan::any_reg && reg < LinearScan::nof_regs) { 3914 if (register_blocked(reg) > 1 || (register_blocked(reg) == 1 && reg != from_reg && reg != from_regHi)) { 3915 return false; 3916 } 3917 } 3918 3919 return true; 3920 } 3921 3922 3923 void MoveResolver::create_insertion_buffer(LIR_List* list) { 3924 assert(!_insertion_buffer.initialized(), "overwriting existing buffer"); 3925 _insertion_buffer.init(list); 3926 } 3927 3928 void MoveResolver::append_insertion_buffer() { 3929 if (_insertion_buffer.initialized()) { 3930 _insertion_buffer.lir_list()->append(&_insertion_buffer); 3931 } 3932 assert(!_insertion_buffer.initialized(), "must be uninitialized now"); 3933 3934 _insert_list = nullptr; 3935 _insert_idx = -1; 3936 } 3937 3938 void MoveResolver::insert_move(Interval* from_interval, Interval* to_interval) { 3939 assert(from_interval->reg_num() != to_interval->reg_num(), "from and to interval equal"); 3940 assert(from_interval->type() == to_interval->type(), "move between different types"); 3941 assert(_insert_list != nullptr && _insert_idx != -1, "must setup insert position first"); 3942 assert(_insertion_buffer.lir_list() == _insert_list, "wrong insertion buffer"); 3943 3944 LIR_Opr from_opr = get_virtual_register(from_interval); 3945 LIR_Opr to_opr = get_virtual_register(to_interval); 3946 3947 if (!_multiple_reads_allowed) { 3948 // the last_use flag is an optimization for FPU stack allocation. When the same 3949 // input interval is used in more than one move, then it is too difficult to determine 3950 // if this move is really the last use. 3951 from_opr = from_opr->make_last_use(); 3952 } 3953 _insertion_buffer.move(_insert_idx, from_opr, to_opr); 3954 3955 TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: inserted move from register %d (%d, %d) to %d (%d, %d)", from_interval->reg_num(), from_interval->assigned_reg(), from_interval->assigned_regHi(), to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi())); 3956 } 3957 3958 void MoveResolver::insert_move(LIR_Opr from_opr, Interval* to_interval) { 3959 assert(from_opr->type() == to_interval->type(), "move between different types"); 3960 assert(_insert_list != nullptr && _insert_idx != -1, "must setup insert position first"); 3961 assert(_insertion_buffer.lir_list() == _insert_list, "wrong insertion buffer"); 3962 3963 LIR_Opr to_opr = get_virtual_register(to_interval); 3964 _insertion_buffer.move(_insert_idx, from_opr, to_opr); 3965 3966 TRACE_LINEAR_SCAN(4, tty->print("MoveResolver: inserted move from constant "); from_opr->print(); tty->print_cr(" to %d (%d, %d)", to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi())); 3967 } 3968 3969 LIR_Opr MoveResolver::get_virtual_register(Interval* interval) { 3970 // Add a little fudge factor for the bailout since the bailout is only checked periodically. This allows us to hand out 3971 // a few extra registers before we really run out which helps to avoid to trip over assertions. 3972 int reg_num = interval->reg_num(); 3973 if (reg_num + 20 >= LIR_Opr::vreg_max) { 3974 _allocator->bailout("out of virtual registers in linear scan"); 3975 if (reg_num + 2 >= LIR_Opr::vreg_max) { 3976 // Wrap it around and continue until bailout really happens to avoid hitting assertions. 3977 reg_num = LIR_Opr::vreg_base; 3978 } 3979 } 3980 LIR_Opr vreg = LIR_OprFact::virtual_register(reg_num, interval->type()); 3981 assert(vreg != LIR_OprFact::illegal(), "ran out of virtual registers"); 3982 return vreg; 3983 } 3984 3985 void MoveResolver::resolve_mappings() { 3986 TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: resolving mappings for Block B%d, index %d", _insert_list->block() != nullptr ? _insert_list->block()->block_id() : -1, _insert_idx)); 3987 DEBUG_ONLY(verify_before_resolve()); 3988 3989 // Block all registers that are used as input operands of a move. 3990 // When a register is blocked, no move to this register is emitted. 3991 // This is necessary for detecting cycles in moves. 3992 int i; 3993 for (i = _mapping_from.length() - 1; i >= 0; i--) { 3994 Interval* from_interval = _mapping_from.at(i); 3995 if (from_interval != nullptr) { 3996 block_registers(from_interval); 3997 } 3998 } 3999 4000 int spill_candidate = -1; 4001 while (_mapping_from.length() > 0) { 4002 bool processed_interval = false; 4003 4004 for (i = _mapping_from.length() - 1; i >= 0; i--) { 4005 Interval* from_interval = _mapping_from.at(i); 4006 Interval* to_interval = _mapping_to.at(i); 4007 4008 if (save_to_process_move(from_interval, to_interval)) { 4009 // this interval can be processed because target is free 4010 if (from_interval != nullptr) { 4011 insert_move(from_interval, to_interval); 4012 unblock_registers(from_interval); 4013 } else { 4014 insert_move(_mapping_from_opr.at(i), to_interval); 4015 } 4016 _mapping_from.remove_at(i); 4017 _mapping_from_opr.remove_at(i); 4018 _mapping_to.remove_at(i); 4019 4020 processed_interval = true; 4021 } else if (from_interval != nullptr && from_interval->assigned_reg() < LinearScan::nof_regs) { 4022 // this interval cannot be processed now because target is not free 4023 // it starts in a register, so it is a possible candidate for spilling 4024 spill_candidate = i; 4025 } 4026 } 4027 4028 if (!processed_interval) { 4029 // no move could be processed because there is a cycle in the move list 4030 // (e.g. r1 -> r2, r2 -> r1), so one interval must be spilled to memory 4031 guarantee(spill_candidate != -1, "no interval in register for spilling found"); 4032 4033 // create a new spill interval and assign a stack slot to it 4034 Interval* from_interval = _mapping_from.at(spill_candidate); 4035 Interval* spill_interval = new Interval(-1); 4036 spill_interval->set_type(from_interval->type()); 4037 4038 // add a dummy range because real position is difficult to calculate 4039 // Note: this range is a special case when the integrity of the allocation is checked 4040 spill_interval->add_range(1, 2); 4041 4042 // do not allocate a new spill slot for temporary interval, but 4043 // use spill slot assigned to from_interval. Otherwise moves from 4044 // one stack slot to another can happen (not allowed by LIR_Assembler 4045 int spill_slot = from_interval->canonical_spill_slot(); 4046 if (spill_slot < 0) { 4047 spill_slot = allocator()->allocate_spill_slot(type2spill_size[spill_interval->type()] == 2); 4048 from_interval->set_canonical_spill_slot(spill_slot); 4049 } 4050 spill_interval->assign_reg(spill_slot); 4051 allocator()->append_interval(spill_interval); 4052 4053 TRACE_LINEAR_SCAN(4, tty->print_cr("created new Interval %d for spilling", spill_interval->reg_num())); 4054 4055 // insert a move from register to stack and update the mapping 4056 insert_move(from_interval, spill_interval); 4057 _mapping_from.at_put(spill_candidate, spill_interval); 4058 unblock_registers(from_interval); 4059 } 4060 } 4061 4062 // reset to default value 4063 _multiple_reads_allowed = false; 4064 4065 // check that all intervals have been processed 4066 DEBUG_ONLY(check_empty()); 4067 } 4068 4069 4070 void MoveResolver::set_insert_position(LIR_List* insert_list, int insert_idx) { 4071 TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: setting insert position to Block B%d, index %d", insert_list->block() != nullptr ? insert_list->block()->block_id() : -1, insert_idx)); 4072 assert(_insert_list == nullptr && _insert_idx == -1, "use move_insert_position instead of set_insert_position when data already set"); 4073 4074 create_insertion_buffer(insert_list); 4075 _insert_list = insert_list; 4076 _insert_idx = insert_idx; 4077 } 4078 4079 void MoveResolver::move_insert_position(LIR_List* insert_list, int insert_idx) { 4080 TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: moving insert position to Block B%d, index %d", insert_list->block() != nullptr ? insert_list->block()->block_id() : -1, insert_idx)); 4081 4082 if (_insert_list != nullptr && (insert_list != _insert_list || insert_idx != _insert_idx)) { 4083 // insert position changed -> resolve current mappings 4084 resolve_mappings(); 4085 } 4086 4087 if (insert_list != _insert_list) { 4088 // block changed -> append insertion_buffer because it is 4089 // bound to a specific block and create a new insertion_buffer 4090 append_insertion_buffer(); 4091 create_insertion_buffer(insert_list); 4092 } 4093 4094 _insert_list = insert_list; 4095 _insert_idx = insert_idx; 4096 } 4097 4098 void MoveResolver::add_mapping(Interval* from_interval, Interval* to_interval) { 4099 TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: adding mapping from %d (%d, %d) to %d (%d, %d)", from_interval->reg_num(), from_interval->assigned_reg(), from_interval->assigned_regHi(), to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi())); 4100 4101 _mapping_from.append(from_interval); 4102 _mapping_from_opr.append(LIR_OprFact::illegalOpr); 4103 _mapping_to.append(to_interval); 4104 } 4105 4106 4107 void MoveResolver::add_mapping(LIR_Opr from_opr, Interval* to_interval) { 4108 TRACE_LINEAR_SCAN(4, tty->print("MoveResolver: adding mapping from "); from_opr->print(); tty->print_cr(" to %d (%d, %d)", to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi())); 4109 assert(from_opr->is_constant(), "only for constants"); 4110 4111 _mapping_from.append(nullptr); 4112 _mapping_from_opr.append(from_opr); 4113 _mapping_to.append(to_interval); 4114 } 4115 4116 void MoveResolver::resolve_and_append_moves() { 4117 if (has_mappings()) { 4118 resolve_mappings(); 4119 } 4120 append_insertion_buffer(); 4121 } 4122 4123 4124 4125 // **** Implementation of Range ************************************* 4126 4127 Range::Range(int from, int to, Range* next) : 4128 _from(from), 4129 _to(to), 4130 _next(next) 4131 { 4132 } 4133 4134 // initialize sentinel 4135 Range* Range::_end = nullptr; 4136 void Range::initialize() { 4137 assert(_end == nullptr, "Range initialized more than once"); 4138 alignas(Range) static uint8_t end_storage[sizeof(Range)]; 4139 _end = ::new(static_cast<void*>(end_storage)) Range(max_jint, max_jint, nullptr); 4140 } 4141 4142 int Range::intersects_at(Range* r2) const { 4143 const Range* r1 = this; 4144 4145 assert(r1 != nullptr && r2 != nullptr, "null ranges not allowed"); 4146 assert(r1 != _end && r2 != _end, "empty ranges not allowed"); 4147 4148 do { 4149 if (r1->from() < r2->from()) { 4150 if (r1->to() <= r2->from()) { 4151 r1 = r1->next(); if (r1 == _end) return -1; 4152 } else { 4153 return r2->from(); 4154 } 4155 } else if (r2->from() < r1->from()) { 4156 if (r2->to() <= r1->from()) { 4157 r2 = r2->next(); if (r2 == _end) return -1; 4158 } else { 4159 return r1->from(); 4160 } 4161 } else { // r1->from() == r2->from() 4162 if (r1->from() == r1->to()) { 4163 r1 = r1->next(); if (r1 == _end) return -1; 4164 } else if (r2->from() == r2->to()) { 4165 r2 = r2->next(); if (r2 == _end) return -1; 4166 } else { 4167 return r1->from(); 4168 } 4169 } 4170 } while (true); 4171 } 4172 4173 #ifndef PRODUCT 4174 void Range::print(outputStream* out) const { 4175 out->print("[%d, %d[ ", _from, _to); 4176 } 4177 #endif 4178 4179 4180 4181 // **** Implementation of Interval ********************************** 4182 4183 // initialize sentinel 4184 Interval* Interval::_end = nullptr; 4185 void Interval::initialize() { 4186 Range::initialize(); 4187 assert(_end == nullptr, "Interval initialized more than once"); 4188 alignas(Interval) static uint8_t end_storage[sizeof(Interval)]; 4189 _end = ::new(static_cast<void*>(end_storage)) Interval(-1); 4190 } 4191 4192 Interval::Interval(int reg_num) : 4193 _reg_num(reg_num), 4194 _type(T_ILLEGAL), 4195 _first(Range::end()), 4196 _use_pos_and_kinds(12), 4197 _current(Range::end()), 4198 _next(_end), 4199 _state(invalidState), 4200 _assigned_reg(LinearScan::any_reg), 4201 _assigned_regHi(LinearScan::any_reg), 4202 _cached_to(-1), 4203 _cached_opr(LIR_OprFact::illegalOpr), 4204 _cached_vm_reg(VMRegImpl::Bad()), 4205 _split_children(nullptr), 4206 _canonical_spill_slot(-1), 4207 _insert_move_when_activated(false), 4208 _spill_state(noDefinitionFound), 4209 _spill_definition_pos(-1), 4210 _register_hint(nullptr) 4211 { 4212 _split_parent = this; 4213 _current_split_child = this; 4214 } 4215 4216 int Interval::calc_to() { 4217 assert(_first != Range::end(), "interval has no range"); 4218 4219 Range* r = _first; 4220 while (r->next() != Range::end()) { 4221 r = r->next(); 4222 } 4223 return r->to(); 4224 } 4225 4226 4227 #ifdef ASSERT 4228 // consistency check of split-children 4229 void Interval::check_split_children() { 4230 if (_split_children != nullptr && _split_children->length() > 0) { 4231 assert(is_split_parent(), "only split parents can have children"); 4232 4233 for (int i = 0; i < _split_children->length(); i++) { 4234 Interval* i1 = _split_children->at(i); 4235 4236 assert(i1->split_parent() == this, "not a split child of this interval"); 4237 assert(i1->type() == type(), "must be equal for all split children"); 4238 assert(i1->canonical_spill_slot() == canonical_spill_slot(), "must be equal for all split children"); 4239 4240 for (int j = i + 1; j < _split_children->length(); j++) { 4241 Interval* i2 = _split_children->at(j); 4242 4243 assert(i1->reg_num() != i2->reg_num(), "same register number"); 4244 4245 if (i1->from() < i2->from()) { 4246 assert(i1->to() <= i2->from() && i1->to() < i2->to(), "intervals overlapping"); 4247 } else { 4248 assert(i2->from() < i1->from(), "intervals start at same op_id"); 4249 assert(i2->to() <= i1->from() && i2->to() < i1->to(), "intervals overlapping"); 4250 } 4251 } 4252 } 4253 } 4254 } 4255 #endif // ASSERT 4256 4257 Interval* Interval::register_hint(bool search_split_child) const { 4258 if (!search_split_child) { 4259 return _register_hint; 4260 } 4261 4262 if (_register_hint != nullptr) { 4263 assert(_register_hint->is_split_parent(), "only split parents are valid hint registers"); 4264 4265 if (_register_hint->assigned_reg() >= 0 && _register_hint->assigned_reg() < LinearScan::nof_regs) { 4266 return _register_hint; 4267 4268 } else if (_register_hint->_split_children != nullptr && _register_hint->_split_children->length() > 0) { 4269 // search the first split child that has a register assigned 4270 int len = _register_hint->_split_children->length(); 4271 for (int i = 0; i < len; i++) { 4272 Interval* cur = _register_hint->_split_children->at(i); 4273 4274 if (cur->assigned_reg() >= 0 && cur->assigned_reg() < LinearScan::nof_regs) { 4275 return cur; 4276 } 4277 } 4278 } 4279 } 4280 4281 // no hint interval found that has a register assigned 4282 return nullptr; 4283 } 4284 4285 4286 Interval* Interval::split_child_at_op_id(int op_id, LIR_OpVisitState::OprMode mode) { 4287 assert(is_split_parent(), "can only be called for split parents"); 4288 assert(op_id >= 0, "invalid op_id (method can not be called for spill moves)"); 4289 4290 Interval* result; 4291 if (_split_children == nullptr || _split_children->length() == 0) { 4292 result = this; 4293 } else { 4294 result = nullptr; 4295 int len = _split_children->length(); 4296 4297 // in outputMode, the end of the interval (op_id == cur->to()) is not valid 4298 int to_offset = (mode == LIR_OpVisitState::outputMode ? 0 : 1); 4299 4300 int i; 4301 for (i = 0; i < len; i++) { 4302 Interval* cur = _split_children->at(i); 4303 if (cur->from() <= op_id && op_id < cur->to() + to_offset) { 4304 if (i > 0) { 4305 // exchange current split child to start of list (faster access for next call) 4306 _split_children->at_put(i, _split_children->at(0)); 4307 _split_children->at_put(0, cur); 4308 } 4309 4310 // interval found 4311 result = cur; 4312 break; 4313 } 4314 } 4315 4316 #ifdef ASSERT 4317 for (i = 0; i < len; i++) { 4318 Interval* tmp = _split_children->at(i); 4319 if (tmp != result && tmp->from() <= op_id && op_id < tmp->to() + to_offset) { 4320 tty->print_cr("two valid result intervals found for op_id %d: %d and %d", op_id, result->reg_num(), tmp->reg_num()); 4321 result->print(); 4322 tmp->print(); 4323 assert(false, "two valid result intervals found"); 4324 } 4325 } 4326 #endif 4327 } 4328 4329 assert(result != nullptr, "no matching interval found"); 4330 assert(result->covers(op_id, mode), "op_id not covered by interval"); 4331 4332 return result; 4333 } 4334 4335 4336 // returns the last split child that ends before the given op_id 4337 Interval* Interval::split_child_before_op_id(int op_id) { 4338 assert(op_id >= 0, "invalid op_id"); 4339 4340 Interval* parent = split_parent(); 4341 Interval* result = nullptr; 4342 4343 assert(parent->_split_children != nullptr, "no split children available"); 4344 int len = parent->_split_children->length(); 4345 assert(len > 0, "no split children available"); 4346 4347 for (int i = len - 1; i >= 0; i--) { 4348 Interval* cur = parent->_split_children->at(i); 4349 if (cur->to() <= op_id && (result == nullptr || result->to() < cur->to())) { 4350 result = cur; 4351 } 4352 } 4353 4354 assert(result != nullptr, "no split child found"); 4355 return result; 4356 } 4357 4358 4359 // Note: use positions are sorted descending -> first use has highest index 4360 int Interval::first_usage(IntervalUseKind min_use_kind) const { 4361 assert(LinearScan::is_virtual_interval(this), "cannot access use positions for fixed intervals"); 4362 4363 for (int i = _use_pos_and_kinds.length() - 2; i >= 0; i -= 2) { 4364 if (_use_pos_and_kinds.at(i + 1) >= min_use_kind) { 4365 return _use_pos_and_kinds.at(i); 4366 } 4367 } 4368 return max_jint; 4369 } 4370 4371 int Interval::next_usage(IntervalUseKind min_use_kind, int from) const { 4372 assert(LinearScan::is_virtual_interval(this), "cannot access use positions for fixed intervals"); 4373 4374 for (int i = _use_pos_and_kinds.length() - 2; i >= 0; i -= 2) { 4375 if (_use_pos_and_kinds.at(i) >= from && _use_pos_and_kinds.at(i + 1) >= min_use_kind) { 4376 return _use_pos_and_kinds.at(i); 4377 } 4378 } 4379 return max_jint; 4380 } 4381 4382 int Interval::next_usage_exact(IntervalUseKind exact_use_kind, int from) const { 4383 assert(LinearScan::is_virtual_interval(this), "cannot access use positions for fixed intervals"); 4384 4385 for (int i = _use_pos_and_kinds.length() - 2; i >= 0; i -= 2) { 4386 if (_use_pos_and_kinds.at(i) >= from && _use_pos_and_kinds.at(i + 1) == exact_use_kind) { 4387 return _use_pos_and_kinds.at(i); 4388 } 4389 } 4390 return max_jint; 4391 } 4392 4393 int Interval::previous_usage(IntervalUseKind min_use_kind, int from) const { 4394 assert(LinearScan::is_virtual_interval(this), "cannot access use positions for fixed intervals"); 4395 4396 int prev = 0; 4397 for (int i = _use_pos_and_kinds.length() - 2; i >= 0; i -= 2) { 4398 if (_use_pos_and_kinds.at(i) > from) { 4399 return prev; 4400 } 4401 if (_use_pos_and_kinds.at(i + 1) >= min_use_kind) { 4402 prev = _use_pos_and_kinds.at(i); 4403 } 4404 } 4405 return prev; 4406 } 4407 4408 void Interval::add_use_pos(int pos, IntervalUseKind use_kind) { 4409 assert(covers(pos, LIR_OpVisitState::inputMode), "use position not covered by live range"); 4410 4411 // do not add use positions for precolored intervals because 4412 // they are never used 4413 if (use_kind != noUse && reg_num() >= LIR_Opr::vreg_base) { 4414 #ifdef ASSERT 4415 assert(_use_pos_and_kinds.length() % 2 == 0, "must be"); 4416 for (int i = 0; i < _use_pos_and_kinds.length(); i += 2) { 4417 assert(pos <= _use_pos_and_kinds.at(i), "already added a use-position with lower position"); 4418 assert(_use_pos_and_kinds.at(i + 1) >= firstValidKind && _use_pos_and_kinds.at(i + 1) <= lastValidKind, "invalid use kind"); 4419 if (i > 0) { 4420 assert(_use_pos_and_kinds.at(i) < _use_pos_and_kinds.at(i - 2), "not sorted descending"); 4421 } 4422 } 4423 #endif 4424 4425 // Note: add_use is called in descending order, so list gets sorted 4426 // automatically by just appending new use positions 4427 int len = _use_pos_and_kinds.length(); 4428 if (len == 0 || _use_pos_and_kinds.at(len - 2) > pos) { 4429 _use_pos_and_kinds.append(pos); 4430 _use_pos_and_kinds.append(use_kind); 4431 } else if (_use_pos_and_kinds.at(len - 1) < use_kind) { 4432 assert(_use_pos_and_kinds.at(len - 2) == pos, "list not sorted correctly"); 4433 _use_pos_and_kinds.at_put(len - 1, use_kind); 4434 } 4435 } 4436 } 4437 4438 void Interval::add_range(int from, int to) { 4439 assert(from < to, "invalid range"); 4440 assert(first() == Range::end() || to < first()->next()->from(), "not inserting at begin of interval"); 4441 assert(from <= first()->to(), "not inserting at begin of interval"); 4442 4443 if (first()->from() <= to) { 4444 // join intersecting ranges 4445 first()->set_from(MIN2(from, first()->from())); 4446 first()->set_to (MAX2(to, first()->to())); 4447 } else { 4448 // insert new range 4449 _first = new Range(from, to, first()); 4450 } 4451 } 4452 4453 Interval* Interval::new_split_child() { 4454 // allocate new interval 4455 Interval* result = new Interval(-1); 4456 result->set_type(type()); 4457 4458 Interval* parent = split_parent(); 4459 result->_split_parent = parent; 4460 result->set_register_hint(parent); 4461 4462 // insert new interval in children-list of parent 4463 if (parent->_split_children == nullptr) { 4464 assert(is_split_parent(), "list must be initialized at first split"); 4465 4466 parent->_split_children = new IntervalList(4); 4467 parent->_split_children->append(this); 4468 } 4469 parent->_split_children->append(result); 4470 4471 return result; 4472 } 4473 4474 // split this interval at the specified position and return 4475 // the remainder as a new interval. 4476 // 4477 // when an interval is split, a bi-directional link is established between the original interval 4478 // (the split parent) and the intervals that are split off this interval (the split children) 4479 // When a split child is split again, the new created interval is also a direct child 4480 // of the original parent (there is no tree of split children stored, but a flat list) 4481 // All split children are spilled to the same stack slot (stored in _canonical_spill_slot) 4482 // 4483 // Note: The new interval has no valid reg_num 4484 Interval* Interval::split(int split_pos) { 4485 assert(LinearScan::is_virtual_interval(this), "cannot split fixed intervals"); 4486 4487 // allocate new interval 4488 Interval* result = new_split_child(); 4489 4490 // split the ranges 4491 Range* prev = nullptr; 4492 Range* cur = _first; 4493 while (cur != Range::end() && cur->to() <= split_pos) { 4494 prev = cur; 4495 cur = cur->next(); 4496 } 4497 assert(cur != Range::end(), "split interval after end of last range"); 4498 4499 if (cur->from() < split_pos) { 4500 result->_first = new Range(split_pos, cur->to(), cur->next()); 4501 cur->set_to(split_pos); 4502 cur->set_next(Range::end()); 4503 4504 } else { 4505 assert(prev != nullptr, "split before start of first range"); 4506 result->_first = cur; 4507 prev->set_next(Range::end()); 4508 } 4509 result->_current = result->_first; 4510 _cached_to = -1; // clear cached value 4511 4512 // split list of use positions 4513 int total_len = _use_pos_and_kinds.length(); 4514 int start_idx = total_len - 2; 4515 while (start_idx >= 0 && _use_pos_and_kinds.at(start_idx) < split_pos) { 4516 start_idx -= 2; 4517 } 4518 4519 intStack new_use_pos_and_kinds(total_len - start_idx); 4520 int i; 4521 for (i = start_idx + 2; i < total_len; i++) { 4522 new_use_pos_and_kinds.append(_use_pos_and_kinds.at(i)); 4523 } 4524 4525 _use_pos_and_kinds.trunc_to(start_idx + 2); 4526 result->_use_pos_and_kinds = _use_pos_and_kinds; 4527 _use_pos_and_kinds = new_use_pos_and_kinds; 4528 4529 #ifdef ASSERT 4530 assert(_use_pos_and_kinds.length() % 2 == 0, "must have use kind for each use pos"); 4531 assert(result->_use_pos_and_kinds.length() % 2 == 0, "must have use kind for each use pos"); 4532 assert(_use_pos_and_kinds.length() + result->_use_pos_and_kinds.length() == total_len, "missed some entries"); 4533 4534 for (i = 0; i < _use_pos_and_kinds.length(); i += 2) { 4535 assert(_use_pos_and_kinds.at(i) < split_pos, "must be"); 4536 assert(_use_pos_and_kinds.at(i + 1) >= firstValidKind && _use_pos_and_kinds.at(i + 1) <= lastValidKind, "invalid use kind"); 4537 } 4538 for (i = 0; i < result->_use_pos_and_kinds.length(); i += 2) { 4539 assert(result->_use_pos_and_kinds.at(i) >= split_pos, "must be"); 4540 assert(result->_use_pos_and_kinds.at(i + 1) >= firstValidKind && result->_use_pos_and_kinds.at(i + 1) <= lastValidKind, "invalid use kind"); 4541 } 4542 #endif 4543 4544 return result; 4545 } 4546 4547 // split this interval at the specified position and return 4548 // the head as a new interval (the original interval is the tail) 4549 // 4550 // Currently, only the first range can be split, and the new interval 4551 // must not have split positions 4552 Interval* Interval::split_from_start(int split_pos) { 4553 assert(LinearScan::is_virtual_interval(this), "cannot split fixed intervals"); 4554 assert(split_pos > from() && split_pos < to(), "can only split inside interval"); 4555 assert(split_pos > _first->from() && split_pos <= _first->to(), "can only split inside first range"); 4556 assert(first_usage(noUse) > split_pos, "can not split when use positions are present"); 4557 4558 // allocate new interval 4559 Interval* result = new_split_child(); 4560 4561 // the new created interval has only one range (checked by assertion above), 4562 // so the splitting of the ranges is very simple 4563 result->add_range(_first->from(), split_pos); 4564 4565 if (split_pos == _first->to()) { 4566 assert(_first->next() != Range::end(), "must not be at end"); 4567 _first = _first->next(); 4568 } else { 4569 _first->set_from(split_pos); 4570 } 4571 4572 return result; 4573 } 4574 4575 4576 // returns true if the op_id is inside the interval 4577 bool Interval::covers(int op_id, LIR_OpVisitState::OprMode mode) const { 4578 Range* cur = _first; 4579 4580 while (cur != Range::end() && cur->to() < op_id) { 4581 cur = cur->next(); 4582 } 4583 if (cur != Range::end()) { 4584 assert(cur->to() != cur->next()->from(), "ranges not separated"); 4585 4586 if (mode == LIR_OpVisitState::outputMode) { 4587 return cur->from() <= op_id && op_id < cur->to(); 4588 } else { 4589 return cur->from() <= op_id && op_id <= cur->to(); 4590 } 4591 } 4592 return false; 4593 } 4594 4595 // returns true if the interval has any hole between hole_from and hole_to 4596 // (even if the hole has only the length 1) 4597 bool Interval::has_hole_between(int hole_from, int hole_to) { 4598 assert(hole_from < hole_to, "check"); 4599 assert(from() <= hole_from && hole_to <= to(), "index out of interval"); 4600 4601 Range* cur = _first; 4602 while (cur != Range::end()) { 4603 assert(cur->to() < cur->next()->from(), "no space between ranges"); 4604 4605 // hole-range starts before this range -> hole 4606 if (hole_from < cur->from()) { 4607 return true; 4608 4609 // hole-range completely inside this range -> no hole 4610 } else if (hole_to <= cur->to()) { 4611 return false; 4612 4613 // overlapping of hole-range with this range -> hole 4614 } else if (hole_from <= cur->to()) { 4615 return true; 4616 } 4617 4618 cur = cur->next(); 4619 } 4620 4621 return false; 4622 } 4623 4624 // Check if there is an intersection with any of the split children of 'interval' 4625 bool Interval::intersects_any_children_of(Interval* interval) const { 4626 if (interval->_split_children != nullptr) { 4627 for (int i = 0; i < interval->_split_children->length(); i++) { 4628 if (intersects(interval->_split_children->at(i))) { 4629 return true; 4630 } 4631 } 4632 } 4633 return false; 4634 } 4635 4636 4637 #ifndef PRODUCT 4638 void Interval::print_on(outputStream* out, bool is_cfg_printer) const { 4639 const char* SpillState2Name[] = { "no definition", "no spill store", "one spill store", "store at definition", "start in memory", "no optimization" }; 4640 const char* UseKind2Name[] = { "N", "L", "S", "M" }; 4641 4642 const char* type_name; 4643 if (reg_num() < LIR_Opr::vreg_base) { 4644 type_name = "fixed"; 4645 } else { 4646 type_name = type2name(type()); 4647 } 4648 out->print("%d %s ", reg_num(), type_name); 4649 4650 if (is_cfg_printer) { 4651 // Special version for compatibility with C1 Visualizer. 4652 LIR_Opr opr = LinearScan::get_operand(reg_num()); 4653 if (opr->is_valid()) { 4654 out->print("\""); 4655 opr->print(out); 4656 out->print("\" "); 4657 } 4658 } else { 4659 // Improved output for normal debugging. 4660 if (reg_num() < LIR_Opr::vreg_base) { 4661 LinearScan::print_reg_num(out, assigned_reg()); 4662 } else if (assigned_reg() != -1 && (LinearScan::num_physical_regs(type()) == 1 || assigned_regHi() != -1)) { 4663 LinearScan::calc_operand_for_interval(this)->print(out); 4664 } else { 4665 // Virtual register that has no assigned register yet. 4666 out->print("[ANY]"); 4667 } 4668 out->print(" "); 4669 } 4670 out->print("%d %d ", split_parent()->reg_num(), (register_hint(false) != nullptr ? register_hint(false)->reg_num() : -1)); 4671 4672 // print ranges 4673 Range* cur = _first; 4674 while (cur != Range::end()) { 4675 cur->print(out); 4676 cur = cur->next(); 4677 assert(cur != nullptr, "range list not closed with range sentinel"); 4678 } 4679 4680 // print use positions 4681 int prev = 0; 4682 assert(_use_pos_and_kinds.length() % 2 == 0, "must be"); 4683 for (int i =_use_pos_and_kinds.length() - 2; i >= 0; i -= 2) { 4684 assert(_use_pos_and_kinds.at(i + 1) >= firstValidKind && _use_pos_and_kinds.at(i + 1) <= lastValidKind, "invalid use kind"); 4685 assert(prev < _use_pos_and_kinds.at(i), "use positions not sorted"); 4686 4687 out->print("%d %s ", _use_pos_and_kinds.at(i), UseKind2Name[_use_pos_and_kinds.at(i + 1)]); 4688 prev = _use_pos_and_kinds.at(i); 4689 } 4690 4691 out->print(" \"%s\"", SpillState2Name[spill_state()]); 4692 out->cr(); 4693 } 4694 4695 void Interval::print_parent() const { 4696 if (_split_parent != this) { 4697 _split_parent->print_on(tty); 4698 } else { 4699 tty->print_cr("Parent: this"); 4700 } 4701 } 4702 4703 void Interval::print_children() const { 4704 if (_split_children == nullptr) { 4705 tty->print_cr("Children: []"); 4706 } else { 4707 tty->print_cr("Children:"); 4708 for (int i = 0; i < _split_children->length(); i++) { 4709 tty->print("%d: ", i); 4710 _split_children->at(i)->print_on(tty); 4711 } 4712 } 4713 } 4714 #endif // NOT PRODUCT 4715 4716 4717 4718 4719 // **** Implementation of IntervalWalker **************************** 4720 4721 IntervalWalker::IntervalWalker(LinearScan* allocator, Interval* unhandled_fixed_first, Interval* unhandled_any_first) 4722 : _compilation(allocator->compilation()) 4723 , _allocator(allocator) 4724 { 4725 _unhandled_first[fixedKind] = unhandled_fixed_first; 4726 _unhandled_first[anyKind] = unhandled_any_first; 4727 _active_first[fixedKind] = Interval::end(); 4728 _inactive_first[fixedKind] = Interval::end(); 4729 _active_first[anyKind] = Interval::end(); 4730 _inactive_first[anyKind] = Interval::end(); 4731 _current_position = -1; 4732 _current = nullptr; 4733 next_interval(); 4734 } 4735 4736 4737 // append interval in order of current range from() 4738 void IntervalWalker::append_sorted(Interval** list, Interval* interval) { 4739 Interval* prev = nullptr; 4740 Interval* cur = *list; 4741 while (cur->current_from() < interval->current_from()) { 4742 prev = cur; cur = cur->next(); 4743 } 4744 if (prev == nullptr) { 4745 *list = interval; 4746 } else { 4747 prev->set_next(interval); 4748 } 4749 interval->set_next(cur); 4750 } 4751 4752 void IntervalWalker::append_to_unhandled(Interval** list, Interval* interval) { 4753 assert(interval->from() >= current()->current_from(), "cannot append new interval before current walk position"); 4754 4755 Interval* prev = nullptr; 4756 Interval* cur = *list; 4757 while (cur->from() < interval->from() || (cur->from() == interval->from() && cur->first_usage(noUse) < interval->first_usage(noUse))) { 4758 prev = cur; cur = cur->next(); 4759 } 4760 if (prev == nullptr) { 4761 *list = interval; 4762 } else { 4763 prev->set_next(interval); 4764 } 4765 interval->set_next(cur); 4766 } 4767 4768 4769 inline bool IntervalWalker::remove_from_list(Interval** list, Interval* i) { 4770 while (*list != Interval::end() && *list != i) { 4771 list = (*list)->next_addr(); 4772 } 4773 if (*list != Interval::end()) { 4774 assert(*list == i, "check"); 4775 *list = (*list)->next(); 4776 return true; 4777 } else { 4778 return false; 4779 } 4780 } 4781 4782 void IntervalWalker::remove_from_list(Interval* i) { 4783 bool deleted; 4784 4785 if (i->state() == activeState) { 4786 deleted = remove_from_list(active_first_addr(anyKind), i); 4787 } else { 4788 assert(i->state() == inactiveState, "invalid state"); 4789 deleted = remove_from_list(inactive_first_addr(anyKind), i); 4790 } 4791 4792 assert(deleted, "interval has not been found in list"); 4793 } 4794 4795 4796 void IntervalWalker::walk_to(IntervalState state, int from) { 4797 assert (state == activeState || state == inactiveState, "wrong state"); 4798 for_each_interval_kind(kind) { 4799 Interval** prev = state == activeState ? active_first_addr(kind) : inactive_first_addr(kind); 4800 Interval* next = *prev; 4801 while (next->current_from() <= from) { 4802 Interval* cur = next; 4803 next = cur->next(); 4804 4805 bool range_has_changed = false; 4806 while (cur->current_to() <= from) { 4807 cur->next_range(); 4808 range_has_changed = true; 4809 } 4810 4811 // also handle move from inactive list to active list 4812 range_has_changed = range_has_changed || (state == inactiveState && cur->current_from() <= from); 4813 4814 if (range_has_changed) { 4815 // remove cur from list 4816 *prev = next; 4817 if (cur->current_at_end()) { 4818 // move to handled state (not maintained as a list) 4819 cur->set_state(handledState); 4820 DEBUG_ONLY(interval_moved(cur, kind, state, handledState);) 4821 } else if (cur->current_from() <= from){ 4822 // sort into active list 4823 append_sorted(active_first_addr(kind), cur); 4824 cur->set_state(activeState); 4825 if (*prev == cur) { 4826 assert(state == activeState, "check"); 4827 prev = cur->next_addr(); 4828 } 4829 DEBUG_ONLY(interval_moved(cur, kind, state, activeState);) 4830 } else { 4831 // sort into inactive list 4832 append_sorted(inactive_first_addr(kind), cur); 4833 cur->set_state(inactiveState); 4834 if (*prev == cur) { 4835 assert(state == inactiveState, "check"); 4836 prev = cur->next_addr(); 4837 } 4838 DEBUG_ONLY(interval_moved(cur, kind, state, inactiveState);) 4839 } 4840 } else { 4841 prev = cur->next_addr(); 4842 continue; 4843 } 4844 } 4845 } 4846 } 4847 4848 4849 void IntervalWalker::next_interval() { 4850 IntervalKind kind; 4851 Interval* any = _unhandled_first[anyKind]; 4852 Interval* fixed = _unhandled_first[fixedKind]; 4853 4854 if (any != Interval::end()) { 4855 // intervals may start at same position -> prefer fixed interval 4856 kind = fixed != Interval::end() && fixed->from() <= any->from() ? fixedKind : anyKind; 4857 4858 assert (kind == fixedKind && fixed->from() <= any->from() || 4859 kind == anyKind && any->from() <= fixed->from(), "wrong interval!!!"); 4860 assert(any == Interval::end() || fixed == Interval::end() || any->from() != fixed->from() || kind == fixedKind, "if fixed and any-Interval start at same position, fixed must be processed first"); 4861 4862 } else if (fixed != Interval::end()) { 4863 kind = fixedKind; 4864 } else { 4865 _current = nullptr; return; 4866 } 4867 _current_kind = kind; 4868 _current = _unhandled_first[kind]; 4869 _unhandled_first[kind] = _current->next(); 4870 _current->set_next(Interval::end()); 4871 _current->rewind_range(); 4872 } 4873 4874 4875 void IntervalWalker::walk_to(int lir_op_id) { 4876 assert(_current_position <= lir_op_id, "can not walk backwards"); 4877 while (current() != nullptr) { 4878 bool is_active = current()->from() <= lir_op_id; 4879 int id = is_active ? current()->from() : lir_op_id; 4880 4881 TRACE_LINEAR_SCAN(2, if (_current_position < id) { tty->cr(); tty->print_cr("walk_to(%d) **************************************************************", id); }) 4882 4883 // set _current_position prior to call of walk_to 4884 _current_position = id; 4885 4886 // call walk_to even if _current_position == id 4887 walk_to(activeState, id); 4888 walk_to(inactiveState, id); 4889 4890 if (is_active) { 4891 current()->set_state(activeState); 4892 if (activate_current()) { 4893 append_sorted(active_first_addr(current_kind()), current()); 4894 DEBUG_ONLY(interval_moved(current(), current_kind(), unhandledState, activeState);) 4895 } 4896 4897 next_interval(); 4898 } else { 4899 return; 4900 } 4901 } 4902 } 4903 4904 #ifdef ASSERT 4905 void IntervalWalker::interval_moved(Interval* interval, IntervalKind kind, IntervalState from, IntervalState to) { 4906 if (TraceLinearScanLevel >= 4) { 4907 #define print_state(state) \ 4908 switch(state) {\ 4909 case unhandledState: tty->print("unhandled"); break;\ 4910 case activeState: tty->print("active"); break;\ 4911 case inactiveState: tty->print("inactive"); break;\ 4912 case handledState: tty->print("handled"); break;\ 4913 default: ShouldNotReachHere(); \ 4914 } 4915 4916 print_state(from); tty->print(" to "); print_state(to); 4917 tty->fill_to(23); 4918 interval->print(); 4919 4920 #undef print_state 4921 } 4922 } 4923 #endif // ASSERT 4924 4925 // **** Implementation of LinearScanWalker ************************** 4926 4927 LinearScanWalker::LinearScanWalker(LinearScan* allocator, Interval* unhandled_fixed_first, Interval* unhandled_any_first) 4928 : IntervalWalker(allocator, unhandled_fixed_first, unhandled_any_first) 4929 , _move_resolver(allocator) 4930 { 4931 for (int i = 0; i < LinearScan::nof_regs; i++) { 4932 _spill_intervals[i] = new IntervalList(2); 4933 } 4934 } 4935 4936 4937 inline void LinearScanWalker::init_use_lists(bool only_process_use_pos) { 4938 for (int i = _first_reg; i <= _last_reg; i++) { 4939 _use_pos[i] = max_jint; 4940 4941 if (!only_process_use_pos) { 4942 _block_pos[i] = max_jint; 4943 _spill_intervals[i]->clear(); 4944 } 4945 } 4946 } 4947 4948 inline void LinearScanWalker::exclude_from_use(int reg) { 4949 assert(reg < LinearScan::nof_regs, "interval must have a register assigned (stack slots not allowed)"); 4950 if (reg >= _first_reg && reg <= _last_reg) { 4951 _use_pos[reg] = 0; 4952 } 4953 } 4954 inline void LinearScanWalker::exclude_from_use(Interval* i) { 4955 assert(i->assigned_reg() != any_reg, "interval has no register assigned"); 4956 4957 exclude_from_use(i->assigned_reg()); 4958 exclude_from_use(i->assigned_regHi()); 4959 } 4960 4961 inline void LinearScanWalker::set_use_pos(int reg, Interval* i, int use_pos, bool only_process_use_pos) { 4962 assert(use_pos != 0, "must use exclude_from_use to set use_pos to 0"); 4963 4964 if (reg >= _first_reg && reg <= _last_reg) { 4965 if (_use_pos[reg] > use_pos) { 4966 _use_pos[reg] = use_pos; 4967 } 4968 if (!only_process_use_pos) { 4969 _spill_intervals[reg]->append(i); 4970 } 4971 } 4972 } 4973 inline void LinearScanWalker::set_use_pos(Interval* i, int use_pos, bool only_process_use_pos) { 4974 assert(i->assigned_reg() != any_reg, "interval has no register assigned"); 4975 if (use_pos != -1) { 4976 set_use_pos(i->assigned_reg(), i, use_pos, only_process_use_pos); 4977 set_use_pos(i->assigned_regHi(), i, use_pos, only_process_use_pos); 4978 } 4979 } 4980 4981 inline void LinearScanWalker::set_block_pos(int reg, Interval* i, int block_pos) { 4982 if (reg >= _first_reg && reg <= _last_reg) { 4983 if (_block_pos[reg] > block_pos) { 4984 _block_pos[reg] = block_pos; 4985 } 4986 if (_use_pos[reg] > block_pos) { 4987 _use_pos[reg] = block_pos; 4988 } 4989 } 4990 } 4991 inline void LinearScanWalker::set_block_pos(Interval* i, int block_pos) { 4992 assert(i->assigned_reg() != any_reg, "interval has no register assigned"); 4993 if (block_pos != -1) { 4994 set_block_pos(i->assigned_reg(), i, block_pos); 4995 set_block_pos(i->assigned_regHi(), i, block_pos); 4996 } 4997 } 4998 4999 5000 void LinearScanWalker::free_exclude_active_fixed() { 5001 Interval* list = active_first(fixedKind); 5002 while (list != Interval::end()) { 5003 assert(list->assigned_reg() < LinearScan::nof_regs, "active interval must have a register assigned"); 5004 exclude_from_use(list); 5005 list = list->next(); 5006 } 5007 } 5008 5009 void LinearScanWalker::free_exclude_active_any() { 5010 Interval* list = active_first(anyKind); 5011 while (list != Interval::end()) { 5012 exclude_from_use(list); 5013 list = list->next(); 5014 } 5015 } 5016 5017 void LinearScanWalker::free_collect_inactive_fixed(Interval* cur) { 5018 Interval* list = inactive_first(fixedKind); 5019 while (list != Interval::end()) { 5020 if (cur->to() <= list->current_from()) { 5021 assert(list->current_intersects_at(cur) == -1, "must not intersect"); 5022 set_use_pos(list, list->current_from(), true); 5023 } else { 5024 set_use_pos(list, list->current_intersects_at(cur), true); 5025 } 5026 list = list->next(); 5027 } 5028 } 5029 5030 void LinearScanWalker::free_collect_inactive_any(Interval* cur) { 5031 Interval* list = inactive_first(anyKind); 5032 while (list != Interval::end()) { 5033 set_use_pos(list, list->current_intersects_at(cur), true); 5034 list = list->next(); 5035 } 5036 } 5037 5038 void LinearScanWalker::spill_exclude_active_fixed() { 5039 Interval* list = active_first(fixedKind); 5040 while (list != Interval::end()) { 5041 exclude_from_use(list); 5042 list = list->next(); 5043 } 5044 } 5045 5046 void LinearScanWalker::spill_block_inactive_fixed(Interval* cur) { 5047 Interval* list = inactive_first(fixedKind); 5048 while (list != Interval::end()) { 5049 if (cur->to() > list->current_from()) { 5050 set_block_pos(list, list->current_intersects_at(cur)); 5051 } else { 5052 assert(list->current_intersects_at(cur) == -1, "invalid optimization: intervals intersect"); 5053 } 5054 5055 list = list->next(); 5056 } 5057 } 5058 5059 void LinearScanWalker::spill_collect_active_any() { 5060 Interval* list = active_first(anyKind); 5061 while (list != Interval::end()) { 5062 set_use_pos(list, MIN2(list->next_usage(loopEndMarker, _current_position), list->to()), false); 5063 list = list->next(); 5064 } 5065 } 5066 5067 void LinearScanWalker::spill_collect_inactive_any(Interval* cur) { 5068 Interval* list = inactive_first(anyKind); 5069 while (list != Interval::end()) { 5070 if (list->current_intersects(cur)) { 5071 set_use_pos(list, MIN2(list->next_usage(loopEndMarker, _current_position), list->to()), false); 5072 } 5073 list = list->next(); 5074 } 5075 } 5076 5077 5078 void LinearScanWalker::insert_move(int op_id, Interval* src_it, Interval* dst_it) { 5079 // output all moves here. When source and target are equal, the move is 5080 // optimized away later in assign_reg_nums 5081 5082 op_id = (op_id + 1) & ~1; 5083 BlockBegin* op_block = allocator()->block_of_op_with_id(op_id); 5084 assert(op_id > 0 && allocator()->block_of_op_with_id(op_id - 2) == op_block, "cannot insert move at block boundary"); 5085 5086 // calculate index of instruction inside instruction list of current block 5087 // the minimal index (for a block with no spill moves) can be calculated because the 5088 // numbering of instructions is known. 5089 // When the block already contains spill moves, the index must be increased until the 5090 // correct index is reached. 5091 LIR_OpList* list = op_block->lir()->instructions_list(); 5092 int index = (op_id - list->at(0)->id()) / 2; 5093 assert(list->at(index)->id() <= op_id, "error in calculation"); 5094 5095 while (list->at(index)->id() != op_id) { 5096 index++; 5097 assert(0 <= index && index < list->length(), "index out of bounds"); 5098 } 5099 assert(1 <= index && index < list->length(), "index out of bounds"); 5100 assert(list->at(index)->id() == op_id, "error in calculation"); 5101 5102 // insert new instruction before instruction at position index 5103 _move_resolver.move_insert_position(op_block->lir(), index - 1); 5104 _move_resolver.add_mapping(src_it, dst_it); 5105 } 5106 5107 5108 int LinearScanWalker::find_optimal_split_pos(BlockBegin* min_block, BlockBegin* max_block, int max_split_pos) { 5109 int from_block_nr = min_block->linear_scan_number(); 5110 int to_block_nr = max_block->linear_scan_number(); 5111 5112 assert(0 <= from_block_nr && from_block_nr < block_count(), "out of range"); 5113 assert(0 <= to_block_nr && to_block_nr < block_count(), "out of range"); 5114 assert(from_block_nr < to_block_nr, "must cross block boundary"); 5115 5116 // Try to split at end of max_block. If this would be after 5117 // max_split_pos, then use the begin of max_block 5118 int optimal_split_pos = max_block->last_lir_instruction_id() + 2; 5119 if (optimal_split_pos > max_split_pos) { 5120 optimal_split_pos = max_block->first_lir_instruction_id(); 5121 } 5122 5123 int min_loop_depth = max_block->loop_depth(); 5124 for (int i = to_block_nr - 1; i >= from_block_nr; i--) { 5125 BlockBegin* cur = block_at(i); 5126 5127 if (cur->loop_depth() < min_loop_depth) { 5128 // block with lower loop-depth found -> split at the end of this block 5129 min_loop_depth = cur->loop_depth(); 5130 optimal_split_pos = cur->last_lir_instruction_id() + 2; 5131 } 5132 } 5133 assert(optimal_split_pos > allocator()->max_lir_op_id() || allocator()->is_block_begin(optimal_split_pos), "algorithm must move split pos to block boundary"); 5134 5135 return optimal_split_pos; 5136 } 5137 5138 5139 int LinearScanWalker::find_optimal_split_pos(Interval* it, int min_split_pos, int max_split_pos, bool do_loop_optimization) { 5140 int optimal_split_pos = -1; 5141 if (min_split_pos == max_split_pos) { 5142 // trivial case, no optimization of split position possible 5143 TRACE_LINEAR_SCAN(4, tty->print_cr(" min-pos and max-pos are equal, no optimization possible")); 5144 optimal_split_pos = min_split_pos; 5145 5146 } else { 5147 assert(min_split_pos < max_split_pos, "must be true then"); 5148 assert(min_split_pos > 0, "cannot access min_split_pos - 1 otherwise"); 5149 5150 // reason for using min_split_pos - 1: when the minimal split pos is exactly at the 5151 // beginning of a block, then min_split_pos is also a possible split position. 5152 // Use the block before as min_block, because then min_block->last_lir_instruction_id() + 2 == min_split_pos 5153 BlockBegin* min_block = allocator()->block_of_op_with_id(min_split_pos - 1); 5154 5155 // reason for using max_split_pos - 1: otherwise there would be an assertion failure 5156 // when an interval ends at the end of the last block of the method 5157 // (in this case, max_split_pos == allocator()->max_lir_op_id() + 2, and there is no 5158 // block at this op_id) 5159 BlockBegin* max_block = allocator()->block_of_op_with_id(max_split_pos - 1); 5160 5161 assert(min_block->linear_scan_number() <= max_block->linear_scan_number(), "invalid order"); 5162 if (min_block == max_block) { 5163 // split position cannot be moved to block boundary, so split as late as possible 5164 TRACE_LINEAR_SCAN(4, tty->print_cr(" cannot move split pos to block boundary because min_pos and max_pos are in same block")); 5165 optimal_split_pos = max_split_pos; 5166 5167 } else if (it->has_hole_between(max_split_pos - 1, max_split_pos) && !allocator()->is_block_begin(max_split_pos)) { 5168 // Do not move split position if the interval has a hole before max_split_pos. 5169 // Intervals resulting from Phi-Functions have more than one definition (marked 5170 // as mustHaveRegister) with a hole before each definition. When the register is needed 5171 // for the second definition, an earlier reloading is unnecessary. 5172 TRACE_LINEAR_SCAN(4, tty->print_cr(" interval has hole just before max_split_pos, so splitting at max_split_pos")); 5173 optimal_split_pos = max_split_pos; 5174 5175 } else { 5176 // search optimal block boundary between min_split_pos and max_split_pos 5177 TRACE_LINEAR_SCAN(4, tty->print_cr(" moving split pos to optimal block boundary between block B%d and B%d", min_block->block_id(), max_block->block_id())); 5178 5179 if (do_loop_optimization) { 5180 // Loop optimization: if a loop-end marker is found between min- and max-position, 5181 // then split before this loop 5182 int loop_end_pos = it->next_usage_exact(loopEndMarker, min_block->last_lir_instruction_id() + 2); 5183 TRACE_LINEAR_SCAN(4, tty->print_cr(" loop optimization: loop end found at pos %d", loop_end_pos)); 5184 5185 assert(loop_end_pos > min_split_pos, "invalid order"); 5186 if (loop_end_pos < max_split_pos) { 5187 // loop-end marker found between min- and max-position 5188 // if it is not the end marker for the same loop as the min-position, then move 5189 // the max-position to this loop block. 5190 // Desired result: uses tagged as shouldHaveRegister inside a loop cause a reloading 5191 // of the interval (normally, only mustHaveRegister causes a reloading) 5192 BlockBegin* loop_block = allocator()->block_of_op_with_id(loop_end_pos); 5193 5194 TRACE_LINEAR_SCAN(4, tty->print_cr(" interval is used in loop that ends in block B%d, so trying to move max_block back from B%d to B%d", loop_block->block_id(), max_block->block_id(), loop_block->block_id())); 5195 assert(loop_block != min_block, "loop_block and min_block must be different because block boundary is needed between"); 5196 5197 optimal_split_pos = find_optimal_split_pos(min_block, loop_block, loop_block->last_lir_instruction_id() + 2); 5198 if (optimal_split_pos == loop_block->last_lir_instruction_id() + 2) { 5199 optimal_split_pos = -1; 5200 TRACE_LINEAR_SCAN(4, tty->print_cr(" loop optimization not necessary")); 5201 } else { 5202 TRACE_LINEAR_SCAN(4, tty->print_cr(" loop optimization successful")); 5203 } 5204 } 5205 } 5206 5207 if (optimal_split_pos == -1) { 5208 // not calculated by loop optimization 5209 optimal_split_pos = find_optimal_split_pos(min_block, max_block, max_split_pos); 5210 } 5211 } 5212 } 5213 TRACE_LINEAR_SCAN(4, tty->print_cr(" optimal split position: %d", optimal_split_pos)); 5214 5215 return optimal_split_pos; 5216 } 5217 5218 5219 /* 5220 split an interval at the optimal position between min_split_pos and 5221 max_split_pos in two parts: 5222 1) the left part has already a location assigned 5223 2) the right part is sorted into to the unhandled-list 5224 */ 5225 void LinearScanWalker::split_before_usage(Interval* it, int min_split_pos, int max_split_pos) { 5226 TRACE_LINEAR_SCAN(2, tty->print ("----- splitting interval: "); it->print()); 5227 TRACE_LINEAR_SCAN(2, tty->print_cr(" between %d and %d", min_split_pos, max_split_pos)); 5228 5229 assert(it->from() < min_split_pos, "cannot split at start of interval"); 5230 assert(current_position() < min_split_pos, "cannot split before current position"); 5231 assert(min_split_pos <= max_split_pos, "invalid order"); 5232 assert(max_split_pos <= it->to(), "cannot split after end of interval"); 5233 5234 int optimal_split_pos = find_optimal_split_pos(it, min_split_pos, max_split_pos, true); 5235 5236 assert(min_split_pos <= optimal_split_pos && optimal_split_pos <= max_split_pos, "out of range"); 5237 assert(optimal_split_pos <= it->to(), "cannot split after end of interval"); 5238 assert(optimal_split_pos > it->from(), "cannot split at start of interval"); 5239 5240 if (optimal_split_pos == it->to() && it->next_usage(mustHaveRegister, min_split_pos) == max_jint) { 5241 // the split position would be just before the end of the interval 5242 // -> no split at all necessary 5243 TRACE_LINEAR_SCAN(4, tty->print_cr(" no split necessary because optimal split position is at end of interval")); 5244 return; 5245 } 5246 5247 // must calculate this before the actual split is performed and before split position is moved to odd op_id 5248 bool move_necessary = !allocator()->is_block_begin(optimal_split_pos) && !it->has_hole_between(optimal_split_pos - 1, optimal_split_pos); 5249 5250 if (!allocator()->is_block_begin(optimal_split_pos)) { 5251 // move position before actual instruction (odd op_id) 5252 optimal_split_pos = (optimal_split_pos - 1) | 1; 5253 } 5254 5255 TRACE_LINEAR_SCAN(4, tty->print_cr(" splitting at position %d", optimal_split_pos)); 5256 assert(allocator()->is_block_begin(optimal_split_pos) || (optimal_split_pos % 2 == 1), "split pos must be odd when not on block boundary"); 5257 assert(!allocator()->is_block_begin(optimal_split_pos) || (optimal_split_pos % 2 == 0), "split pos must be even on block boundary"); 5258 5259 Interval* split_part = it->split(optimal_split_pos); 5260 5261 allocator()->append_interval(split_part); 5262 allocator()->copy_register_flags(it, split_part); 5263 split_part->set_insert_move_when_activated(move_necessary); 5264 append_to_unhandled(unhandled_first_addr(anyKind), split_part); 5265 5266 TRACE_LINEAR_SCAN(2, tty->print_cr(" split interval in two parts (insert_move_when_activated: %d)", move_necessary)); 5267 TRACE_LINEAR_SCAN(2, tty->print (" "); it->print()); 5268 TRACE_LINEAR_SCAN(2, tty->print (" "); split_part->print()); 5269 } 5270 5271 /* 5272 split an interval at the optimal position between min_split_pos and 5273 max_split_pos in two parts: 5274 1) the left part has already a location assigned 5275 2) the right part is always on the stack and therefore ignored in further processing 5276 */ 5277 void LinearScanWalker::split_for_spilling(Interval* it) { 5278 // calculate allowed range of splitting position 5279 int max_split_pos = current_position(); 5280 int min_split_pos = MAX2(it->previous_usage(shouldHaveRegister, max_split_pos) + 1, it->from()); 5281 5282 TRACE_LINEAR_SCAN(2, tty->print ("----- splitting and spilling interval: "); it->print()); 5283 TRACE_LINEAR_SCAN(2, tty->print_cr(" between %d and %d", min_split_pos, max_split_pos)); 5284 5285 assert(it->state() == activeState, "why spill interval that is not active?"); 5286 assert(it->from() <= min_split_pos, "cannot split before start of interval"); 5287 assert(min_split_pos <= max_split_pos, "invalid order"); 5288 assert(max_split_pos < it->to(), "cannot split at end end of interval"); 5289 assert(current_position() < it->to(), "interval must not end before current position"); 5290 5291 if (min_split_pos == it->from()) { 5292 // the whole interval is never used, so spill it entirely to memory 5293 TRACE_LINEAR_SCAN(2, tty->print_cr(" spilling entire interval because split pos is at beginning of interval")); 5294 assert(it->first_usage(shouldHaveRegister) > current_position(), "interval must not have use position before current_position"); 5295 5296 allocator()->assign_spill_slot(it); 5297 allocator()->change_spill_state(it, min_split_pos); 5298 5299 // Also kick parent intervals out of register to memory when they have no use 5300 // position. This avoids short interval in register surrounded by intervals in 5301 // memory -> avoid useless moves from memory to register and back 5302 Interval* parent = it; 5303 while (parent != nullptr && parent->is_split_child()) { 5304 parent = parent->split_child_before_op_id(parent->from()); 5305 5306 if (parent->assigned_reg() < LinearScan::nof_regs) { 5307 if (parent->first_usage(shouldHaveRegister) == max_jint) { 5308 // parent is never used, so kick it out of its assigned register 5309 TRACE_LINEAR_SCAN(4, tty->print_cr(" kicking out interval %d out of its register because it is never used", parent->reg_num())); 5310 allocator()->assign_spill_slot(parent); 5311 } else { 5312 // do not go further back because the register is actually used by the interval 5313 parent = nullptr; 5314 } 5315 } 5316 } 5317 5318 } else { 5319 // search optimal split pos, split interval and spill only the right hand part 5320 int optimal_split_pos = find_optimal_split_pos(it, min_split_pos, max_split_pos, false); 5321 5322 assert(min_split_pos <= optimal_split_pos && optimal_split_pos <= max_split_pos, "out of range"); 5323 assert(optimal_split_pos < it->to(), "cannot split at end of interval"); 5324 assert(optimal_split_pos >= it->from(), "cannot split before start of interval"); 5325 5326 if (!allocator()->is_block_begin(optimal_split_pos)) { 5327 // move position before actual instruction (odd op_id) 5328 optimal_split_pos = (optimal_split_pos - 1) | 1; 5329 } 5330 5331 TRACE_LINEAR_SCAN(4, tty->print_cr(" splitting at position %d", optimal_split_pos)); 5332 assert(allocator()->is_block_begin(optimal_split_pos) || (optimal_split_pos % 2 == 1), "split pos must be odd when not on block boundary"); 5333 assert(!allocator()->is_block_begin(optimal_split_pos) || (optimal_split_pos % 2 == 0), "split pos must be even on block boundary"); 5334 5335 Interval* spilled_part = it->split(optimal_split_pos); 5336 allocator()->append_interval(spilled_part); 5337 allocator()->assign_spill_slot(spilled_part); 5338 allocator()->change_spill_state(spilled_part, optimal_split_pos); 5339 5340 if (!allocator()->is_block_begin(optimal_split_pos)) { 5341 TRACE_LINEAR_SCAN(4, tty->print_cr(" inserting move from interval %d to %d", it->reg_num(), spilled_part->reg_num())); 5342 insert_move(optimal_split_pos, it, spilled_part); 5343 } 5344 5345 // the current_split_child is needed later when moves are inserted for reloading 5346 assert(spilled_part->current_split_child() == it, "overwriting wrong current_split_child"); 5347 spilled_part->make_current_split_child(); 5348 5349 TRACE_LINEAR_SCAN(2, tty->print_cr(" split interval in two parts")); 5350 TRACE_LINEAR_SCAN(2, tty->print (" "); it->print()); 5351 TRACE_LINEAR_SCAN(2, tty->print (" "); spilled_part->print()); 5352 } 5353 } 5354 5355 5356 void LinearScanWalker::split_stack_interval(Interval* it) { 5357 int min_split_pos = current_position() + 1; 5358 int max_split_pos = MIN2(it->first_usage(shouldHaveRegister), it->to()); 5359 5360 split_before_usage(it, min_split_pos, max_split_pos); 5361 } 5362 5363 void LinearScanWalker::split_when_partial_register_available(Interval* it, int register_available_until) { 5364 int min_split_pos = MAX2(it->previous_usage(shouldHaveRegister, register_available_until), it->from() + 1); 5365 int max_split_pos = register_available_until; 5366 5367 split_before_usage(it, min_split_pos, max_split_pos); 5368 } 5369 5370 void LinearScanWalker::split_and_spill_interval(Interval* it) { 5371 assert(it->state() == activeState || it->state() == inactiveState, "other states not allowed"); 5372 5373 int current_pos = current_position(); 5374 if (it->state() == inactiveState) { 5375 // the interval is currently inactive, so no spill slot is needed for now. 5376 // when the split part is activated, the interval has a new chance to get a register, 5377 // so in the best case no stack slot is necessary 5378 assert(it->has_hole_between(current_pos - 1, current_pos + 1), "interval can not be inactive otherwise"); 5379 split_before_usage(it, current_pos + 1, current_pos + 1); 5380 5381 } else { 5382 // search the position where the interval must have a register and split 5383 // at the optimal position before. 5384 // The new created part is added to the unhandled list and will get a register 5385 // when it is activated 5386 int min_split_pos = current_pos + 1; 5387 int max_split_pos = MIN2(it->next_usage(mustHaveRegister, min_split_pos), it->to()); 5388 5389 split_before_usage(it, min_split_pos, max_split_pos); 5390 5391 assert(it->next_usage(mustHaveRegister, current_pos) == max_jint, "the remaining part is spilled to stack and therefore has no register"); 5392 split_for_spilling(it); 5393 } 5394 } 5395 5396 int LinearScanWalker::find_free_reg(int reg_needed_until, int interval_to, int hint_reg, int ignore_reg, bool* need_split) { 5397 int min_full_reg = any_reg; 5398 int max_partial_reg = any_reg; 5399 5400 for (int i = _first_reg; i <= _last_reg; i++) { 5401 if (i == ignore_reg) { 5402 // this register must be ignored 5403 5404 } else if (_use_pos[i] >= interval_to) { 5405 // this register is free for the full interval 5406 if (min_full_reg == any_reg || i == hint_reg || (_use_pos[i] < _use_pos[min_full_reg] && min_full_reg != hint_reg)) { 5407 min_full_reg = i; 5408 } 5409 } else if (_use_pos[i] > reg_needed_until) { 5410 // this register is at least free until reg_needed_until 5411 if (max_partial_reg == any_reg || i == hint_reg || (_use_pos[i] > _use_pos[max_partial_reg] && max_partial_reg != hint_reg)) { 5412 max_partial_reg = i; 5413 } 5414 } 5415 } 5416 5417 if (min_full_reg != any_reg) { 5418 return min_full_reg; 5419 } else if (max_partial_reg != any_reg) { 5420 *need_split = true; 5421 return max_partial_reg; 5422 } else { 5423 return any_reg; 5424 } 5425 } 5426 5427 int LinearScanWalker::find_free_double_reg(int reg_needed_until, int interval_to, int hint_reg, bool* need_split) { 5428 assert((_last_reg - _first_reg + 1) % 2 == 0, "adjust algorithm"); 5429 5430 int min_full_reg = any_reg; 5431 int max_partial_reg = any_reg; 5432 5433 for (int i = _first_reg; i < _last_reg; i+=2) { 5434 if (_use_pos[i] >= interval_to && _use_pos[i + 1] >= interval_to) { 5435 // this register is free for the full interval 5436 if (min_full_reg == any_reg || i == hint_reg || (_use_pos[i] < _use_pos[min_full_reg] && min_full_reg != hint_reg)) { 5437 min_full_reg = i; 5438 } 5439 } else if (_use_pos[i] > reg_needed_until && _use_pos[i + 1] > reg_needed_until) { 5440 // this register is at least free until reg_needed_until 5441 if (max_partial_reg == any_reg || i == hint_reg || (_use_pos[i] > _use_pos[max_partial_reg] && max_partial_reg != hint_reg)) { 5442 max_partial_reg = i; 5443 } 5444 } 5445 } 5446 5447 if (min_full_reg != any_reg) { 5448 return min_full_reg; 5449 } else if (max_partial_reg != any_reg) { 5450 *need_split = true; 5451 return max_partial_reg; 5452 } else { 5453 return any_reg; 5454 } 5455 } 5456 5457 bool LinearScanWalker::alloc_free_reg(Interval* cur) { 5458 TRACE_LINEAR_SCAN(2, tty->print("trying to find free register for "); cur->print()); 5459 5460 init_use_lists(true); 5461 free_exclude_active_fixed(); 5462 free_exclude_active_any(); 5463 free_collect_inactive_fixed(cur); 5464 free_collect_inactive_any(cur); 5465 assert(unhandled_first(fixedKind) == Interval::end(), "must not have unhandled fixed intervals because all fixed intervals have a use at position 0"); 5466 5467 // _use_pos contains the start of the next interval that has this register assigned 5468 // (either as a fixed register or a normal allocated register in the past) 5469 // only intervals overlapping with cur are processed, non-overlapping invervals can be ignored safely 5470 #ifdef ASSERT 5471 if (TraceLinearScanLevel >= 4) { 5472 tty->print_cr(" state of registers:"); 5473 for (int i = _first_reg; i <= _last_reg; i++) { 5474 tty->print(" reg %d (", i); 5475 LinearScan::print_reg_num(i); 5476 tty->print_cr("): use_pos: %d", _use_pos[i]); 5477 } 5478 } 5479 #endif 5480 5481 int hint_reg, hint_regHi; 5482 Interval* register_hint = cur->register_hint(); 5483 if (register_hint != nullptr) { 5484 hint_reg = register_hint->assigned_reg(); 5485 hint_regHi = register_hint->assigned_regHi(); 5486 5487 if (_num_phys_regs == 2 && allocator()->is_precolored_cpu_interval(register_hint)) { 5488 assert(hint_reg != any_reg && hint_regHi == any_reg, "must be for fixed intervals"); 5489 hint_regHi = hint_reg + 1; // connect e.g. eax-edx 5490 } 5491 #ifdef ASSERT 5492 if (TraceLinearScanLevel >= 4) { 5493 tty->print(" hint registers %d (", hint_reg); 5494 LinearScan::print_reg_num(hint_reg); 5495 tty->print("), %d (", hint_regHi); 5496 LinearScan::print_reg_num(hint_regHi); 5497 tty->print(") from interval "); 5498 register_hint->print(); 5499 } 5500 #endif 5501 } else { 5502 hint_reg = any_reg; 5503 hint_regHi = any_reg; 5504 } 5505 assert(hint_reg == any_reg || hint_reg != hint_regHi, "hint reg and regHi equal"); 5506 assert(cur->assigned_reg() == any_reg && cur->assigned_regHi() == any_reg, "register already assigned to interval"); 5507 5508 // the register must be free at least until this position 5509 int reg_needed_until = cur->from() + 1; 5510 int interval_to = cur->to(); 5511 5512 bool need_split = false; 5513 int split_pos; 5514 int reg; 5515 int regHi = any_reg; 5516 5517 if (_adjacent_regs) { 5518 reg = find_free_double_reg(reg_needed_until, interval_to, hint_reg, &need_split); 5519 regHi = reg + 1; 5520 if (reg == any_reg) { 5521 return false; 5522 } 5523 split_pos = MIN2(_use_pos[reg], _use_pos[regHi]); 5524 5525 } else { 5526 reg = find_free_reg(reg_needed_until, interval_to, hint_reg, any_reg, &need_split); 5527 if (reg == any_reg) { 5528 return false; 5529 } 5530 split_pos = _use_pos[reg]; 5531 5532 if (_num_phys_regs == 2) { 5533 regHi = find_free_reg(reg_needed_until, interval_to, hint_regHi, reg, &need_split); 5534 5535 if (_use_pos[reg] < interval_to && regHi == any_reg) { 5536 // do not split interval if only one register can be assigned until the split pos 5537 // (when one register is found for the whole interval, split&spill is only 5538 // performed for the hi register) 5539 return false; 5540 5541 } else if (regHi != any_reg) { 5542 split_pos = MIN2(split_pos, _use_pos[regHi]); 5543 5544 // sort register numbers to prevent e.g. a move from eax,ebx to ebx,eax 5545 if (reg > regHi) { 5546 int temp = reg; 5547 reg = regHi; 5548 regHi = temp; 5549 } 5550 } 5551 } 5552 } 5553 5554 cur->assign_reg(reg, regHi); 5555 #ifdef ASSERT 5556 if (TraceLinearScanLevel >= 2) { 5557 tty->print(" selected registers %d (", reg); 5558 LinearScan::print_reg_num(reg); 5559 tty->print("), %d (", regHi); 5560 LinearScan::print_reg_num(regHi); 5561 tty->print_cr(")"); 5562 } 5563 #endif 5564 assert(split_pos > 0, "invalid split_pos"); 5565 if (need_split) { 5566 // register not available for full interval, so split it 5567 split_when_partial_register_available(cur, split_pos); 5568 } 5569 5570 // only return true if interval is completely assigned 5571 return _num_phys_regs == 1 || regHi != any_reg; 5572 } 5573 5574 5575 int LinearScanWalker::find_locked_reg(int reg_needed_until, int interval_to, int ignore_reg, bool* need_split) { 5576 int max_reg = any_reg; 5577 5578 for (int i = _first_reg; i <= _last_reg; i++) { 5579 if (i == ignore_reg) { 5580 // this register must be ignored 5581 5582 } else if (_use_pos[i] > reg_needed_until) { 5583 if (max_reg == any_reg || _use_pos[i] > _use_pos[max_reg]) { 5584 max_reg = i; 5585 } 5586 } 5587 } 5588 5589 if (max_reg != any_reg && _block_pos[max_reg] <= interval_to) { 5590 *need_split = true; 5591 } 5592 5593 return max_reg; 5594 } 5595 5596 int LinearScanWalker::find_locked_double_reg(int reg_needed_until, int interval_to, bool* need_split) { 5597 assert((_last_reg - _first_reg + 1) % 2 == 0, "adjust algorithm"); 5598 5599 int max_reg = any_reg; 5600 5601 for (int i = _first_reg; i < _last_reg; i+=2) { 5602 if (_use_pos[i] > reg_needed_until && _use_pos[i + 1] > reg_needed_until) { 5603 if (max_reg == any_reg || _use_pos[i] > _use_pos[max_reg]) { 5604 max_reg = i; 5605 } 5606 } 5607 } 5608 5609 if (max_reg != any_reg && 5610 (_block_pos[max_reg] <= interval_to || _block_pos[max_reg + 1] <= interval_to)) { 5611 *need_split = true; 5612 } 5613 5614 return max_reg; 5615 } 5616 5617 void LinearScanWalker::split_and_spill_intersecting_intervals(int reg, int regHi) { 5618 assert(reg != any_reg, "no register assigned"); 5619 5620 for (int i = 0; i < _spill_intervals[reg]->length(); i++) { 5621 Interval* it = _spill_intervals[reg]->at(i); 5622 remove_from_list(it); 5623 split_and_spill_interval(it); 5624 } 5625 5626 if (regHi != any_reg) { 5627 IntervalList* processed = _spill_intervals[reg]; 5628 for (int i = 0; i < _spill_intervals[regHi]->length(); i++) { 5629 Interval* it = _spill_intervals[regHi]->at(i); 5630 if (processed->find(it) == -1) { 5631 remove_from_list(it); 5632 split_and_spill_interval(it); 5633 } 5634 } 5635 } 5636 } 5637 5638 5639 // Split an Interval and spill it to memory so that cur can be placed in a register 5640 void LinearScanWalker::alloc_locked_reg(Interval* cur) { 5641 TRACE_LINEAR_SCAN(2, tty->print("need to split and spill to get register for "); cur->print()); 5642 5643 // collect current usage of registers 5644 init_use_lists(false); 5645 spill_exclude_active_fixed(); 5646 assert(unhandled_first(fixedKind) == Interval::end(), "must not have unhandled fixed intervals because all fixed intervals have a use at position 0"); 5647 spill_block_inactive_fixed(cur); 5648 spill_collect_active_any(); 5649 spill_collect_inactive_any(cur); 5650 5651 #ifdef ASSERT 5652 if (TraceLinearScanLevel >= 4) { 5653 tty->print_cr(" state of registers:"); 5654 for (int i = _first_reg; i <= _last_reg; i++) { 5655 tty->print(" reg %d(", i); 5656 LinearScan::print_reg_num(i); 5657 tty->print("): use_pos: %d, block_pos: %d, intervals: ", _use_pos[i], _block_pos[i]); 5658 for (int j = 0; j < _spill_intervals[i]->length(); j++) { 5659 tty->print("%d ", _spill_intervals[i]->at(j)->reg_num()); 5660 } 5661 tty->cr(); 5662 } 5663 } 5664 #endif 5665 5666 // the register must be free at least until this position 5667 int reg_needed_until = MIN2(cur->first_usage(mustHaveRegister), cur->from() + 1); 5668 int interval_to = cur->to(); 5669 assert (reg_needed_until > 0 && reg_needed_until < max_jint, "interval has no use"); 5670 5671 int split_pos = 0; 5672 int use_pos = 0; 5673 bool need_split = false; 5674 int reg, regHi; 5675 5676 if (_adjacent_regs) { 5677 reg = find_locked_double_reg(reg_needed_until, interval_to, &need_split); 5678 regHi = reg + 1; 5679 5680 if (reg != any_reg) { 5681 use_pos = MIN2(_use_pos[reg], _use_pos[regHi]); 5682 split_pos = MIN2(_block_pos[reg], _block_pos[regHi]); 5683 } 5684 } else { 5685 reg = find_locked_reg(reg_needed_until, interval_to, cur->assigned_reg(), &need_split); 5686 regHi = any_reg; 5687 5688 if (reg != any_reg) { 5689 use_pos = _use_pos[reg]; 5690 split_pos = _block_pos[reg]; 5691 5692 if (_num_phys_regs == 2) { 5693 if (cur->assigned_reg() != any_reg) { 5694 regHi = reg; 5695 reg = cur->assigned_reg(); 5696 } else { 5697 regHi = find_locked_reg(reg_needed_until, interval_to, reg, &need_split); 5698 if (regHi != any_reg) { 5699 use_pos = MIN2(use_pos, _use_pos[regHi]); 5700 split_pos = MIN2(split_pos, _block_pos[regHi]); 5701 } 5702 } 5703 5704 if (regHi != any_reg && reg > regHi) { 5705 // sort register numbers to prevent e.g. a move from eax,ebx to ebx,eax 5706 int temp = reg; 5707 reg = regHi; 5708 regHi = temp; 5709 } 5710 } 5711 } 5712 } 5713 5714 if (reg == any_reg || (_num_phys_regs == 2 && regHi == any_reg) || use_pos <= cur->first_usage(mustHaveRegister)) { 5715 // the first use of cur is later than the spilling position -> spill cur 5716 TRACE_LINEAR_SCAN(4, tty->print_cr("able to spill current interval. first_usage(register): %d, use_pos: %d", cur->first_usage(mustHaveRegister), use_pos)); 5717 5718 if (cur->first_usage(mustHaveRegister) <= cur->from() + 1) { 5719 assert(false, "cannot spill interval that is used in first instruction (possible reason: no register found)"); 5720 // assign a reasonable register and do a bailout in product mode to avoid errors 5721 allocator()->assign_spill_slot(cur); 5722 BAILOUT("LinearScan: no register found"); 5723 } 5724 5725 split_and_spill_interval(cur); 5726 } else { 5727 #ifdef ASSERT 5728 if (TraceLinearScanLevel >= 4) { 5729 tty->print("decided to use register %d (", reg); 5730 LinearScan::print_reg_num(reg); 5731 tty->print("), %d (", regHi); 5732 LinearScan::print_reg_num(regHi); 5733 tty->print_cr(")"); 5734 } 5735 #endif 5736 assert(reg != any_reg && (_num_phys_regs == 1 || regHi != any_reg), "no register found"); 5737 assert(split_pos > 0, "invalid split_pos"); 5738 assert(need_split == false || split_pos > cur->from(), "splitting interval at from"); 5739 5740 cur->assign_reg(reg, regHi); 5741 if (need_split) { 5742 // register not available for full interval, so split it 5743 split_when_partial_register_available(cur, split_pos); 5744 } 5745 5746 // perform splitting and spilling for all affected intervals 5747 split_and_spill_intersecting_intervals(reg, regHi); 5748 } 5749 } 5750 5751 bool LinearScanWalker::no_allocation_possible(Interval* cur) { 5752 #ifdef X86 5753 // fast calculation of intervals that can never get a register because the 5754 // the next instruction is a call that blocks all registers 5755 // Note: this does not work if callee-saved registers are available (e.g. on Sparc) 5756 5757 // check if this interval is the result of a split operation 5758 // (an interval got a register until this position) 5759 int pos = cur->from(); 5760 if ((pos & 1) == 1) { 5761 // the current instruction is a call that blocks all registers 5762 if (pos < allocator()->max_lir_op_id() && allocator()->has_call(pos + 1)) { 5763 TRACE_LINEAR_SCAN(4, tty->print_cr(" free register cannot be available because all registers blocked by following call")); 5764 5765 // safety check that there is really no register available 5766 assert(alloc_free_reg(cur) == false, "found a register for this interval"); 5767 return true; 5768 } 5769 5770 } 5771 #endif 5772 return false; 5773 } 5774 5775 void LinearScanWalker::init_vars_for_alloc(Interval* cur) { 5776 BasicType type = cur->type(); 5777 _num_phys_regs = LinearScan::num_physical_regs(type); 5778 _adjacent_regs = LinearScan::requires_adjacent_regs(type); 5779 5780 if (pd_init_regs_for_alloc(cur)) { 5781 // the appropriate register range was selected. 5782 } else if (type == T_FLOAT || type == T_DOUBLE) { 5783 _first_reg = pd_first_fpu_reg; 5784 _last_reg = pd_last_fpu_reg; 5785 } else { 5786 _first_reg = pd_first_cpu_reg; 5787 _last_reg = FrameMap::last_cpu_reg(); 5788 } 5789 5790 assert(0 <= _first_reg && _first_reg < LinearScan::nof_regs, "out of range"); 5791 assert(0 <= _last_reg && _last_reg < LinearScan::nof_regs, "out of range"); 5792 } 5793 5794 5795 bool LinearScanWalker::is_move(LIR_Op* op, Interval* from, Interval* to) { 5796 if (op->code() != lir_move) { 5797 return false; 5798 } 5799 assert(op->as_Op1() != nullptr, "move must be LIR_Op1"); 5800 5801 LIR_Opr in = ((LIR_Op1*)op)->in_opr(); 5802 LIR_Opr res = ((LIR_Op1*)op)->result_opr(); 5803 return in->is_virtual() && res->is_virtual() && in->vreg_number() == from->reg_num() && res->vreg_number() == to->reg_num(); 5804 } 5805 5806 // optimization (especially for phi functions of nested loops): 5807 // assign same spill slot to non-intersecting intervals 5808 void LinearScanWalker::combine_spilled_intervals(Interval* cur) { 5809 if (cur->is_split_child()) { 5810 // optimization is only suitable for split parents 5811 return; 5812 } 5813 5814 Interval* register_hint = cur->register_hint(false); 5815 if (register_hint == nullptr) { 5816 // cur is not the target of a move, otherwise register_hint would be set 5817 return; 5818 } 5819 assert(register_hint->is_split_parent(), "register hint must be split parent"); 5820 5821 if (cur->spill_state() != noOptimization || register_hint->spill_state() != noOptimization) { 5822 // combining the stack slots for intervals where spill move optimization is applied 5823 // is not benefitial and would cause problems 5824 return; 5825 } 5826 5827 int begin_pos = cur->from(); 5828 int end_pos = cur->to(); 5829 if (end_pos > allocator()->max_lir_op_id() || (begin_pos & 1) != 0 || (end_pos & 1) != 0) { 5830 // safety check that lir_op_with_id is allowed 5831 return; 5832 } 5833 5834 if (!is_move(allocator()->lir_op_with_id(begin_pos), register_hint, cur) || !is_move(allocator()->lir_op_with_id(end_pos), cur, register_hint)) { 5835 // cur and register_hint are not connected with two moves 5836 return; 5837 } 5838 5839 Interval* begin_hint = register_hint->split_child_at_op_id(begin_pos, LIR_OpVisitState::inputMode); 5840 Interval* end_hint = register_hint->split_child_at_op_id(end_pos, LIR_OpVisitState::outputMode); 5841 if (begin_hint == end_hint || begin_hint->to() != begin_pos || end_hint->from() != end_pos) { 5842 // register_hint must be split, otherwise the re-writing of use positions does not work 5843 return; 5844 } 5845 5846 assert(begin_hint->assigned_reg() != any_reg, "must have register assigned"); 5847 assert(end_hint->assigned_reg() == any_reg, "must not have register assigned"); 5848 assert(cur->first_usage(mustHaveRegister) == begin_pos, "must have use position at begin of interval because of move"); 5849 assert(end_hint->first_usage(mustHaveRegister) == end_pos, "must have use position at begin of interval because of move"); 5850 5851 if (begin_hint->assigned_reg() < LinearScan::nof_regs) { 5852 // register_hint is not spilled at begin_pos, so it would not be benefitial to immediately spill cur 5853 return; 5854 } 5855 assert(register_hint->canonical_spill_slot() != -1, "must be set when part of interval was spilled"); 5856 assert(!cur->intersects(register_hint), "cur should not intersect register_hint"); 5857 5858 if (cur->intersects_any_children_of(register_hint)) { 5859 // Bail out if cur intersects any split children of register_hint, which have the same spill slot as their parent. An overlap of two intervals with 5860 // the same spill slot could result in a situation where both intervals are spilled at the same time to the same stack location which is not correct. 5861 return; 5862 } 5863 5864 // modify intervals such that cur gets the same stack slot as register_hint 5865 // delete use positions to prevent the intervals to get a register at beginning 5866 cur->set_canonical_spill_slot(register_hint->canonical_spill_slot()); 5867 cur->remove_first_use_pos(); 5868 end_hint->remove_first_use_pos(); 5869 } 5870 5871 5872 // allocate a physical register or memory location to an interval 5873 bool LinearScanWalker::activate_current() { 5874 Interval* cur = current(); 5875 bool result = true; 5876 5877 TRACE_LINEAR_SCAN(2, tty->print ("+++++ activating interval "); cur->print()); 5878 TRACE_LINEAR_SCAN(4, tty->print_cr(" split_parent: %d, insert_move_when_activated: %d", cur->split_parent()->reg_num(), cur->insert_move_when_activated())); 5879 5880 if (cur->assigned_reg() >= LinearScan::nof_regs) { 5881 // activating an interval that has a stack slot assigned -> split it at first use position 5882 // used for method parameters 5883 TRACE_LINEAR_SCAN(4, tty->print_cr(" interval has spill slot assigned (method parameter) -> split it before first use")); 5884 5885 split_stack_interval(cur); 5886 result = false; 5887 5888 } else if (allocator()->gen()->is_vreg_flag_set(cur->reg_num(), LIRGenerator::must_start_in_memory)) { 5889 // activating an interval that must start in a stack slot, but may get a register later 5890 // used for lir_roundfp: rounding is done by store to stack and reload later 5891 TRACE_LINEAR_SCAN(4, tty->print_cr(" interval must start in stack slot -> split it before first use")); 5892 assert(cur->assigned_reg() == any_reg && cur->assigned_regHi() == any_reg, "register already assigned"); 5893 5894 allocator()->assign_spill_slot(cur); 5895 split_stack_interval(cur); 5896 result = false; 5897 5898 } else if (cur->assigned_reg() == any_reg) { 5899 // interval has not assigned register -> normal allocation 5900 // (this is the normal case for most intervals) 5901 TRACE_LINEAR_SCAN(4, tty->print_cr(" normal allocation of register")); 5902 5903 // assign same spill slot to non-intersecting intervals 5904 combine_spilled_intervals(cur); 5905 5906 init_vars_for_alloc(cur); 5907 if (no_allocation_possible(cur) || !alloc_free_reg(cur)) { 5908 // no empty register available. 5909 // split and spill another interval so that this interval gets a register 5910 alloc_locked_reg(cur); 5911 } 5912 5913 // spilled intervals need not be move to active-list 5914 if (cur->assigned_reg() >= LinearScan::nof_regs) { 5915 result = false; 5916 } 5917 } 5918 5919 // load spilled values that become active from stack slot to register 5920 if (cur->insert_move_when_activated()) { 5921 assert(cur->is_split_child(), "must be"); 5922 assert(cur->current_split_child() != nullptr, "must be"); 5923 assert(cur->current_split_child()->reg_num() != cur->reg_num(), "cannot insert move between same interval"); 5924 TRACE_LINEAR_SCAN(4, tty->print_cr("Inserting move from interval %d to %d because insert_move_when_activated is set", cur->current_split_child()->reg_num(), cur->reg_num())); 5925 5926 insert_move(cur->from(), cur->current_split_child(), cur); 5927 } 5928 cur->make_current_split_child(); 5929 5930 return result; // true = interval is moved to active list 5931 } 5932 5933 5934 // Implementation of EdgeMoveOptimizer 5935 5936 EdgeMoveOptimizer::EdgeMoveOptimizer() : 5937 _edge_instructions(4), 5938 _edge_instructions_idx(4) 5939 { 5940 } 5941 5942 void EdgeMoveOptimizer::optimize(BlockList* code) { 5943 EdgeMoveOptimizer optimizer = EdgeMoveOptimizer(); 5944 5945 // ignore the first block in the list (index 0 is not processed) 5946 for (int i = code->length() - 1; i >= 1; i--) { 5947 BlockBegin* block = code->at(i); 5948 5949 if (block->number_of_preds() > 1 && !block->is_set(BlockBegin::exception_entry_flag)) { 5950 optimizer.optimize_moves_at_block_end(block); 5951 } 5952 if (block->number_of_sux() == 2) { 5953 optimizer.optimize_moves_at_block_begin(block); 5954 } 5955 } 5956 } 5957 5958 5959 // clear all internal data structures 5960 void EdgeMoveOptimizer::init_instructions() { 5961 _edge_instructions.clear(); 5962 _edge_instructions_idx.clear(); 5963 } 5964 5965 // append a lir-instruction-list and the index of the current operation in to the list 5966 void EdgeMoveOptimizer::append_instructions(LIR_OpList* instructions, int instructions_idx) { 5967 _edge_instructions.append(instructions); 5968 _edge_instructions_idx.append(instructions_idx); 5969 } 5970 5971 // return the current operation of the given edge (predecessor or successor) 5972 LIR_Op* EdgeMoveOptimizer::instruction_at(int edge) { 5973 LIR_OpList* instructions = _edge_instructions.at(edge); 5974 int idx = _edge_instructions_idx.at(edge); 5975 5976 if (idx < instructions->length()) { 5977 return instructions->at(idx); 5978 } else { 5979 return nullptr; 5980 } 5981 } 5982 5983 // removes the current operation of the given edge (predecessor or successor) 5984 void EdgeMoveOptimizer::remove_cur_instruction(int edge, bool decrement_index) { 5985 LIR_OpList* instructions = _edge_instructions.at(edge); 5986 int idx = _edge_instructions_idx.at(edge); 5987 instructions->remove_at(idx); 5988 5989 if (decrement_index) { 5990 _edge_instructions_idx.at_put(edge, idx - 1); 5991 } 5992 } 5993 5994 5995 bool EdgeMoveOptimizer::operations_different(LIR_Op* op1, LIR_Op* op2) { 5996 if (op1 == nullptr || op2 == nullptr) { 5997 // at least one block is already empty -> no optimization possible 5998 return true; 5999 } 6000 6001 if (op1->code() == lir_move && op2->code() == lir_move) { 6002 assert(op1->as_Op1() != nullptr, "move must be LIR_Op1"); 6003 assert(op2->as_Op1() != nullptr, "move must be LIR_Op1"); 6004 LIR_Op1* move1 = (LIR_Op1*)op1; 6005 LIR_Op1* move2 = (LIR_Op1*)op2; 6006 if (move1->info() == move2->info() && move1->in_opr() == move2->in_opr() && move1->result_opr() == move2->result_opr()) { 6007 // these moves are exactly equal and can be optimized 6008 return false; 6009 } 6010 6011 } else if (op1->code() == lir_fxch && op2->code() == lir_fxch) { 6012 assert(op1->as_Op1() != nullptr, "fxch must be LIR_Op1"); 6013 assert(op2->as_Op1() != nullptr, "fxch must be LIR_Op1"); 6014 LIR_Op1* fxch1 = (LIR_Op1*)op1; 6015 LIR_Op1* fxch2 = (LIR_Op1*)op2; 6016 if (fxch1->in_opr()->as_jint() == fxch2->in_opr()->as_jint()) { 6017 // equal FPU stack operations can be optimized 6018 return false; 6019 } 6020 6021 } else if (op1->code() == lir_fpop_raw && op2->code() == lir_fpop_raw) { 6022 // equal FPU stack operations can be optimized 6023 return false; 6024 } 6025 6026 // no optimization possible 6027 return true; 6028 } 6029 6030 void EdgeMoveOptimizer::optimize_moves_at_block_end(BlockBegin* block) { 6031 TRACE_LINEAR_SCAN(4, tty->print_cr("optimizing moves at end of block B%d", block->block_id())); 6032 6033 if (block->is_predecessor(block)) { 6034 // currently we can't handle this correctly. 6035 return; 6036 } 6037 6038 init_instructions(); 6039 int num_preds = block->number_of_preds(); 6040 assert(num_preds > 1, "do not call otherwise"); 6041 assert(!block->is_set(BlockBegin::exception_entry_flag), "exception handlers not allowed"); 6042 6043 // setup a list with the lir-instructions of all predecessors 6044 int i; 6045 for (i = 0; i < num_preds; i++) { 6046 BlockBegin* pred = block->pred_at(i); 6047 LIR_OpList* pred_instructions = pred->lir()->instructions_list(); 6048 6049 if (pred->number_of_sux() != 1) { 6050 // this can happen with switch-statements where multiple edges are between 6051 // the same blocks. 6052 return; 6053 } 6054 6055 assert(pred->number_of_sux() == 1, "can handle only one successor"); 6056 assert(pred->sux_at(0) == block, "invalid control flow"); 6057 assert(pred_instructions->last()->code() == lir_branch, "block with successor must end with branch"); 6058 assert(pred_instructions->last()->as_OpBranch() != nullptr, "branch must be LIR_OpBranch"); 6059 assert(pred_instructions->last()->as_OpBranch()->cond() == lir_cond_always, "block must end with unconditional branch"); 6060 6061 if (pred_instructions->last()->info() != nullptr) { 6062 // can not optimize instructions when debug info is needed 6063 return; 6064 } 6065 6066 // ignore the unconditional branch at the end of the block 6067 append_instructions(pred_instructions, pred_instructions->length() - 2); 6068 } 6069 6070 6071 // process lir-instructions while all predecessors end with the same instruction 6072 while (true) { 6073 LIR_Op* op = instruction_at(0); 6074 for (i = 1; i < num_preds; i++) { 6075 if (operations_different(op, instruction_at(i))) { 6076 // these instructions are different and cannot be optimized -> 6077 // no further optimization possible 6078 return; 6079 } 6080 } 6081 6082 TRACE_LINEAR_SCAN(4, tty->print("found instruction that is equal in all %d predecessors: ", num_preds); op->print()); 6083 6084 // insert the instruction at the beginning of the current block 6085 block->lir()->insert_before(1, op); 6086 6087 // delete the instruction at the end of all predecessors 6088 for (i = 0; i < num_preds; i++) { 6089 remove_cur_instruction(i, true); 6090 } 6091 } 6092 } 6093 6094 6095 void EdgeMoveOptimizer::optimize_moves_at_block_begin(BlockBegin* block) { 6096 TRACE_LINEAR_SCAN(4, tty->print_cr("optimization moves at begin of block B%d", block->block_id())); 6097 6098 init_instructions(); 6099 int num_sux = block->number_of_sux(); 6100 6101 LIR_OpList* cur_instructions = block->lir()->instructions_list(); 6102 6103 assert(num_sux == 2, "method should not be called otherwise"); 6104 assert(cur_instructions->last()->code() == lir_branch, "block with successor must end with branch"); 6105 assert(cur_instructions->last()->as_OpBranch() != nullptr, "branch must be LIR_OpBranch"); 6106 assert(cur_instructions->last()->as_OpBranch()->cond() == lir_cond_always, "block must end with unconditional branch"); 6107 6108 if (cur_instructions->last()->info() != nullptr) { 6109 // can no optimize instructions when debug info is needed 6110 return; 6111 } 6112 6113 LIR_Op* branch = cur_instructions->at(cur_instructions->length() - 2); 6114 if (branch->info() != nullptr || (branch->code() != lir_branch && branch->code() != lir_cond_float_branch)) { 6115 // not a valid case for optimization 6116 // currently, only blocks that end with two branches (conditional branch followed 6117 // by unconditional branch) are optimized 6118 return; 6119 } 6120 6121 // now it is guaranteed that the block ends with two branch instructions. 6122 // the instructions are inserted at the end of the block before these two branches 6123 int insert_idx = cur_instructions->length() - 2; 6124 6125 int i; 6126 #ifdef ASSERT 6127 for (i = insert_idx - 1; i >= 0; i--) { 6128 LIR_Op* op = cur_instructions->at(i); 6129 if ((op->code() == lir_branch || op->code() == lir_cond_float_branch) && ((LIR_OpBranch*)op)->block() != nullptr) { 6130 assert(false, "block with two successors can have only two branch instructions"); 6131 } 6132 } 6133 #endif 6134 6135 // setup a list with the lir-instructions of all successors 6136 for (i = 0; i < num_sux; i++) { 6137 BlockBegin* sux = block->sux_at(i); 6138 LIR_OpList* sux_instructions = sux->lir()->instructions_list(); 6139 6140 assert(sux_instructions->at(0)->code() == lir_label, "block must start with label"); 6141 6142 if (sux->number_of_preds() != 1) { 6143 // this can happen with switch-statements where multiple edges are between 6144 // the same blocks. 6145 return; 6146 } 6147 assert(sux->pred_at(0) == block, "invalid control flow"); 6148 assert(!sux->is_set(BlockBegin::exception_entry_flag), "exception handlers not allowed"); 6149 6150 // ignore the label at the beginning of the block 6151 append_instructions(sux_instructions, 1); 6152 } 6153 6154 // process lir-instructions while all successors begin with the same instruction 6155 while (true) { 6156 LIR_Op* op = instruction_at(0); 6157 for (i = 1; i < num_sux; i++) { 6158 if (operations_different(op, instruction_at(i))) { 6159 // these instructions are different and cannot be optimized -> 6160 // no further optimization possible 6161 return; 6162 } 6163 } 6164 6165 TRACE_LINEAR_SCAN(4, tty->print("----- found instruction that is equal in all %d successors: ", num_sux); op->print()); 6166 6167 // insert instruction at end of current block 6168 block->lir()->insert_before(insert_idx, op); 6169 insert_idx++; 6170 6171 // delete the instructions at the beginning of all successors 6172 for (i = 0; i < num_sux; i++) { 6173 remove_cur_instruction(i, false); 6174 } 6175 } 6176 } 6177 6178 6179 // Implementation of ControlFlowOptimizer 6180 6181 ControlFlowOptimizer::ControlFlowOptimizer() : 6182 _original_preds(4) 6183 { 6184 } 6185 6186 void ControlFlowOptimizer::optimize(BlockList* code) { 6187 ControlFlowOptimizer optimizer = ControlFlowOptimizer(); 6188 6189 // push the OSR entry block to the end so that we're not jumping over it. 6190 BlockBegin* osr_entry = code->at(0)->end()->as_Base()->osr_entry(); 6191 if (osr_entry) { 6192 int index = osr_entry->linear_scan_number(); 6193 assert(code->at(index) == osr_entry, "wrong index"); 6194 code->remove_at(index); 6195 code->append(osr_entry); 6196 } 6197 6198 optimizer.reorder_short_loops(code); 6199 optimizer.delete_empty_blocks(code); 6200 optimizer.delete_unnecessary_jumps(code); 6201 optimizer.delete_jumps_to_return(code); 6202 } 6203 6204 void ControlFlowOptimizer::reorder_short_loop(BlockList* code, BlockBegin* header_block, int header_idx) { 6205 int i = header_idx + 1; 6206 int max_end = MIN2(header_idx + ShortLoopSize, code->length()); 6207 while (i < max_end && code->at(i)->loop_depth() >= header_block->loop_depth()) { 6208 i++; 6209 } 6210 6211 if (i == code->length() || code->at(i)->loop_depth() < header_block->loop_depth()) { 6212 int end_idx = i - 1; 6213 BlockBegin* end_block = code->at(end_idx); 6214 6215 if (end_block->number_of_sux() == 1 && end_block->sux_at(0) == header_block) { 6216 // short loop from header_idx to end_idx found -> reorder blocks such that 6217 // the header_block is the last block instead of the first block of the loop 6218 TRACE_LINEAR_SCAN(1, tty->print_cr("Reordering short loop: length %d, header B%d, end B%d", 6219 end_idx - header_idx + 1, 6220 header_block->block_id(), end_block->block_id())); 6221 6222 for (int j = header_idx; j < end_idx; j++) { 6223 code->at_put(j, code->at(j + 1)); 6224 } 6225 code->at_put(end_idx, header_block); 6226 6227 // correct the flags so that any loop alignment occurs in the right place. 6228 assert(code->at(end_idx)->is_set(BlockBegin::backward_branch_target_flag), "must be backward branch target"); 6229 code->at(end_idx)->clear(BlockBegin::backward_branch_target_flag); 6230 code->at(header_idx)->set(BlockBegin::backward_branch_target_flag); 6231 } 6232 } 6233 } 6234 6235 void ControlFlowOptimizer::reorder_short_loops(BlockList* code) { 6236 for (int i = code->length() - 1; i >= 0; i--) { 6237 BlockBegin* block = code->at(i); 6238 6239 if (block->is_set(BlockBegin::linear_scan_loop_header_flag)) { 6240 reorder_short_loop(code, block, i); 6241 } 6242 } 6243 6244 DEBUG_ONLY(verify(code)); 6245 } 6246 6247 // only blocks with exactly one successor can be deleted. Such blocks 6248 // must always end with an unconditional branch to this successor 6249 bool ControlFlowOptimizer::can_delete_block(BlockBegin* block) { 6250 if (block->number_of_sux() != 1 || block->number_of_exception_handlers() != 0 || block->is_entry_block()) { 6251 return false; 6252 } 6253 6254 LIR_OpList* instructions = block->lir()->instructions_list(); 6255 6256 assert(instructions->length() >= 2, "block must have label and branch"); 6257 assert(instructions->at(0)->code() == lir_label, "first instruction must always be a label"); 6258 assert(instructions->last()->as_OpBranch() != nullptr, "last instruction must always be a branch"); 6259 assert(instructions->last()->as_OpBranch()->cond() == lir_cond_always, "branch must be unconditional"); 6260 assert(instructions->last()->as_OpBranch()->block() == block->sux_at(0), "branch target must be the successor"); 6261 6262 // block must have exactly one successor 6263 6264 if (instructions->length() == 2 && instructions->last()->info() == nullptr) { 6265 return true; 6266 } 6267 return false; 6268 } 6269 6270 // substitute branch targets in all branch-instructions of this blocks 6271 void ControlFlowOptimizer::substitute_branch_target(BlockBegin* block, BlockBegin* target_from, BlockBegin* target_to) { 6272 TRACE_LINEAR_SCAN(3, tty->print_cr("Deleting empty block: substituting from B%d to B%d inside B%d", target_from->block_id(), target_to->block_id(), block->block_id())); 6273 6274 LIR_OpList* instructions = block->lir()->instructions_list(); 6275 6276 assert(instructions->at(0)->code() == lir_label, "first instruction must always be a label"); 6277 for (int i = instructions->length() - 1; i >= 1; i--) { 6278 LIR_Op* op = instructions->at(i); 6279 6280 if (op->code() == lir_branch || op->code() == lir_cond_float_branch) { 6281 assert(op->as_OpBranch() != nullptr, "branch must be of type LIR_OpBranch"); 6282 LIR_OpBranch* branch = (LIR_OpBranch*)op; 6283 6284 if (branch->block() == target_from) { 6285 branch->change_block(target_to); 6286 } 6287 if (branch->ublock() == target_from) { 6288 branch->change_ublock(target_to); 6289 } 6290 } 6291 } 6292 } 6293 6294 void ControlFlowOptimizer::delete_empty_blocks(BlockList* code) { 6295 int old_pos = 0; 6296 int new_pos = 0; 6297 int num_blocks = code->length(); 6298 6299 while (old_pos < num_blocks) { 6300 BlockBegin* block = code->at(old_pos); 6301 6302 if (can_delete_block(block)) { 6303 BlockBegin* new_target = block->sux_at(0); 6304 6305 // propagate backward branch target flag for correct code alignment 6306 if (block->is_set(BlockBegin::backward_branch_target_flag)) { 6307 new_target->set(BlockBegin::backward_branch_target_flag); 6308 } 6309 6310 // collect a list with all predecessors that contains each predecessor only once 6311 // the predecessors of cur are changed during the substitution, so a copy of the 6312 // predecessor list is necessary 6313 int j; 6314 _original_preds.clear(); 6315 for (j = block->number_of_preds() - 1; j >= 0; j--) { 6316 BlockBegin* pred = block->pred_at(j); 6317 if (_original_preds.find(pred) == -1) { 6318 _original_preds.append(pred); 6319 } 6320 } 6321 6322 for (j = _original_preds.length() - 1; j >= 0; j--) { 6323 BlockBegin* pred = _original_preds.at(j); 6324 substitute_branch_target(pred, block, new_target); 6325 pred->substitute_sux(block, new_target); 6326 } 6327 } else { 6328 // adjust position of this block in the block list if blocks before 6329 // have been deleted 6330 if (new_pos != old_pos) { 6331 code->at_put(new_pos, code->at(old_pos)); 6332 } 6333 new_pos++; 6334 } 6335 old_pos++; 6336 } 6337 code->trunc_to(new_pos); 6338 6339 DEBUG_ONLY(verify(code)); 6340 } 6341 6342 void ControlFlowOptimizer::delete_unnecessary_jumps(BlockList* code) { 6343 // skip the last block because there a branch is always necessary 6344 for (int i = code->length() - 2; i >= 0; i--) { 6345 BlockBegin* block = code->at(i); 6346 LIR_OpList* instructions = block->lir()->instructions_list(); 6347 6348 LIR_Op* last_op = instructions->last(); 6349 if (last_op->code() == lir_branch) { 6350 assert(last_op->as_OpBranch() != nullptr, "branch must be of type LIR_OpBranch"); 6351 LIR_OpBranch* last_branch = (LIR_OpBranch*)last_op; 6352 6353 assert(last_branch->block() != nullptr, "last branch must always have a block as target"); 6354 assert(last_branch->label() == last_branch->block()->label(), "must be equal"); 6355 6356 if (last_branch->info() == nullptr) { 6357 if (last_branch->block() == code->at(i + 1)) { 6358 6359 TRACE_LINEAR_SCAN(3, tty->print_cr("Deleting unconditional branch at end of block B%d", block->block_id())); 6360 6361 // delete last branch instruction 6362 instructions->trunc_to(instructions->length() - 1); 6363 6364 } else { 6365 LIR_Op* prev_op = instructions->at(instructions->length() - 2); 6366 if (prev_op->code() == lir_branch || prev_op->code() == lir_cond_float_branch) { 6367 assert(prev_op->as_OpBranch() != nullptr, "branch must be of type LIR_OpBranch"); 6368 LIR_OpBranch* prev_branch = (LIR_OpBranch*)prev_op; 6369 6370 if (prev_branch->stub() == nullptr) { 6371 6372 LIR_Op2* prev_cmp = nullptr; 6373 // There might be a cmove inserted for profiling which depends on the same 6374 // compare. If we change the condition of the respective compare, we have 6375 // to take care of this cmove as well. 6376 LIR_Op4* prev_cmove = nullptr; 6377 6378 for(int j = instructions->length() - 3; j >= 0 && prev_cmp == nullptr; j--) { 6379 prev_op = instructions->at(j); 6380 // check for the cmove 6381 if (prev_op->code() == lir_cmove) { 6382 assert(prev_op->as_Op4() != nullptr, "cmove must be of type LIR_Op4"); 6383 prev_cmove = (LIR_Op4*)prev_op; 6384 assert(prev_branch->cond() == prev_cmove->condition(), "should be the same"); 6385 } 6386 if (prev_op->code() == lir_cmp) { 6387 assert(prev_op->as_Op2() != nullptr, "branch must be of type LIR_Op2"); 6388 prev_cmp = (LIR_Op2*)prev_op; 6389 assert(prev_branch->cond() == prev_cmp->condition(), "should be the same"); 6390 } 6391 } 6392 // Guarantee because it is dereferenced below. 6393 guarantee(prev_cmp != nullptr, "should have found comp instruction for branch"); 6394 if (prev_branch->block() == code->at(i + 1) && prev_branch->info() == nullptr) { 6395 6396 TRACE_LINEAR_SCAN(3, tty->print_cr("Negating conditional branch and deleting unconditional branch at end of block B%d", block->block_id())); 6397 6398 // eliminate a conditional branch to the immediate successor 6399 prev_branch->change_block(last_branch->block()); 6400 prev_branch->negate_cond(); 6401 prev_cmp->set_condition(prev_branch->cond()); 6402 instructions->trunc_to(instructions->length() - 1); 6403 // if we do change the condition, we have to change the cmove as well 6404 if (prev_cmove != nullptr) { 6405 prev_cmove->set_condition(prev_branch->cond()); 6406 LIR_Opr t = prev_cmove->in_opr1(); 6407 prev_cmove->set_in_opr1(prev_cmove->in_opr2()); 6408 prev_cmove->set_in_opr2(t); 6409 } 6410 } 6411 } 6412 } 6413 } 6414 } 6415 } 6416 } 6417 6418 DEBUG_ONLY(verify(code)); 6419 } 6420 6421 void ControlFlowOptimizer::delete_jumps_to_return(BlockList* code) { 6422 #ifdef ASSERT 6423 ResourceBitMap return_converted(BlockBegin::number_of_blocks()); 6424 #endif 6425 6426 for (int i = code->length() - 1; i >= 0; i--) { 6427 BlockBegin* block = code->at(i); 6428 LIR_OpList* cur_instructions = block->lir()->instructions_list(); 6429 LIR_Op* cur_last_op = cur_instructions->last(); 6430 6431 assert(cur_instructions->at(0)->code() == lir_label, "first instruction must always be a label"); 6432 if (cur_instructions->length() == 2 && cur_last_op->code() == lir_return) { 6433 // the block contains only a label and a return 6434 // if a predecessor ends with an unconditional jump to this block, then the jump 6435 // can be replaced with a return instruction 6436 // 6437 // Note: the original block with only a return statement cannot be deleted completely 6438 // because the predecessors might have other (conditional) jumps to this block 6439 // -> this may lead to unnecessary return instructions in the final code 6440 6441 assert(cur_last_op->info() == nullptr, "return instructions do not have debug information"); 6442 assert(block->number_of_sux() == 0 || 6443 (return_converted.at(block->block_id()) && block->number_of_sux() == 1), 6444 "blocks that end with return must not have successors"); 6445 6446 assert(cur_last_op->as_Op1() != nullptr, "return must be LIR_Op1"); 6447 LIR_Opr return_opr = ((LIR_Op1*)cur_last_op)->in_opr(); 6448 6449 for (int j = block->number_of_preds() - 1; j >= 0; j--) { 6450 BlockBegin* pred = block->pred_at(j); 6451 LIR_OpList* pred_instructions = pred->lir()->instructions_list(); 6452 LIR_Op* pred_last_op = pred_instructions->last(); 6453 6454 if (pred_last_op->code() == lir_branch) { 6455 assert(pred_last_op->as_OpBranch() != nullptr, "branch must be LIR_OpBranch"); 6456 LIR_OpBranch* pred_last_branch = (LIR_OpBranch*)pred_last_op; 6457 6458 if (pred_last_branch->block() == block && pred_last_branch->cond() == lir_cond_always && pred_last_branch->info() == nullptr) { 6459 // replace the jump to a return with a direct return 6460 // Note: currently the edge between the blocks is not deleted 6461 pred_instructions->at_put(pred_instructions->length() - 1, new LIR_OpReturn(return_opr)); 6462 #ifdef ASSERT 6463 return_converted.set_bit(pred->block_id()); 6464 #endif 6465 } 6466 } 6467 } 6468 } 6469 } 6470 } 6471 6472 6473 #ifdef ASSERT 6474 void ControlFlowOptimizer::verify(BlockList* code) { 6475 for (int i = 0; i < code->length(); i++) { 6476 BlockBegin* block = code->at(i); 6477 LIR_OpList* instructions = block->lir()->instructions_list(); 6478 6479 int j; 6480 for (j = 0; j < instructions->length(); j++) { 6481 LIR_OpBranch* op_branch = instructions->at(j)->as_OpBranch(); 6482 6483 if (op_branch != nullptr) { 6484 assert(op_branch->block() == nullptr || code->find(op_branch->block()) != -1, "branch target not valid"); 6485 assert(op_branch->ublock() == nullptr || code->find(op_branch->ublock()) != -1, "branch target not valid"); 6486 } 6487 } 6488 6489 for (j = 0; j < block->number_of_sux() - 1; j++) { 6490 BlockBegin* sux = block->sux_at(j); 6491 assert(code->find(sux) != -1, "successor not valid"); 6492 } 6493 6494 for (j = 0; j < block->number_of_preds() - 1; j++) { 6495 BlockBegin* pred = block->pred_at(j); 6496 assert(code->find(pred) != -1, "successor not valid"); 6497 } 6498 } 6499 } 6500 #endif 6501 6502 6503 #ifndef PRODUCT 6504 6505 // Implementation of LinearStatistic 6506 6507 const char* LinearScanStatistic::counter_name(int counter_idx) { 6508 switch (counter_idx) { 6509 case counter_method: return "compiled methods"; 6510 case counter_fpu_method: return "methods using fpu"; 6511 case counter_loop_method: return "methods with loops"; 6512 case counter_exception_method:return "methods with xhandler"; 6513 6514 case counter_loop: return "loops"; 6515 case counter_block: return "blocks"; 6516 case counter_loop_block: return "blocks inside loop"; 6517 case counter_exception_block: return "exception handler entries"; 6518 case counter_interval: return "intervals"; 6519 case counter_fixed_interval: return "fixed intervals"; 6520 case counter_range: return "ranges"; 6521 case counter_fixed_range: return "fixed ranges"; 6522 case counter_use_pos: return "use positions"; 6523 case counter_fixed_use_pos: return "fixed use positions"; 6524 case counter_spill_slots: return "spill slots"; 6525 6526 // counter for classes of lir instructions 6527 case counter_instruction: return "total instructions"; 6528 case counter_label: return "labels"; 6529 case counter_entry: return "method entries"; 6530 case counter_return: return "method returns"; 6531 case counter_call: return "method calls"; 6532 case counter_move: return "moves"; 6533 case counter_cmp: return "compare"; 6534 case counter_cond_branch: return "conditional branches"; 6535 case counter_uncond_branch: return "unconditional branches"; 6536 case counter_stub_branch: return "branches to stub"; 6537 case counter_alu: return "artithmetic + logic"; 6538 case counter_alloc: return "allocations"; 6539 case counter_sync: return "synchronisation"; 6540 case counter_throw: return "throw"; 6541 case counter_unwind: return "unwind"; 6542 case counter_typecheck: return "type+null-checks"; 6543 case counter_fpu_stack: return "fpu-stack"; 6544 case counter_misc_inst: return "other instructions"; 6545 case counter_other_inst: return "misc. instructions"; 6546 6547 // counter for different types of moves 6548 case counter_move_total: return "total moves"; 6549 case counter_move_reg_reg: return "register->register"; 6550 case counter_move_reg_stack: return "register->stack"; 6551 case counter_move_stack_reg: return "stack->register"; 6552 case counter_move_stack_stack:return "stack->stack"; 6553 case counter_move_reg_mem: return "register->memory"; 6554 case counter_move_mem_reg: return "memory->register"; 6555 case counter_move_const_any: return "constant->any"; 6556 6557 case blank_line_1: return ""; 6558 case blank_line_2: return ""; 6559 6560 default: ShouldNotReachHere(); return ""; 6561 } 6562 } 6563 6564 LinearScanStatistic::Counter LinearScanStatistic::base_counter(int counter_idx) { 6565 if (counter_idx == counter_fpu_method || counter_idx == counter_loop_method || counter_idx == counter_exception_method) { 6566 return counter_method; 6567 } else if (counter_idx == counter_loop_block || counter_idx == counter_exception_block) { 6568 return counter_block; 6569 } else if (counter_idx >= counter_instruction && counter_idx <= counter_other_inst) { 6570 return counter_instruction; 6571 } else if (counter_idx >= counter_move_total && counter_idx <= counter_move_const_any) { 6572 return counter_move_total; 6573 } 6574 return invalid_counter; 6575 } 6576 6577 LinearScanStatistic::LinearScanStatistic() { 6578 for (int i = 0; i < number_of_counters; i++) { 6579 _counters_sum[i] = 0; 6580 _counters_max[i] = -1; 6581 } 6582 6583 } 6584 6585 // add the method-local numbers to the total sum 6586 void LinearScanStatistic::sum_up(LinearScanStatistic &method_statistic) { 6587 for (int i = 0; i < number_of_counters; i++) { 6588 _counters_sum[i] += method_statistic._counters_sum[i]; 6589 _counters_max[i] = MAX2(_counters_max[i], method_statistic._counters_sum[i]); 6590 } 6591 } 6592 6593 void LinearScanStatistic::print(const char* title) { 6594 if (CountLinearScan || TraceLinearScanLevel > 0) { 6595 tty->cr(); 6596 tty->print_cr("***** LinearScan statistic - %s *****", title); 6597 6598 for (int i = 0; i < number_of_counters; i++) { 6599 if (_counters_sum[i] > 0 || _counters_max[i] >= 0) { 6600 tty->print("%25s: %8d", counter_name(i), _counters_sum[i]); 6601 6602 LinearScanStatistic::Counter cntr = base_counter(i); 6603 if (cntr != invalid_counter) { 6604 tty->print(" (%5.1f%%) ", _counters_sum[i] * 100.0 / _counters_sum[cntr]); 6605 } else { 6606 tty->print(" "); 6607 } 6608 6609 if (_counters_max[i] >= 0) { 6610 tty->print("%8d", _counters_max[i]); 6611 } 6612 } 6613 tty->cr(); 6614 } 6615 } 6616 } 6617 6618 void LinearScanStatistic::collect(LinearScan* allocator) { 6619 inc_counter(counter_method); 6620 if (allocator->has_fpu_registers()) { 6621 inc_counter(counter_fpu_method); 6622 } 6623 if (allocator->num_loops() > 0) { 6624 inc_counter(counter_loop_method); 6625 } 6626 inc_counter(counter_loop, allocator->num_loops()); 6627 inc_counter(counter_spill_slots, allocator->max_spills()); 6628 6629 int i; 6630 for (i = 0; i < allocator->interval_count(); i++) { 6631 Interval* cur = allocator->interval_at(i); 6632 6633 if (cur != nullptr) { 6634 inc_counter(counter_interval); 6635 inc_counter(counter_use_pos, cur->num_use_positions()); 6636 if (LinearScan::is_precolored_interval(cur)) { 6637 inc_counter(counter_fixed_interval); 6638 inc_counter(counter_fixed_use_pos, cur->num_use_positions()); 6639 } 6640 6641 Range* range = cur->first(); 6642 while (range != Range::end()) { 6643 inc_counter(counter_range); 6644 if (LinearScan::is_precolored_interval(cur)) { 6645 inc_counter(counter_fixed_range); 6646 } 6647 range = range->next(); 6648 } 6649 } 6650 } 6651 6652 bool has_xhandlers = false; 6653 // Note: only count blocks that are in code-emit order 6654 for (i = 0; i < allocator->ir()->code()->length(); i++) { 6655 BlockBegin* cur = allocator->ir()->code()->at(i); 6656 6657 inc_counter(counter_block); 6658 if (cur->loop_depth() > 0) { 6659 inc_counter(counter_loop_block); 6660 } 6661 if (cur->is_set(BlockBegin::exception_entry_flag)) { 6662 inc_counter(counter_exception_block); 6663 has_xhandlers = true; 6664 } 6665 6666 LIR_OpList* instructions = cur->lir()->instructions_list(); 6667 for (int j = 0; j < instructions->length(); j++) { 6668 LIR_Op* op = instructions->at(j); 6669 6670 inc_counter(counter_instruction); 6671 6672 switch (op->code()) { 6673 case lir_label: inc_counter(counter_label); break; 6674 case lir_std_entry: 6675 case lir_osr_entry: inc_counter(counter_entry); break; 6676 case lir_return: inc_counter(counter_return); break; 6677 6678 case lir_rtcall: 6679 case lir_static_call: 6680 case lir_optvirtual_call: inc_counter(counter_call); break; 6681 6682 case lir_move: { 6683 inc_counter(counter_move); 6684 inc_counter(counter_move_total); 6685 6686 LIR_Opr in = op->as_Op1()->in_opr(); 6687 LIR_Opr res = op->as_Op1()->result_opr(); 6688 if (in->is_register()) { 6689 if (res->is_register()) { 6690 inc_counter(counter_move_reg_reg); 6691 } else if (res->is_stack()) { 6692 inc_counter(counter_move_reg_stack); 6693 } else if (res->is_address()) { 6694 inc_counter(counter_move_reg_mem); 6695 } else { 6696 ShouldNotReachHere(); 6697 } 6698 } else if (in->is_stack()) { 6699 if (res->is_register()) { 6700 inc_counter(counter_move_stack_reg); 6701 } else { 6702 inc_counter(counter_move_stack_stack); 6703 } 6704 } else if (in->is_address()) { 6705 assert(res->is_register(), "must be"); 6706 inc_counter(counter_move_mem_reg); 6707 } else if (in->is_constant()) { 6708 inc_counter(counter_move_const_any); 6709 } else { 6710 ShouldNotReachHere(); 6711 } 6712 break; 6713 } 6714 6715 case lir_cmp: inc_counter(counter_cmp); break; 6716 6717 case lir_branch: 6718 case lir_cond_float_branch: { 6719 LIR_OpBranch* branch = op->as_OpBranch(); 6720 if (branch->block() == nullptr) { 6721 inc_counter(counter_stub_branch); 6722 } else if (branch->cond() == lir_cond_always) { 6723 inc_counter(counter_uncond_branch); 6724 } else { 6725 inc_counter(counter_cond_branch); 6726 } 6727 break; 6728 } 6729 6730 case lir_neg: 6731 case lir_add: 6732 case lir_sub: 6733 case lir_mul: 6734 case lir_div: 6735 case lir_rem: 6736 case lir_sqrt: 6737 case lir_abs: 6738 case lir_f2hf: 6739 case lir_hf2f: 6740 case lir_log10: 6741 case lir_logic_and: 6742 case lir_logic_or: 6743 case lir_logic_xor: 6744 case lir_shl: 6745 case lir_shr: 6746 case lir_ushr: inc_counter(counter_alu); break; 6747 6748 case lir_alloc_object: 6749 case lir_alloc_array: inc_counter(counter_alloc); break; 6750 6751 case lir_monaddr: 6752 case lir_lock: 6753 case lir_unlock: inc_counter(counter_sync); break; 6754 6755 case lir_throw: inc_counter(counter_throw); break; 6756 6757 case lir_unwind: inc_counter(counter_unwind); break; 6758 6759 case lir_null_check: 6760 case lir_leal: 6761 case lir_instanceof: 6762 case lir_checkcast: 6763 case lir_store_check: inc_counter(counter_typecheck); break; 6764 6765 case lir_fpop_raw: 6766 case lir_fxch: 6767 case lir_fld: inc_counter(counter_fpu_stack); break; 6768 6769 case lir_nop: 6770 case lir_push: 6771 case lir_pop: 6772 case lir_convert: 6773 case lir_roundfp: 6774 case lir_cmove: inc_counter(counter_misc_inst); break; 6775 6776 default: inc_counter(counter_other_inst); break; 6777 } 6778 } 6779 } 6780 6781 if (has_xhandlers) { 6782 inc_counter(counter_exception_method); 6783 } 6784 } 6785 6786 void LinearScanStatistic::compute(LinearScan* allocator, LinearScanStatistic &global_statistic) { 6787 if (CountLinearScan || TraceLinearScanLevel > 0) { 6788 6789 LinearScanStatistic local_statistic = LinearScanStatistic(); 6790 6791 local_statistic.collect(allocator); 6792 global_statistic.sum_up(local_statistic); 6793 6794 if (TraceLinearScanLevel > 2) { 6795 local_statistic.print("current local statistic"); 6796 } 6797 } 6798 } 6799 6800 6801 // Implementation of LinearTimers 6802 6803 LinearScanTimers::LinearScanTimers() { 6804 for (int i = 0; i < number_of_timers; i++) { 6805 timer(i)->reset(); 6806 } 6807 } 6808 6809 const char* LinearScanTimers::timer_name(int idx) { 6810 switch (idx) { 6811 case timer_do_nothing: return "Nothing (Time Check)"; 6812 case timer_number_instructions: return "Number Instructions"; 6813 case timer_compute_local_live_sets: return "Local Live Sets"; 6814 case timer_compute_global_live_sets: return "Global Live Sets"; 6815 case timer_build_intervals: return "Build Intervals"; 6816 case timer_sort_intervals_before: return "Sort Intervals Before"; 6817 case timer_allocate_registers: return "Allocate Registers"; 6818 case timer_resolve_data_flow: return "Resolve Data Flow"; 6819 case timer_sort_intervals_after: return "Sort Intervals After"; 6820 case timer_eliminate_spill_moves: return "Spill optimization"; 6821 case timer_assign_reg_num: return "Assign Reg Num"; 6822 case timer_allocate_fpu_stack: return "Allocate FPU Stack"; 6823 case timer_optimize_lir: return "Optimize LIR"; 6824 default: ShouldNotReachHere(); return ""; 6825 } 6826 } 6827 6828 void LinearScanTimers::begin_method() { 6829 if (TimeEachLinearScan) { 6830 // reset all timers to measure only current method 6831 for (int i = 0; i < number_of_timers; i++) { 6832 timer(i)->reset(); 6833 } 6834 } 6835 } 6836 6837 void LinearScanTimers::end_method(LinearScan* allocator) { 6838 if (TimeEachLinearScan) { 6839 6840 double c = timer(timer_do_nothing)->seconds(); 6841 double total = 0; 6842 for (int i = 1; i < number_of_timers; i++) { 6843 total += timer(i)->seconds() - c; 6844 } 6845 6846 if (total >= 0.0005) { 6847 // print all information in one line for automatic processing 6848 tty->print("@"); allocator->compilation()->method()->print_name(); 6849 6850 tty->print("@ %d ", allocator->compilation()->method()->code_size()); 6851 tty->print("@ %d ", allocator->block_at(allocator->block_count() - 1)->last_lir_instruction_id() / 2); 6852 tty->print("@ %d ", allocator->block_count()); 6853 tty->print("@ %d ", allocator->num_virtual_regs()); 6854 tty->print("@ %d ", allocator->interval_count()); 6855 tty->print("@ %d ", allocator->_num_calls); 6856 tty->print("@ %d ", allocator->num_loops()); 6857 6858 tty->print("@ %6.6f ", total); 6859 for (int i = 1; i < number_of_timers; i++) { 6860 tty->print("@ %4.1f ", ((timer(i)->seconds() - c) / total) * 100); 6861 } 6862 tty->cr(); 6863 } 6864 } 6865 } 6866 6867 void LinearScanTimers::print(double total_time) { 6868 if (TimeLinearScan) { 6869 // correction value: sum of dummy-timer that only measures the time that 6870 // is necessary to start and stop itself 6871 double c = timer(timer_do_nothing)->seconds(); 6872 6873 for (int i = 0; i < number_of_timers; i++) { 6874 double t = timer(i)->seconds(); 6875 tty->print_cr(" %25s: %6.3f s (%4.1f%%) corrected: %6.3f s (%4.1f%%)", timer_name(i), t, (t / total_time) * 100.0, t - c, (t - c) / (total_time - 2 * number_of_timers * c) * 100); 6876 } 6877 } 6878 } 6879 6880 #endif // #ifndef PRODUCT