155 Unique_Node_List worklist;
156 VectorSet visited(Thread::current()->resource_area());
157 worklist.push(xroot);
158 while (worklist.size() > 0) {
159 Node* n = worklist.pop();
160 visited <<= n->_idx;
161 assert(C->node_arena()->contains(n), "dead node");
162 for (uint j = 0; j < n->req(); j++) {
163 Node* in = n->in(j);
164 if (in != NULL) {
165 assert(C->node_arena()->contains(in), "dead node");
166 if (!visited.test(in->_idx)) {
167 worklist.push(in);
168 }
169 }
170 }
171 }
172 }
173 #endif
174
175
176 //---------------------------match---------------------------------------------
177 void Matcher::match( ) {
178 if( MaxLabelRootDepth < 100 ) { // Too small?
179 assert(false, "invalid MaxLabelRootDepth, increase it to 100 minimum");
180 MaxLabelRootDepth = 100;
181 }
182 // One-time initialization of some register masks.
183 init_spill_mask( C->root()->in(1) );
184 _return_addr_mask = return_addr();
185 #ifdef _LP64
186 // Pointers take 2 slots in 64-bit land
187 _return_addr_mask.Insert(OptoReg::add(return_addr(),1));
188 #endif
189
190 // Map a Java-signature return type into return register-value
191 // machine registers for 0, 1 and 2 returned values.
192 const TypeTuple *range = C->tf()->range();
193 if( range->cnt() > TypeFunc::Parms ) { // If not a void function
194 // Get ideal-register return type
195 uint ireg = range->field_at(TypeFunc::Parms)->ideal_reg();
196 // Get machine return register
197 uint sop = C->start()->Opcode();
198 OptoRegPair regs = return_value(ireg, false);
199
200 // And mask for same
201 _return_value_mask = RegMask(regs.first());
202 if( OptoReg::is_valid(regs.second()) )
203 _return_value_mask.Insert(regs.second());
204 }
205
206 // ---------------
207 // Frame Layout
208
209 // Need the method signature to determine the incoming argument types,
210 // because the types determine which registers the incoming arguments are
211 // in, and this affects the matched code.
212 const TypeTuple *domain = C->tf()->domain();
213 uint argcnt = domain->cnt() - TypeFunc::Parms;
214 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
215 VMRegPair *vm_parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
216 _parm_regs = NEW_RESOURCE_ARRAY( OptoRegPair, argcnt );
217 _calling_convention_mask = NEW_RESOURCE_ARRAY( RegMask, argcnt );
218 uint i;
219 for( i = 0; i<argcnt; i++ ) {
220 sig_bt[i] = domain->field_at(i+TypeFunc::Parms)->basic_type();
221 }
222
223 // Pass array of ideal registers and length to USER code (from the AD file)
224 // that will convert this to an array of register numbers.
225 const StartNode *start = C->start();
226 start->calling_convention( sig_bt, vm_parm_regs, argcnt );
227 #ifdef ASSERT
228 // Sanity check users' calling convention. Real handy while trying to
229 // get the initial port correct.
230 { for (uint i = 0; i<argcnt; i++) {
231 if( !vm_parm_regs[i].first()->is_valid() && !vm_parm_regs[i].second()->is_valid() ) {
232 assert(domain->field_at(i+TypeFunc::Parms)==Type::HALF, "only allowed on halve" );
452 idealreg2mhdebugmask[Op_RegF] = &rms[15];
453 idealreg2mhdebugmask[Op_RegD] = &rms[16];
454 idealreg2mhdebugmask[Op_RegP] = &rms[17];
455
456 idealreg2spillmask [Op_VecS] = &rms[18];
457 idealreg2spillmask [Op_VecD] = &rms[19];
458 idealreg2spillmask [Op_VecX] = &rms[20];
459 idealreg2spillmask [Op_VecY] = &rms[21];
460 idealreg2spillmask [Op_VecZ] = &rms[22];
461
462 OptoReg::Name i;
463
464 // At first, start with the empty mask
465 C->FIRST_STACK_mask().Clear();
466
467 // Add in the incoming argument area
468 OptoReg::Name init_in = OptoReg::add(_old_SP, C->out_preserve_stack_slots());
469 for (i = init_in; i < _in_arg_limit; i = OptoReg::add(i,1)) {
470 C->FIRST_STACK_mask().Insert(i);
471 }
472 // Add in all bits past the outgoing argument area
473 guarantee(RegMask::can_represent_arg(OptoReg::add(_out_arg_limit,-1)),
474 "must be able to represent all call arguments in reg mask");
475 OptoReg::Name init = _out_arg_limit;
476 for (i = init; RegMask::can_represent(i); i = OptoReg::add(i,1)) {
477 C->FIRST_STACK_mask().Insert(i);
478 }
479 // Finally, set the "infinite stack" bit.
480 C->FIRST_STACK_mask().set_AllStack();
481
482 // Make spill masks. Registers for their class, plus FIRST_STACK_mask.
483 RegMask aligned_stack_mask = C->FIRST_STACK_mask();
484 // Keep spill masks aligned.
485 aligned_stack_mask.clear_to_pairs();
486 assert(aligned_stack_mask.is_AllStack(), "should be infinite stack");
487
488 *idealreg2spillmask[Op_RegP] = *idealreg2regmask[Op_RegP];
489 #ifdef _LP64
490 *idealreg2spillmask[Op_RegN] = *idealreg2regmask[Op_RegN];
491 idealreg2spillmask[Op_RegN]->OR(C->FIRST_STACK_mask());
644 init_first_stack_mask();
645
646 Node *root = C->root(); // Short name for root
647 // Count number of save-on-entry registers.
648 uint soe_cnt = number_of_saved_registers();
649 uint i;
650
651 // Find the procedure Start Node
652 StartNode *start = C->start();
653 assert( start, "Expect a start node" );
654
655 // Save argument registers in the trampolining stubs
656 if( C->save_argument_registers() )
657 for( i = 0; i < _last_Mach_Reg; i++ )
658 if( is_spillable_arg(i) )
659 soe_cnt++;
660
661 // Input RegMask array shared by all Returns.
662 // The type for doubles and longs has a count of 2, but
663 // there is only 1 returned value
664 uint ret_edge_cnt = TypeFunc::Parms + ((C->tf()->range()->cnt() == TypeFunc::Parms) ? 0 : 1);
665 RegMask *ret_rms = init_input_masks( ret_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
666 // Returns have 0 or 1 returned values depending on call signature.
667 // Return register is specified by return_value in the AD file.
668 if (ret_edge_cnt > TypeFunc::Parms)
669 ret_rms[TypeFunc::Parms+0] = _return_value_mask;
670
671 // Input RegMask array shared by all Rethrows.
672 uint reth_edge_cnt = TypeFunc::Parms+1;
673 RegMask *reth_rms = init_input_masks( reth_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
674 // Rethrow takes exception oop only, but in the argument 0 slot.
675 OptoReg::Name reg = find_receiver(false);
676 if (reg >= 0) {
677 reth_rms[TypeFunc::Parms] = mreg2regmask[reg];
678 #ifdef _LP64
679 // Need two slots for ptrs in 64-bit land
680 reth_rms[TypeFunc::Parms].Insert(OptoReg::add(OptoReg::Name(reg), 1));
681 #endif
682 }
683
684 // Input RegMask array shared by all TailCalls
685 uint tail_call_edge_cnt = TypeFunc::Parms+2;
686 RegMask *tail_call_rms = init_input_masks( tail_call_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
687
688 // Input RegMask array shared by all TailJumps
689 uint tail_jump_edge_cnt = TypeFunc::Parms+2;
716 }
717
718 // Input RegMask array shared by all Halts
719 uint halt_edge_cnt = TypeFunc::Parms;
720 RegMask *halt_rms = init_input_masks( halt_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
721
722 // Capture the return input masks into each exit flavor
723 for( i=1; i < root->req(); i++ ) {
724 MachReturnNode *exit = root->in(i)->as_MachReturn();
725 switch( exit->ideal_Opcode() ) {
726 case Op_Return : exit->_in_rms = ret_rms; break;
727 case Op_Rethrow : exit->_in_rms = reth_rms; break;
728 case Op_TailCall : exit->_in_rms = tail_call_rms; break;
729 case Op_TailJump : exit->_in_rms = tail_jump_rms; break;
730 case Op_Halt : exit->_in_rms = halt_rms; break;
731 default : ShouldNotReachHere();
732 }
733 }
734
735 // Next unused projection number from Start.
736 int proj_cnt = C->tf()->domain()->cnt();
737
738 // Do all the save-on-entry registers. Make projections from Start for
739 // them, and give them a use at the exit points. To the allocator, they
740 // look like incoming register arguments.
741 for( i = 0; i < _last_Mach_Reg; i++ ) {
742 if( is_save_on_entry(i) ) {
743
744 // Add the save-on-entry to the mask array
745 ret_rms [ ret_edge_cnt] = mreg2regmask[i];
746 reth_rms [ reth_edge_cnt] = mreg2regmask[i];
747 tail_call_rms[tail_call_edge_cnt] = mreg2regmask[i];
748 tail_jump_rms[tail_jump_edge_cnt] = mreg2regmask[i];
749 // Halts need the SOE registers, but only in the stack as debug info.
750 // A just-prior uncommon-trap or deoptimization will use the SOE regs.
751 halt_rms [ halt_edge_cnt] = *idealreg2spillmask[_register_save_type[i]];
752
753 Node *mproj;
754
755 // Is this a RegF low half of a RegD? Double up 2 adjacent RegF's
756 // into a single RegD.
997 Node *oldn = n;
998 // Old-space or new-space check
999 if (!C->node_arena()->contains(n)) {
1000 // Old space!
1001 Node* m;
1002 if (has_new_node(n)) { // Not yet Label/Reduced
1003 m = new_node(n);
1004 } else {
1005 if (!is_dontcare(n)) { // Matcher can match this guy
1006 // Calls match special. They match alone with no children.
1007 // Their children, the incoming arguments, match normally.
1008 m = n->is_SafePoint() ? match_sfpt(n->as_SafePoint()):match_tree(n);
1009 if (C->failing()) return NULL;
1010 if (m == NULL) { Matcher::soft_match_failure(); return NULL; }
1011 if (n->is_MemBar()) {
1012 m->as_MachMemBar()->set_adr_type(n->adr_type());
1013 }
1014 } else { // Nothing the matcher cares about
1015 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Multi()) { // Projections?
1016 // Convert to machine-dependent projection
1017 m = n->in(0)->as_Multi()->match( n->as_Proj(), this );
1018 #ifdef ASSERT
1019 _new2old_map.map(m->_idx, n);
1020 #endif
1021 if (m->in(0) != NULL) // m might be top
1022 collect_null_checks(m, n);
1023 } else { // Else just a regular 'ol guy
1024 m = n->clone(); // So just clone into new-space
1025 #ifdef ASSERT
1026 _new2old_map.map(m->_idx, n);
1027 #endif
1028 // Def-Use edges will be added incrementally as Uses
1029 // of this node are matched.
1030 assert(m->outcnt() == 0, "no Uses of this clone yet");
1031 }
1032 }
1033
1034 set_new_node(n, m); // Map old to new
1035 if (_old_node_note_array != NULL) {
1036 Node_Notes* nn = C->locate_node_notes(_old_node_note_array,
1037 n->_idx);
1142 }
1143 return OptoReg::as_OptoReg(reg);
1144 }
1145
1146
1147 //------------------------------match_sfpt-------------------------------------
1148 // Helper function to match call instructions. Calls match special.
1149 // They match alone with no children. Their children, the incoming
1150 // arguments, match normally.
1151 MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) {
1152 MachSafePointNode *msfpt = NULL;
1153 MachCallNode *mcall = NULL;
1154 uint cnt;
1155 // Split out case for SafePoint vs Call
1156 CallNode *call;
1157 const TypeTuple *domain;
1158 ciMethod* method = NULL;
1159 bool is_method_handle_invoke = false; // for special kill effects
1160 if( sfpt->is_Call() ) {
1161 call = sfpt->as_Call();
1162 domain = call->tf()->domain();
1163 cnt = domain->cnt();
1164
1165 // Match just the call, nothing else
1166 MachNode *m = match_tree(call);
1167 if (C->failing()) return NULL;
1168 if( m == NULL ) { Matcher::soft_match_failure(); return NULL; }
1169
1170 // Copy data from the Ideal SafePoint to the machine version
1171 mcall = m->as_MachCall();
1172
1173 mcall->set_tf( call->tf());
1174 mcall->set_entry_point(call->entry_point());
1175 mcall->set_cnt( call->cnt());
1176
1177 if( mcall->is_MachCallJava() ) {
1178 MachCallJavaNode *mcall_java = mcall->as_MachCallJava();
1179 const CallJavaNode *call_java = call->as_CallJava();
1180 assert(call_java->validate_symbolic_info(), "inconsistent info");
1181 method = call_java->method();
1182 mcall_java->_method = method;
1217 msfpt->_in_rms = NEW_RESOURCE_ARRAY( RegMask, cnt );
1218 // Empty them all.
1219 for (uint i = 0; i < cnt; i++) ::new (&(msfpt->_in_rms[i])) RegMask();
1220
1221 // Do all the pre-defined non-Empty register masks
1222 msfpt->_in_rms[TypeFunc::ReturnAdr] = _return_addr_mask;
1223 msfpt->_in_rms[TypeFunc::FramePtr ] = c_frame_ptr_mask;
1224
1225 // Place first outgoing argument can possibly be put.
1226 OptoReg::Name begin_out_arg_area = OptoReg::add(_new_SP, C->out_preserve_stack_slots());
1227 assert( is_even(begin_out_arg_area), "" );
1228 // Compute max outgoing register number per call site.
1229 OptoReg::Name out_arg_limit_per_call = begin_out_arg_area;
1230 // Calls to C may hammer extra stack slots above and beyond any arguments.
1231 // These are usually backing store for register arguments for varargs.
1232 if( call != NULL && call->is_CallRuntime() )
1233 out_arg_limit_per_call = OptoReg::add(out_arg_limit_per_call,C->varargs_C_out_slots_killed());
1234
1235
1236 // Do the normal argument list (parameters) register masks
1237 int argcnt = cnt - TypeFunc::Parms;
1238 if( argcnt > 0 ) { // Skip it all if we have no args
1239 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
1240 VMRegPair *parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
1241 int i;
1242 for( i = 0; i < argcnt; i++ ) {
1243 sig_bt[i] = domain->field_at(i+TypeFunc::Parms)->basic_type();
1244 }
1245 // V-call to pick proper calling convention
1246 call->calling_convention( sig_bt, parm_regs, argcnt );
1247
1248 #ifdef ASSERT
1249 // Sanity check users' calling convention. Really handy during
1250 // the initial porting effort. Fairly expensive otherwise.
1251 { for (int i = 0; i<argcnt; i++) {
1252 if( !parm_regs[i].first()->is_valid() &&
1253 !parm_regs[i].second()->is_valid() ) continue;
1254 VMReg reg1 = parm_regs[i].first();
1255 VMReg reg2 = parm_regs[i].second();
1256 for (int j = 0; j < i; j++) {
1257 if( !parm_regs[j].first()->is_valid() &&
1258 !parm_regs[j].second()->is_valid() ) continue;
1259 VMReg reg3 = parm_regs[j].first();
1260 VMReg reg4 = parm_regs[j].second();
1261 if( !reg1->is_valid() ) {
1262 assert( !reg2->is_valid(), "valid halvsies" );
1263 } else if( !reg3->is_valid() ) {
1264 assert( !reg4->is_valid(), "valid halvsies" );
1265 } else {
1266 assert( reg1 != reg2, "calling conv. must produce distinct regs");
1267 assert( reg1 != reg3, "calling conv. must produce distinct regs");
1268 assert( reg1 != reg4, "calling conv. must produce distinct regs");
1269 assert( reg2 != reg3, "calling conv. must produce distinct regs");
1270 assert( reg2 != reg4 || !reg2->is_valid(), "calling conv. must produce distinct regs");
1271 assert( reg3 != reg4, "calling conv. must produce distinct regs");
1272 }
1273 }
1274 }
1275 }
1276 #endif
1277
1278 // Visit each argument. Compute its outgoing register mask.
1279 // Return results now can have 2 bits returned.
1280 // Compute max over all outgoing arguments both per call-site
1281 // and over the entire method.
1282 for( i = 0; i < argcnt; i++ ) {
1283 // Address of incoming argument mask to fill in
1284 RegMask *rm = &mcall->_in_rms[i+TypeFunc::Parms];
1285 if( !parm_regs[i].first()->is_valid() &&
1286 !parm_regs[i].second()->is_valid() ) {
1287 continue; // Avoid Halves
1288 }
1289 // Grab first register, adjust stack slots and insert in mask.
1290 OptoReg::Name reg1 = warp_outgoing_stk_arg(parm_regs[i].first(), begin_out_arg_area, out_arg_limit_per_call );
1291 if (OptoReg::is_valid(reg1))
1292 rm->Insert( reg1 );
1293 // Grab second register (if any), adjust stack slots and insert in mask.
1294 OptoReg::Name reg2 = warp_outgoing_stk_arg(parm_regs[i].second(), begin_out_arg_area, out_arg_limit_per_call );
1295 if (OptoReg::is_valid(reg2))
1296 rm->Insert( reg2 );
1297 } // End of for all arguments
1298
1299 // Compute number of stack slots needed to restore stack in case of
1300 // Pascal-style argument popping.
1301 mcall->_argsize = out_arg_limit_per_call - begin_out_arg_area;
1302 }
1303
1304 // Compute the max stack slot killed by any call. These will not be
1305 // available for debug info, and will be used to adjust FIRST_STACK_mask
1306 // after all call sites have been visited.
1307 if( _out_arg_limit < out_arg_limit_per_call)
1308 _out_arg_limit = out_arg_limit_per_call;
1309
1310 if (mcall) {
1311 // Kill the outgoing argument area, including any non-argument holes and
1312 // any legacy C-killed slots. Use Fat-Projections to do the killing.
1313 // Since the max-per-method covers the max-per-call-site and debug info
1314 // is excluded on the max-per-method basis, debug info cannot land in
1315 // this killed area.
1316 uint r_cnt = mcall->tf()->range()->cnt();
1317 MachProjNode *proj = new MachProjNode( mcall, r_cnt+10000, RegMask::Empty, MachProjNode::fat_proj );
1318 if (!RegMask::can_represent_arg(OptoReg::Name(out_arg_limit_per_call-1))) {
1319 C->record_method_not_compilable("unsupported outgoing calling sequence");
1320 } else {
1321 for (int i = begin_out_arg_area; i < out_arg_limit_per_call; i++)
1322 proj->_rout.Insert(OptoReg::Name(i));
1323 }
1324 if (proj->_rout.is_NotEmpty()) {
1325 push_projection(proj);
1326 }
1327 }
1328 // Transfer the safepoint information from the call to the mcall
1329 // Move the JVMState list
1330 msfpt->set_jvms(sfpt->jvms());
1331 for (JVMState* jvms = msfpt->jvms(); jvms; jvms = jvms->caller()) {
1332 jvms->set_map(sfpt);
1333 }
1334
1335 // Debug inputs begin just after the last incoming parameter
1336 assert((mcall == NULL) || (mcall->jvms() == NULL) ||
1337 (mcall->jvms()->debug_start() + mcall->_jvmadj == mcall->tf()->domain()->cnt()), "");
1338
1339 // Move the OopMap
1340 msfpt->_oop_map = sfpt->_oop_map;
1341
1342 // Add additional edges.
1343 if (msfpt->mach_constant_base_node_input() != (uint)-1 && !msfpt->is_MachCallLeaf()) {
1344 // For these calls we can not add MachConstantBase in expand(), as the
1345 // ins are not complete then.
1346 msfpt->ins_req(msfpt->mach_constant_base_node_input(), C->mach_constant_base_node());
1347 if (msfpt->jvms() &&
1348 msfpt->mach_constant_base_node_input() <= msfpt->jvms()->debug_start() + msfpt->_jvmadj) {
1349 // We added an edge before jvms, so we must adapt the position of the ins.
1350 msfpt->jvms()->adapt_position(+1);
1351 }
1352 }
1353
1354 // Registers killed by the call are set in the local scheduling pass
1355 // of Global Code Motion.
1356 return msfpt;
1357 }
2349 break;
2350 }
2351 case Op_FmaD:
2352 case Op_FmaF:
2353 case Op_FmaVD:
2354 case Op_FmaVF: {
2355 // Restructure into a binary tree for Matching.
2356 Node* pair = new BinaryNode(n->in(1), n->in(2));
2357 n->set_req(2, pair);
2358 n->set_req(1, n->in(3));
2359 n->del_req(3);
2360 break;
2361 }
2362 case Op_MulAddS2I: {
2363 Node* pair1 = new BinaryNode(n->in(1), n->in(2));
2364 Node* pair2 = new BinaryNode(n->in(3), n->in(4));
2365 n->set_req(1, pair1);
2366 n->set_req(2, pair2);
2367 n->del_req(4);
2368 n->del_req(3);
2369 break;
2370 }
2371 default:
2372 break;
2373 }
2374 }
2375
2376 #ifdef ASSERT
2377 // machine-independent root to machine-dependent root
2378 void Matcher::dump_old2new_map() {
2379 _old2new_map.dump();
2380 }
2381 #endif
2382
2383 //---------------------------collect_null_checks-------------------------------
2384 // Find null checks in the ideal graph; write a machine-specific node for
2385 // it. Used by later implicit-null-check handling. Actually collects
2386 // either an IfTrue or IfFalse for the common NOT-null path, AND the ideal
2387 // value being tested.
2388 void Matcher::collect_null_checks( Node *proj, Node *orig_proj ) {
|
155 Unique_Node_List worklist;
156 VectorSet visited(Thread::current()->resource_area());
157 worklist.push(xroot);
158 while (worklist.size() > 0) {
159 Node* n = worklist.pop();
160 visited <<= n->_idx;
161 assert(C->node_arena()->contains(n), "dead node");
162 for (uint j = 0; j < n->req(); j++) {
163 Node* in = n->in(j);
164 if (in != NULL) {
165 assert(C->node_arena()->contains(in), "dead node");
166 if (!visited.test(in->_idx)) {
167 worklist.push(in);
168 }
169 }
170 }
171 }
172 }
173 #endif
174
175 // Array of RegMask, one per returned values (value type instances can
176 // be returned as multiple return values, one per field)
177 RegMask* Matcher::return_values_mask(const TypeTuple *range) {
178 uint cnt = range->cnt() - TypeFunc::Parms;
179 if (cnt == 0) {
180 return NULL;
181 }
182 RegMask* mask = NEW_RESOURCE_ARRAY(RegMask, cnt);
183
184 if (!ValueTypeReturnedAsFields) {
185 // Get ideal-register return type
186 uint ireg = range->field_at(TypeFunc::Parms)->ideal_reg();
187 // Get machine return register
188 OptoRegPair regs = return_value(ireg, false);
189
190 // And mask for same
191 mask[0].Clear();
192 mask[0].Insert(regs.first());
193 if (OptoReg::is_valid(regs.second())) {
194 mask[0].Insert(regs.second());
195 }
196 } else {
197 BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, cnt);
198 VMRegPair* vm_parm_regs = NEW_RESOURCE_ARRAY(VMRegPair, cnt);
199
200 for (uint i = 0; i < cnt; i++) {
201 sig_bt[i] = range->field_at(i+TypeFunc::Parms)->basic_type();
202 }
203
204 int regs = SharedRuntime::java_return_convention(sig_bt, vm_parm_regs, cnt);
205 assert(regs > 0, "should have been tested during graph construction");
206 for (uint i = 0; i < cnt; i++) {
207 mask[i].Clear();
208
209 OptoReg::Name reg1 = OptoReg::as_OptoReg(vm_parm_regs[i].first());
210 if (OptoReg::is_valid(reg1)) {
211 mask[i].Insert(reg1);
212 }
213 OptoReg::Name reg2 = OptoReg::as_OptoReg(vm_parm_regs[i].second());
214 if (OptoReg::is_valid(reg2)) {
215 mask[i].Insert(reg2);
216 }
217 }
218 }
219 return mask;
220 }
221
222 //---------------------------match---------------------------------------------
223 void Matcher::match( ) {
224 if( MaxLabelRootDepth < 100 ) { // Too small?
225 assert(false, "invalid MaxLabelRootDepth, increase it to 100 minimum");
226 MaxLabelRootDepth = 100;
227 }
228 // One-time initialization of some register masks.
229 init_spill_mask( C->root()->in(1) );
230 _return_addr_mask = return_addr();
231 #ifdef _LP64
232 // Pointers take 2 slots in 64-bit land
233 _return_addr_mask.Insert(OptoReg::add(return_addr(),1));
234 #endif
235
236 // Map Java-signature return types into return register-value
237 // machine registers.
238 const TypeTuple *range = C->tf()->range_cc();
239 _return_values_mask = return_values_mask(range);
240
241 // ---------------
242 // Frame Layout
243
244 // Need the method signature to determine the incoming argument types,
245 // because the types determine which registers the incoming arguments are
246 // in, and this affects the matched code.
247 const TypeTuple *domain = C->tf()->domain_cc();
248 uint argcnt = domain->cnt() - TypeFunc::Parms;
249 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
250 VMRegPair *vm_parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
251 _parm_regs = NEW_RESOURCE_ARRAY( OptoRegPair, argcnt );
252 _calling_convention_mask = NEW_RESOURCE_ARRAY( RegMask, argcnt );
253 uint i;
254 for( i = 0; i<argcnt; i++ ) {
255 sig_bt[i] = domain->field_at(i+TypeFunc::Parms)->basic_type();
256 }
257
258 // Pass array of ideal registers and length to USER code (from the AD file)
259 // that will convert this to an array of register numbers.
260 const StartNode *start = C->start();
261 start->calling_convention( sig_bt, vm_parm_regs, argcnt );
262 #ifdef ASSERT
263 // Sanity check users' calling convention. Real handy while trying to
264 // get the initial port correct.
265 { for (uint i = 0; i<argcnt; i++) {
266 if( !vm_parm_regs[i].first()->is_valid() && !vm_parm_regs[i].second()->is_valid() ) {
267 assert(domain->field_at(i+TypeFunc::Parms)==Type::HALF, "only allowed on halve" );
487 idealreg2mhdebugmask[Op_RegF] = &rms[15];
488 idealreg2mhdebugmask[Op_RegD] = &rms[16];
489 idealreg2mhdebugmask[Op_RegP] = &rms[17];
490
491 idealreg2spillmask [Op_VecS] = &rms[18];
492 idealreg2spillmask [Op_VecD] = &rms[19];
493 idealreg2spillmask [Op_VecX] = &rms[20];
494 idealreg2spillmask [Op_VecY] = &rms[21];
495 idealreg2spillmask [Op_VecZ] = &rms[22];
496
497 OptoReg::Name i;
498
499 // At first, start with the empty mask
500 C->FIRST_STACK_mask().Clear();
501
502 // Add in the incoming argument area
503 OptoReg::Name init_in = OptoReg::add(_old_SP, C->out_preserve_stack_slots());
504 for (i = init_in; i < _in_arg_limit; i = OptoReg::add(i,1)) {
505 C->FIRST_STACK_mask().Insert(i);
506 }
507
508 // Check if the method has a reserved entry in the argument stack area that
509 // should not be used for spilling because it may hold the return address.
510 if (C->method() != NULL && C->method()->has_scalarized_args()) {
511 ExtendedSignature sig_cc = ExtendedSignature(C->method()->get_sig_cc(), SigEntryFilter());
512 for (int off = 0; !sig_cc.at_end(); ) {
513 BasicType bt = (*sig_cc)._bt;
514 off += type2size[bt];
515 while (SigEntry::next_is_reserved(sig_cc, bt)) {
516 // Remove reserved stack slot from mask to avoid spilling
517 OptoRegPair reg = _parm_regs[off];
518 assert(OptoReg::is_valid(reg.first()), "invalid reserved register");
519 C->FIRST_STACK_mask().Remove(reg.first());
520 C->FIRST_STACK_mask().Remove(reg.first()+1); // Always occupies two stack slots
521 off += type2size[bt];
522 }
523 }
524 }
525
526 // Add in all bits past the outgoing argument area
527 guarantee(RegMask::can_represent_arg(OptoReg::add(_out_arg_limit,-1)),
528 "must be able to represent all call arguments in reg mask");
529 OptoReg::Name init = _out_arg_limit;
530 for (i = init; RegMask::can_represent(i); i = OptoReg::add(i,1)) {
531 C->FIRST_STACK_mask().Insert(i);
532 }
533 // Finally, set the "infinite stack" bit.
534 C->FIRST_STACK_mask().set_AllStack();
535
536 // Make spill masks. Registers for their class, plus FIRST_STACK_mask.
537 RegMask aligned_stack_mask = C->FIRST_STACK_mask();
538 // Keep spill masks aligned.
539 aligned_stack_mask.clear_to_pairs();
540 assert(aligned_stack_mask.is_AllStack(), "should be infinite stack");
541
542 *idealreg2spillmask[Op_RegP] = *idealreg2regmask[Op_RegP];
543 #ifdef _LP64
544 *idealreg2spillmask[Op_RegN] = *idealreg2regmask[Op_RegN];
545 idealreg2spillmask[Op_RegN]->OR(C->FIRST_STACK_mask());
698 init_first_stack_mask();
699
700 Node *root = C->root(); // Short name for root
701 // Count number of save-on-entry registers.
702 uint soe_cnt = number_of_saved_registers();
703 uint i;
704
705 // Find the procedure Start Node
706 StartNode *start = C->start();
707 assert( start, "Expect a start node" );
708
709 // Save argument registers in the trampolining stubs
710 if( C->save_argument_registers() )
711 for( i = 0; i < _last_Mach_Reg; i++ )
712 if( is_spillable_arg(i) )
713 soe_cnt++;
714
715 // Input RegMask array shared by all Returns.
716 // The type for doubles and longs has a count of 2, but
717 // there is only 1 returned value
718 uint ret_edge_cnt = C->tf()->range_cc()->cnt();
719 RegMask *ret_rms = init_input_masks( ret_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
720 for (i = TypeFunc::Parms; i < ret_edge_cnt; i++) {
721 ret_rms[i] = _return_values_mask[i-TypeFunc::Parms];
722 }
723
724 // Input RegMask array shared by all Rethrows.
725 uint reth_edge_cnt = TypeFunc::Parms+1;
726 RegMask *reth_rms = init_input_masks( reth_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
727 // Rethrow takes exception oop only, but in the argument 0 slot.
728 OptoReg::Name reg = find_receiver(false);
729 if (reg >= 0) {
730 reth_rms[TypeFunc::Parms] = mreg2regmask[reg];
731 #ifdef _LP64
732 // Need two slots for ptrs in 64-bit land
733 reth_rms[TypeFunc::Parms].Insert(OptoReg::add(OptoReg::Name(reg), 1));
734 #endif
735 }
736
737 // Input RegMask array shared by all TailCalls
738 uint tail_call_edge_cnt = TypeFunc::Parms+2;
739 RegMask *tail_call_rms = init_input_masks( tail_call_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
740
741 // Input RegMask array shared by all TailJumps
742 uint tail_jump_edge_cnt = TypeFunc::Parms+2;
769 }
770
771 // Input RegMask array shared by all Halts
772 uint halt_edge_cnt = TypeFunc::Parms;
773 RegMask *halt_rms = init_input_masks( halt_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
774
775 // Capture the return input masks into each exit flavor
776 for( i=1; i < root->req(); i++ ) {
777 MachReturnNode *exit = root->in(i)->as_MachReturn();
778 switch( exit->ideal_Opcode() ) {
779 case Op_Return : exit->_in_rms = ret_rms; break;
780 case Op_Rethrow : exit->_in_rms = reth_rms; break;
781 case Op_TailCall : exit->_in_rms = tail_call_rms; break;
782 case Op_TailJump : exit->_in_rms = tail_jump_rms; break;
783 case Op_Halt : exit->_in_rms = halt_rms; break;
784 default : ShouldNotReachHere();
785 }
786 }
787
788 // Next unused projection number from Start.
789 int proj_cnt = C->tf()->domain_cc()->cnt();
790
791 // Do all the save-on-entry registers. Make projections from Start for
792 // them, and give them a use at the exit points. To the allocator, they
793 // look like incoming register arguments.
794 for( i = 0; i < _last_Mach_Reg; i++ ) {
795 if( is_save_on_entry(i) ) {
796
797 // Add the save-on-entry to the mask array
798 ret_rms [ ret_edge_cnt] = mreg2regmask[i];
799 reth_rms [ reth_edge_cnt] = mreg2regmask[i];
800 tail_call_rms[tail_call_edge_cnt] = mreg2regmask[i];
801 tail_jump_rms[tail_jump_edge_cnt] = mreg2regmask[i];
802 // Halts need the SOE registers, but only in the stack as debug info.
803 // A just-prior uncommon-trap or deoptimization will use the SOE regs.
804 halt_rms [ halt_edge_cnt] = *idealreg2spillmask[_register_save_type[i]];
805
806 Node *mproj;
807
808 // Is this a RegF low half of a RegD? Double up 2 adjacent RegF's
809 // into a single RegD.
1050 Node *oldn = n;
1051 // Old-space or new-space check
1052 if (!C->node_arena()->contains(n)) {
1053 // Old space!
1054 Node* m;
1055 if (has_new_node(n)) { // Not yet Label/Reduced
1056 m = new_node(n);
1057 } else {
1058 if (!is_dontcare(n)) { // Matcher can match this guy
1059 // Calls match special. They match alone with no children.
1060 // Their children, the incoming arguments, match normally.
1061 m = n->is_SafePoint() ? match_sfpt(n->as_SafePoint()):match_tree(n);
1062 if (C->failing()) return NULL;
1063 if (m == NULL) { Matcher::soft_match_failure(); return NULL; }
1064 if (n->is_MemBar()) {
1065 m->as_MachMemBar()->set_adr_type(n->adr_type());
1066 }
1067 } else { // Nothing the matcher cares about
1068 if (n->is_Proj() && n->in(0) != NULL && n->in(0)->is_Multi()) { // Projections?
1069 // Convert to machine-dependent projection
1070 RegMask* mask = NULL;
1071 if (n->in(0)->is_Call()) {
1072 mask = return_values_mask(n->in(0)->as_Call()->tf()->range_cc());
1073 }
1074 m = n->in(0)->as_Multi()->match(n->as_Proj(), this, mask);
1075 #ifdef ASSERT
1076 _new2old_map.map(m->_idx, n);
1077 #endif
1078 if (m->in(0) != NULL) // m might be top
1079 collect_null_checks(m, n);
1080 } else { // Else just a regular 'ol guy
1081 m = n->clone(); // So just clone into new-space
1082 #ifdef ASSERT
1083 _new2old_map.map(m->_idx, n);
1084 #endif
1085 // Def-Use edges will be added incrementally as Uses
1086 // of this node are matched.
1087 assert(m->outcnt() == 0, "no Uses of this clone yet");
1088 }
1089 }
1090
1091 set_new_node(n, m); // Map old to new
1092 if (_old_node_note_array != NULL) {
1093 Node_Notes* nn = C->locate_node_notes(_old_node_note_array,
1094 n->_idx);
1199 }
1200 return OptoReg::as_OptoReg(reg);
1201 }
1202
1203
1204 //------------------------------match_sfpt-------------------------------------
1205 // Helper function to match call instructions. Calls match special.
1206 // They match alone with no children. Their children, the incoming
1207 // arguments, match normally.
1208 MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) {
1209 MachSafePointNode *msfpt = NULL;
1210 MachCallNode *mcall = NULL;
1211 uint cnt;
1212 // Split out case for SafePoint vs Call
1213 CallNode *call;
1214 const TypeTuple *domain;
1215 ciMethod* method = NULL;
1216 bool is_method_handle_invoke = false; // for special kill effects
1217 if( sfpt->is_Call() ) {
1218 call = sfpt->as_Call();
1219 domain = call->tf()->domain_cc();
1220 cnt = domain->cnt();
1221
1222 // Match just the call, nothing else
1223 MachNode *m = match_tree(call);
1224 if (C->failing()) return NULL;
1225 if( m == NULL ) { Matcher::soft_match_failure(); return NULL; }
1226
1227 // Copy data from the Ideal SafePoint to the machine version
1228 mcall = m->as_MachCall();
1229
1230 mcall->set_tf( call->tf());
1231 mcall->set_entry_point(call->entry_point());
1232 mcall->set_cnt( call->cnt());
1233
1234 if( mcall->is_MachCallJava() ) {
1235 MachCallJavaNode *mcall_java = mcall->as_MachCallJava();
1236 const CallJavaNode *call_java = call->as_CallJava();
1237 assert(call_java->validate_symbolic_info(), "inconsistent info");
1238 method = call_java->method();
1239 mcall_java->_method = method;
1274 msfpt->_in_rms = NEW_RESOURCE_ARRAY( RegMask, cnt );
1275 // Empty them all.
1276 for (uint i = 0; i < cnt; i++) ::new (&(msfpt->_in_rms[i])) RegMask();
1277
1278 // Do all the pre-defined non-Empty register masks
1279 msfpt->_in_rms[TypeFunc::ReturnAdr] = _return_addr_mask;
1280 msfpt->_in_rms[TypeFunc::FramePtr ] = c_frame_ptr_mask;
1281
1282 // Place first outgoing argument can possibly be put.
1283 OptoReg::Name begin_out_arg_area = OptoReg::add(_new_SP, C->out_preserve_stack_slots());
1284 assert( is_even(begin_out_arg_area), "" );
1285 // Compute max outgoing register number per call site.
1286 OptoReg::Name out_arg_limit_per_call = begin_out_arg_area;
1287 // Calls to C may hammer extra stack slots above and beyond any arguments.
1288 // These are usually backing store for register arguments for varargs.
1289 if( call != NULL && call->is_CallRuntime() )
1290 out_arg_limit_per_call = OptoReg::add(out_arg_limit_per_call,C->varargs_C_out_slots_killed());
1291
1292
1293 // Do the normal argument list (parameters) register masks
1294 // Null entry point is a special cast where the target of the call
1295 // is in a register.
1296 int adj = (call != NULL && call->entry_point() == NULL) ? 1 : 0;
1297 int argcnt = cnt - TypeFunc::Parms - adj;
1298 if( argcnt > 0 ) { // Skip it all if we have no args
1299 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
1300 VMRegPair *parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
1301 int i;
1302 for( i = 0; i < argcnt; i++ ) {
1303 sig_bt[i] = domain->field_at(i+TypeFunc::Parms+adj)->basic_type();
1304 }
1305 // V-call to pick proper calling convention
1306 call->calling_convention( sig_bt, parm_regs, argcnt );
1307
1308 #ifdef ASSERT
1309 // Sanity check users' calling convention. Really handy during
1310 // the initial porting effort. Fairly expensive otherwise.
1311 { for (int i = 0; i<argcnt; i++) {
1312 if( !parm_regs[i].first()->is_valid() &&
1313 !parm_regs[i].second()->is_valid() ) continue;
1314 VMReg reg1 = parm_regs[i].first();
1315 VMReg reg2 = parm_regs[i].second();
1316 for (int j = 0; j < i; j++) {
1317 if( !parm_regs[j].first()->is_valid() &&
1318 !parm_regs[j].second()->is_valid() ) continue;
1319 VMReg reg3 = parm_regs[j].first();
1320 VMReg reg4 = parm_regs[j].second();
1321 if( !reg1->is_valid() ) {
1322 assert( !reg2->is_valid(), "valid halvsies" );
1323 } else if( !reg3->is_valid() ) {
1324 assert( !reg4->is_valid(), "valid halvsies" );
1325 } else {
1326 assert( reg1 != reg2, "calling conv. must produce distinct regs");
1327 assert( reg1 != reg3, "calling conv. must produce distinct regs");
1328 assert( reg1 != reg4, "calling conv. must produce distinct regs");
1329 assert( reg2 != reg3, "calling conv. must produce distinct regs");
1330 assert( reg2 != reg4 || !reg2->is_valid(), "calling conv. must produce distinct regs");
1331 assert( reg3 != reg4, "calling conv. must produce distinct regs");
1332 }
1333 }
1334 }
1335 }
1336 #endif
1337
1338 // Visit each argument. Compute its outgoing register mask.
1339 // Return results now can have 2 bits returned.
1340 // Compute max over all outgoing arguments both per call-site
1341 // and over the entire method.
1342 for( i = 0; i < argcnt; i++ ) {
1343 // Address of incoming argument mask to fill in
1344 RegMask *rm = &mcall->_in_rms[i+TypeFunc::Parms+adj];
1345 if( !parm_regs[i].first()->is_valid() &&
1346 !parm_regs[i].second()->is_valid() ) {
1347 continue; // Avoid Halves
1348 }
1349 // Grab first register, adjust stack slots and insert in mask.
1350 OptoReg::Name reg1 = warp_outgoing_stk_arg(parm_regs[i].first(), begin_out_arg_area, out_arg_limit_per_call );
1351 if (OptoReg::is_valid(reg1)) {
1352 rm->Insert( reg1 );
1353 }
1354 // Grab second register (if any), adjust stack slots and insert in mask.
1355 OptoReg::Name reg2 = warp_outgoing_stk_arg(parm_regs[i].second(), begin_out_arg_area, out_arg_limit_per_call );
1356 if (OptoReg::is_valid(reg2)) {
1357 rm->Insert( reg2 );
1358 }
1359 } // End of for all arguments
1360
1361 // Compute number of stack slots needed to restore stack in case of
1362 // Pascal-style argument popping.
1363 mcall->_argsize = out_arg_limit_per_call - begin_out_arg_area;
1364 }
1365
1366 // Compute the max stack slot killed by any call. These will not be
1367 // available for debug info, and will be used to adjust FIRST_STACK_mask
1368 // after all call sites have been visited.
1369 if( _out_arg_limit < out_arg_limit_per_call)
1370 _out_arg_limit = out_arg_limit_per_call;
1371
1372 if (mcall) {
1373 // Kill the outgoing argument area, including any non-argument holes and
1374 // any legacy C-killed slots. Use Fat-Projections to do the killing.
1375 // Since the max-per-method covers the max-per-call-site and debug info
1376 // is excluded on the max-per-method basis, debug info cannot land in
1377 // this killed area.
1378 uint r_cnt = mcall->tf()->range_sig()->cnt();
1379 MachProjNode *proj = new MachProjNode( mcall, r_cnt+10000, RegMask::Empty, MachProjNode::fat_proj );
1380 if (!RegMask::can_represent_arg(OptoReg::Name(out_arg_limit_per_call-1))) {
1381 C->record_method_not_compilable("unsupported outgoing calling sequence");
1382 } else {
1383 for (int i = begin_out_arg_area; i < out_arg_limit_per_call; i++)
1384 proj->_rout.Insert(OptoReg::Name(i));
1385 }
1386 if (proj->_rout.is_NotEmpty()) {
1387 push_projection(proj);
1388 }
1389 }
1390 // Transfer the safepoint information from the call to the mcall
1391 // Move the JVMState list
1392 msfpt->set_jvms(sfpt->jvms());
1393 for (JVMState* jvms = msfpt->jvms(); jvms; jvms = jvms->caller()) {
1394 jvms->set_map(sfpt);
1395 }
1396
1397 // Debug inputs begin just after the last incoming parameter
1398 assert((mcall == NULL) || (mcall->jvms() == NULL) ||
1399 (mcall->jvms()->debug_start() + mcall->_jvmadj == mcall->tf()->domain_cc()->cnt()), "");
1400
1401 // Move the OopMap
1402 msfpt->_oop_map = sfpt->_oop_map;
1403
1404 // Add additional edges.
1405 if (msfpt->mach_constant_base_node_input() != (uint)-1 && !msfpt->is_MachCallLeaf()) {
1406 // For these calls we can not add MachConstantBase in expand(), as the
1407 // ins are not complete then.
1408 msfpt->ins_req(msfpt->mach_constant_base_node_input(), C->mach_constant_base_node());
1409 if (msfpt->jvms() &&
1410 msfpt->mach_constant_base_node_input() <= msfpt->jvms()->debug_start() + msfpt->_jvmadj) {
1411 // We added an edge before jvms, so we must adapt the position of the ins.
1412 msfpt->jvms()->adapt_position(+1);
1413 }
1414 }
1415
1416 // Registers killed by the call are set in the local scheduling pass
1417 // of Global Code Motion.
1418 return msfpt;
1419 }
2411 break;
2412 }
2413 case Op_FmaD:
2414 case Op_FmaF:
2415 case Op_FmaVD:
2416 case Op_FmaVF: {
2417 // Restructure into a binary tree for Matching.
2418 Node* pair = new BinaryNode(n->in(1), n->in(2));
2419 n->set_req(2, pair);
2420 n->set_req(1, n->in(3));
2421 n->del_req(3);
2422 break;
2423 }
2424 case Op_MulAddS2I: {
2425 Node* pair1 = new BinaryNode(n->in(1), n->in(2));
2426 Node* pair2 = new BinaryNode(n->in(3), n->in(4));
2427 n->set_req(1, pair1);
2428 n->set_req(2, pair2);
2429 n->del_req(4);
2430 n->del_req(3);
2431 break;
2432 }
2433 case Op_ClearArray: {
2434 Node* pair = new BinaryNode(n->in(2), n->in(3));
2435 n->set_req(2, pair);
2436 n->set_req(3, n->in(4));
2437 n->del_req(4);
2438 break;
2439 }
2440 default:
2441 break;
2442 }
2443 }
2444
2445 #ifdef ASSERT
2446 // machine-independent root to machine-dependent root
2447 void Matcher::dump_old2new_map() {
2448 _old2new_map.dump();
2449 }
2450 #endif
2451
2452 //---------------------------collect_null_checks-------------------------------
2453 // Find null checks in the ideal graph; write a machine-specific node for
2454 // it. Used by later implicit-null-check handling. Actually collects
2455 // either an IfTrue or IfFalse for the common NOT-null path, AND the ideal
2456 // value being tested.
2457 void Matcher::collect_null_checks( Node *proj, Node *orig_proj ) {
|