149 Node* n = worklist.pop();
150 if (visited.test_set(n->_idx)) {
151 continue;
152 }
153 assert(C->node_arena()->contains(n), "dead node");
154 assert(!n->is_Initialize() || n->as_Initialize()->number_of_projs(TypeFunc::Memory) == 1,
155 "after matching, Initialize should have a single memory projection");
156 for (uint j = 0; j < n->req(); j++) {
157 Node* in = n->in(j);
158 if (in != nullptr) {
159 worklist.push(in);
160 }
161 }
162 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
163 worklist.push(n->fast_out(j));
164 }
165 }
166 }
167 #endif
168
169
170 //---------------------------match---------------------------------------------
171 void Matcher::match( ) {
172 if( MaxLabelRootDepth < 100 ) { // Too small?
173 assert(false, "invalid MaxLabelRootDepth, increase it to 100 minimum");
174 MaxLabelRootDepth = 100;
175 }
176 // One-time initialization of some register masks.
177 init_spill_mask( C->root()->in(1) );
178 if (C->failing()) {
179 return;
180 }
181 assert(_return_addr_mask.is_empty(),
182 "return address mask must be empty initially");
183 _return_addr_mask.insert(return_addr());
184 #ifdef _LP64
185 // Pointers take 2 slots in 64-bit land
186 _return_addr_mask.insert(OptoReg::add(return_addr(), 1));
187 #endif
188
189 // Map a Java-signature return type into return register-value
190 // machine registers for 0, 1 and 2 returned values.
191 const TypeTuple *range = C->tf()->range();
192 if( range->cnt() > TypeFunc::Parms ) { // If not a void function
193 // Get ideal-register return type
194 uint ireg = range->field_at(TypeFunc::Parms)->ideal_reg();
195 // Get machine return register
196 uint sop = C->start()->Opcode();
197 OptoRegPair regs = return_value(ireg);
198
199 // And mask for same
200 _return_value_mask.assignFrom(RegMask(regs.first()));
201 if( OptoReg::is_valid(regs.second()) )
202 _return_value_mask.insert(regs.second());
203 }
204
205 // ---------------
206 // Frame Layout
207
208 // Need the method signature to determine the incoming argument types,
209 // because the types determine which registers the incoming arguments are
210 // in, and this affects the matched code.
211 const TypeTuple *domain = C->tf()->domain();
212 uint argcnt = domain->cnt() - TypeFunc::Parms;
213 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
214 VMRegPair *vm_parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
215 _parm_regs = NEW_RESOURCE_ARRAY( OptoRegPair, argcnt );
216 _calling_convention_mask = NEW_RESOURCE_ARRAY( RegMask, argcnt );
217 uint i;
218 for( i = 0; i<argcnt; i++ ) {
219 sig_bt[i] = domain->field_at(i+TypeFunc::Parms)->basic_type();
220 new (_calling_convention_mask + i) RegMask(C->comp_arena());
221 }
222
223 // Pass array of ideal registers and length to USER code (from the AD file)
224 // that will convert this to an array of register numbers.
225 const StartNode *start = C->start();
226 start->calling_convention( sig_bt, vm_parm_regs, argcnt );
227 #ifdef ASSERT
228 // Sanity check users' calling convention. Real handy while trying to
229 // get the initial port correct.
230 { for (uint i = 0; i<argcnt; i++) {
231 if( !vm_parm_regs[i].first()->is_valid() && !vm_parm_regs[i].second()->is_valid() ) {
463 // Initialize empty placeholder masks into the newly allocated arena
464 for (int i = 0; i < NOF_STACK_MASKS; i++) {
465 new (rms + i) RegMask(C->comp_arena());
466 }
467
468 int index = 0;
469 for (int i = Op_RegN; i <= Op_RegVectMask; ++i) {
470 idealreg2spillmask[i] = &rms[index++];
471 idealreg2debugmask[i] = &rms[index++];
472 }
473 assert(index == NOF_STACK_MASKS, "wrong size");
474
475 // At first, start with the empty mask
476 C->FIRST_STACK_mask().clear();
477
478 // Add in the incoming argument area
479 OptoReg::Name init_in = OptoReg::add(_old_SP, C->out_preserve_stack_slots());
480 for (OptoReg::Name i = init_in; i < _in_arg_limit; i = OptoReg::add(i, 1)) {
481 C->FIRST_STACK_mask().insert(i);
482 }
483 // Add in all bits past the outgoing argument area
484 C->FIRST_STACK_mask().set_all_from(_out_arg_limit);
485
486 // Make spill masks. Registers for their class, plus FIRST_STACK_mask.
487 RegMask aligned_stack_mask(C->FIRST_STACK_mask(), C->comp_arena());
488 // Keep spill masks aligned.
489 aligned_stack_mask.clear_to_pairs();
490 assert(aligned_stack_mask.is_infinite_stack(), "should be infinite stack");
491 RegMask scalable_stack_mask(aligned_stack_mask, C->comp_arena());
492
493 idealreg2spillmask[Op_RegP]->assignFrom(*idealreg2regmask[Op_RegP]);
494 #ifdef _LP64
495 idealreg2spillmask[Op_RegN]->assignFrom(*idealreg2regmask[Op_RegN]);
496 idealreg2spillmask[Op_RegN]->or_with(C->FIRST_STACK_mask());
497 idealreg2spillmask[Op_RegP]->or_with(aligned_stack_mask);
498 #else
499 idealreg2spillmask[Op_RegP]->or_with(C->FIRST_STACK_mask());
500 #endif
501 idealreg2spillmask[Op_RegI]->assignFrom(*idealreg2regmask[Op_RegI]);
502 idealreg2spillmask[Op_RegI]->or_with(C->FIRST_STACK_mask());
683 _register_save_policy[reg] == 'E' ||
684 _register_save_policy[reg] == 'A'; // Save-on-entry register?
685 }
686
687 //---------------------------Fixup_Save_On_Entry-------------------------------
688 void Matcher::Fixup_Save_On_Entry( ) {
689 init_first_stack_mask();
690
691 Node *root = C->root(); // Short name for root
692 // Count number of save-on-entry registers.
693 uint soe_cnt = number_of_saved_registers();
694 uint i;
695
696 // Find the procedure Start Node
697 StartNode *start = C->start();
698 assert( start, "Expect a start node" );
699
700 // Input RegMask array shared by all Returns.
701 // The type for doubles and longs has a count of 2, but
702 // there is only 1 returned value
703 uint ret_edge_cnt = TypeFunc::Parms + ((C->tf()->range()->cnt() == TypeFunc::Parms) ? 0 : 1);
704 RegMask *ret_rms = init_input_masks( ret_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
705 // Returns have 0 or 1 returned values depending on call signature.
706 // Return register is specified by return_value in the AD file.
707 if (ret_edge_cnt > TypeFunc::Parms) {
708 ret_rms[TypeFunc::Parms + 0].assignFrom(_return_value_mask);
709 }
710
711 // Input RegMask array shared by all ForwardExceptions
712 uint forw_exc_edge_cnt = TypeFunc::Parms;
713 RegMask* forw_exc_rms = init_input_masks( forw_exc_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
714
715 // Input RegMask array shared by all Rethrows.
716 uint reth_edge_cnt = TypeFunc::Parms+1;
717 RegMask *reth_rms = init_input_masks( reth_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
718 // Rethrow takes exception oop only, but in the argument 0 slot.
719 OptoReg::Name reg = find_receiver();
720 if (reg >= 0) {
721 reth_rms[TypeFunc::Parms].assignFrom(mreg2regmask[reg]);
722 #ifdef _LP64
723 // Need two slots for ptrs in 64-bit land
724 reth_rms[TypeFunc::Parms].insert(OptoReg::add(OptoReg::Name(reg), 1));
725 #endif
726 }
727
728 // Input RegMask array shared by all TailCalls
761
762 // Input RegMask array shared by all Halts
763 uint halt_edge_cnt = TypeFunc::Parms;
764 RegMask *halt_rms = init_input_masks( halt_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
765
766 // Capture the return input masks into each exit flavor
767 for( i=1; i < root->req(); i++ ) {
768 MachReturnNode *exit = root->in(i)->as_MachReturn();
769 switch( exit->ideal_Opcode() ) {
770 case Op_Return : exit->_in_rms = ret_rms; break;
771 case Op_Rethrow : exit->_in_rms = reth_rms; break;
772 case Op_TailCall : exit->_in_rms = tail_call_rms; break;
773 case Op_TailJump : exit->_in_rms = tail_jump_rms; break;
774 case Op_ForwardException: exit->_in_rms = forw_exc_rms; break;
775 case Op_Halt : exit->_in_rms = halt_rms; break;
776 default : ShouldNotReachHere();
777 }
778 }
779
780 // Next unused projection number from Start.
781 int proj_cnt = C->tf()->domain()->cnt();
782
783 // Do all the save-on-entry registers. Make projections from Start for
784 // them, and give them a use at the exit points. To the allocator, they
785 // look like incoming register arguments.
786 for( i = 0; i < _last_Mach_Reg; i++ ) {
787 if( is_save_on_entry(i) ) {
788
789 // Add the save-on-entry to the mask array
790 ret_rms [ ret_edge_cnt].assignFrom(mreg2regmask[i]);
791 reth_rms [ reth_edge_cnt].assignFrom(mreg2regmask[i]);
792 tail_call_rms[tail_call_edge_cnt].assignFrom(mreg2regmask[i]);
793 tail_jump_rms[tail_jump_edge_cnt].assignFrom(mreg2regmask[i]);
794 forw_exc_rms [ forw_exc_edge_cnt].assignFrom(mreg2regmask[i]);
795 // Halts need the SOE registers, but only in the stack as debug info.
796 // A just-prior uncommon-trap or deoptimization will use the SOE regs.
797 halt_rms [ halt_edge_cnt].assignFrom(*idealreg2spillmask[_register_save_type[i]]);
798
799 Node *mproj;
800
801 // Is this a RegF low half of a RegD? Double up 2 adjacent RegF's
1041 m = n->is_SafePoint() ? match_sfpt(n->as_SafePoint()):match_tree(n);
1042 if (C->failing()) return nullptr;
1043 if (m == nullptr) { Matcher::soft_match_failure(); return nullptr; }
1044 if (n->is_MemBar()) {
1045 m->as_MachMemBar()->set_adr_type(n->adr_type());
1046 }
1047 } else { // Nothing the matcher cares about
1048 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Multi()) { // Projections?
1049 if (n->in(0)->is_Initialize() && n->as_Proj()->_con == TypeFunc::Memory) {
1050 // Initialize may have multiple NarrowMem projections. They would all match to identical raw mem MachProjs.
1051 // We don't need multiple MachProjs. Create one if none already exist, otherwise use existing one.
1052 m = n->in(0)->as_Initialize()->mem_mach_proj();
1053 if (m == nullptr && has_new_node(n->in(0))) {
1054 InitializeNode* new_init = new_node(n->in(0))->as_Initialize();
1055 m = new_init->mem_mach_proj();
1056 }
1057 assert(m == nullptr || m->is_MachProj(), "no mem projection yet or a MachProj created during matching");
1058 }
1059 if (m == nullptr) {
1060 // Convert to machine-dependent projection
1061 m = n->in(0)->as_Multi()->match( n->as_Proj(), this );
1062 NOT_PRODUCT(record_new2old(m, n);)
1063 }
1064 if (m->in(0) != nullptr) // m might be top
1065 collect_null_checks(m, n);
1066 } else { // Else just a regular 'ol guy
1067 m = n->clone(); // So just clone into new-space
1068 NOT_PRODUCT(record_new2old(m, n);)
1069 // Def-Use edges will be added incrementally as Uses
1070 // of this node are matched.
1071 assert(m->outcnt() == 0, "no Uses of this clone yet");
1072 }
1073 }
1074
1075 set_new_node(n, m); // Map old to new
1076 if (_old_node_note_array != nullptr) {
1077 Node_Notes* nn = C->locate_node_notes(_old_node_note_array,
1078 n->_idx);
1079 C->set_node_notes_at(m->_idx, nn);
1080 }
1081 DEBUG_ONLY(match_alias_type(C, n, m));
1177 return warped;
1178 }
1179 return OptoReg::as_OptoReg(reg);
1180 }
1181
1182
1183 //------------------------------match_sfpt-------------------------------------
1184 // Helper function to match call instructions. Calls match special.
1185 // They match alone with no children. Their children, the incoming
1186 // arguments, match normally.
1187 MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) {
1188 MachSafePointNode *msfpt = nullptr;
1189 MachCallNode *mcall = nullptr;
1190 uint cnt;
1191 // Split out case for SafePoint vs Call
1192 CallNode *call;
1193 const TypeTuple *domain;
1194 ciMethod* method = nullptr;
1195 if( sfpt->is_Call() ) {
1196 call = sfpt->as_Call();
1197 domain = call->tf()->domain();
1198 cnt = domain->cnt();
1199
1200 // Match just the call, nothing else
1201 MachNode *m = match_tree(call);
1202 if (C->failing()) return nullptr;
1203 if( m == nullptr ) { Matcher::soft_match_failure(); return nullptr; }
1204
1205 // Copy data from the Ideal SafePoint to the machine version
1206 mcall = m->as_MachCall();
1207
1208 mcall->set_tf( call->tf());
1209 mcall->set_entry_point( call->entry_point());
1210 mcall->set_cnt( call->cnt());
1211 mcall->set_guaranteed_safepoint(call->guaranteed_safepoint());
1212
1213 if( mcall->is_MachCallJava() ) {
1214 MachCallJavaNode *mcall_java = mcall->as_MachCallJava();
1215 const CallJavaNode *call_java = call->as_CallJava();
1216 assert(call_java->validate_symbolic_info(), "inconsistent info");
1217 method = call_java->method();
1253 for (uint i = 0; i < cnt; i++) {
1254 ::new (msfpt->_in_rms + i) RegMask(C->comp_arena());
1255 }
1256
1257 // Do all the pre-defined non-Empty register masks
1258 msfpt->_in_rms[TypeFunc::ReturnAdr].assignFrom(_return_addr_mask);
1259 msfpt->_in_rms[TypeFunc::FramePtr ].assignFrom(c_frame_ptr_mask);
1260
1261 // Place first outgoing argument can possibly be put.
1262 OptoReg::Name begin_out_arg_area = OptoReg::add(_new_SP, C->out_preserve_stack_slots());
1263 assert( is_even(begin_out_arg_area), "" );
1264 // Compute max outgoing register number per call site.
1265 OptoReg::Name out_arg_limit_per_call = begin_out_arg_area;
1266 // Calls to C may hammer extra stack slots above and beyond any arguments.
1267 // These are usually backing store for register arguments for varargs.
1268 if( call != nullptr && call->is_CallRuntime() )
1269 out_arg_limit_per_call = OptoReg::add(out_arg_limit_per_call,C->varargs_C_out_slots_killed());
1270
1271
1272 // Do the normal argument list (parameters) register masks
1273 int argcnt = cnt - TypeFunc::Parms;
1274 if( argcnt > 0 ) { // Skip it all if we have no args
1275 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
1276 VMRegPair *parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
1277 int i;
1278 for( i = 0; i < argcnt; i++ ) {
1279 sig_bt[i] = domain->field_at(i+TypeFunc::Parms)->basic_type();
1280 }
1281 // V-call to pick proper calling convention
1282 call->calling_convention( sig_bt, parm_regs, argcnt );
1283
1284 #ifdef ASSERT
1285 // Sanity check users' calling convention. Really handy during
1286 // the initial porting effort. Fairly expensive otherwise.
1287 { for (int i = 0; i<argcnt; i++) {
1288 if( !parm_regs[i].first()->is_valid() &&
1289 !parm_regs[i].second()->is_valid() ) continue;
1290 VMReg reg1 = parm_regs[i].first();
1291 VMReg reg2 = parm_regs[i].second();
1292 for (int j = 0; j < i; j++) {
1293 if( !parm_regs[j].first()->is_valid() &&
1294 !parm_regs[j].second()->is_valid() ) continue;
1295 VMReg reg3 = parm_regs[j].first();
1296 VMReg reg4 = parm_regs[j].second();
1297 if( !reg1->is_valid() ) {
1298 assert( !reg2->is_valid(), "valid halvsies" );
1299 } else if( !reg3->is_valid() ) {
1300 assert( !reg4->is_valid(), "valid halvsies" );
1301 } else {
1302 assert( reg1 != reg2, "calling conv. must produce distinct regs");
1303 assert( reg1 != reg3, "calling conv. must produce distinct regs");
1304 assert( reg1 != reg4, "calling conv. must produce distinct regs");
1305 assert( reg2 != reg3, "calling conv. must produce distinct regs");
1306 assert( reg2 != reg4 || !reg2->is_valid(), "calling conv. must produce distinct regs");
1307 assert( reg3 != reg4, "calling conv. must produce distinct regs");
1308 }
1309 }
1310 }
1311 }
1312 #endif
1313
1314 // Visit each argument. Compute its outgoing register mask.
1315 // Return results now can have 2 bits returned.
1316 // Compute max over all outgoing arguments both per call-site
1317 // and over the entire method.
1318 for( i = 0; i < argcnt; i++ ) {
1319 // Address of incoming argument mask to fill in
1320 RegMask *rm = &mcall->_in_rms[i+TypeFunc::Parms];
1321 VMReg first = parm_regs[i].first();
1322 VMReg second = parm_regs[i].second();
1323 if(!first->is_valid() &&
1324 !second->is_valid()) {
1325 continue; // Avoid Halves
1326 }
1327 // Handle case where arguments are in vector registers.
1328 if(call->in(TypeFunc::Parms + i)->bottom_type()->isa_vect()) {
1329 OptoReg::Name reg_fst = OptoReg::as_OptoReg(first);
1330 OptoReg::Name reg_snd = OptoReg::as_OptoReg(second);
1331 assert (reg_fst <= reg_snd, "fst=%d snd=%d", reg_fst, reg_snd);
1332 for (OptoReg::Name r = reg_fst; r <= reg_snd; r++) {
1333 rm->insert(r);
1334 }
1335 }
1336 // Grab first register, adjust stack slots and insert in mask.
1337 OptoReg::Name reg1 = warp_outgoing_stk_arg(first, begin_out_arg_area, out_arg_limit_per_call );
1338 if (OptoReg::is_valid(reg1))
1339 rm->insert(reg1);
1340 // Grab second register (if any), adjust stack slots and insert in mask.
1341 OptoReg::Name reg2 = warp_outgoing_stk_arg(second, begin_out_arg_area, out_arg_limit_per_call );
1342 if (OptoReg::is_valid(reg2))
1343 rm->insert(reg2);
1344 } // End of for all arguments
1345 }
1346
1347 // Compute the max stack slot killed by any call. These will not be
1348 // available for debug info, and will be used to adjust FIRST_STACK_mask
1349 // after all call sites have been visited.
1350 if( _out_arg_limit < out_arg_limit_per_call)
1351 _out_arg_limit = out_arg_limit_per_call;
1352
1353 if (mcall) {
1354 // Kill the outgoing argument area, including any non-argument holes and
1355 // any legacy C-killed slots. Use Fat-Projections to do the killing.
1356 // Since the max-per-method covers the max-per-call-site and debug info
1357 // is excluded on the max-per-method basis, debug info cannot land in
1358 // this killed area.
1359 uint r_cnt = mcall->tf()->range()->cnt();
1360 MachProjNode* proj = new MachProjNode(mcall, r_cnt + 10000, RegMask::EMPTY, MachProjNode::fat_proj);
1361 for (int i = begin_out_arg_area; i < out_arg_limit_per_call; i++) {
1362 proj->_rout.insert(OptoReg::Name(i));
1363 }
1364 if (!proj->_rout.is_empty()) {
1365 push_projection(proj);
1366 }
1367 }
1368 // Transfer the safepoint information from the call to the mcall
1369 // Move the JVMState list
1370 msfpt->set_jvms(sfpt->jvms());
1371 for (JVMState* jvms = msfpt->jvms(); jvms; jvms = jvms->caller()) {
1372 jvms->set_map(sfpt);
1373 }
1374
1375 // Debug inputs begin just after the last incoming parameter
1376 assert((mcall == nullptr) || (mcall->jvms() == nullptr) ||
1377 (mcall->jvms()->debug_start() + mcall->_jvmadj == mcall->tf()->domain()->cnt()), "");
1378
1379 // Add additional edges.
1380 if (msfpt->mach_constant_base_node_input() != (uint)-1 && !msfpt->is_MachCallLeaf()) {
1381 // For these calls we can not add MachConstantBase in expand(), as the
1382 // ins are not complete then.
1383 msfpt->ins_req(msfpt->mach_constant_base_node_input(), C->mach_constant_base_node());
1384 if (msfpt->jvms() &&
1385 msfpt->mach_constant_base_node_input() <= msfpt->jvms()->debug_start() + msfpt->_jvmadj) {
1386 // We added an edge before jvms, so we must adapt the position of the ins.
1387 msfpt->jvms()->adapt_position(+1);
1388 }
1389 }
1390
1391 // Registers killed by the call are set in the local scheduling pass
1392 // of Global Code Motion.
1393 return msfpt;
1394 }
1395
1396 //---------------------------match_tree----------------------------------------
1397 // Match a Ideal Node DAG - turn it into a tree; Label & Reduce. Used as part
2046 set_shared(n); // Flag as shared and
2047 if (n->is_DecodeNarrowPtr()) {
2048 // Oop field/array element loads must be shared but since
2049 // they are shared through a DecodeN they may appear to have
2050 // a single use so force sharing here.
2051 set_shared(n->in(1));
2052 }
2053 mstack.pop(); // remove node from stack
2054 continue;
2055 }
2056 nstate = Visit; // Not already visited; so visit now
2057 }
2058 if (nstate == Visit) {
2059 mstack.set_state(Post_Visit);
2060 set_visited(n); // Flag as visited now
2061 bool mem_op = false;
2062 int mem_addr_idx = MemNode::Address;
2063 if (find_shared_visit(mstack, n, nop, mem_op, mem_addr_idx)) {
2064 continue;
2065 }
2066 for (int i = n->req() - 1; i >= 0; --i) { // For my children
2067 Node* m = n->in(i); // Get ith input
2068 if (m == nullptr) {
2069 continue; // Ignore nulls
2070 }
2071 if (clone_node(n, m, mstack)) {
2072 continue;
2073 }
2074
2075 // Clone addressing expressions as they are "free" in memory access instructions
2076 if (mem_op && i == mem_addr_idx && m->is_AddP() &&
2077 // When there are other uses besides address expressions
2078 // put it on stack and mark as shared.
2079 !is_visited(m)) {
2080 // Some inputs for address expression are not put on stack
2081 // to avoid marking them as shared and forcing them into register
2082 // if they are used only in address expressions.
2083 // But they should be marked as shared if there are other uses
2084 // besides address expressions.
2085
2086 if (pd_clone_address_expressions(m->as_AddP(), mstack, address_visited)) {
2357 case Op_FmaHF:
2358 case Op_FmaVD:
2359 case Op_FmaVF:
2360 case Op_FmaVHF: {
2361 // Restructure into a binary tree for Matching.
2362 Node* pair = new BinaryNode(n->in(1), n->in(2));
2363 n->set_req(2, pair);
2364 n->set_req(1, n->in(3));
2365 n->del_req(3);
2366 break;
2367 }
2368 case Op_MulAddS2I: {
2369 Node* pair1 = new BinaryNode(n->in(1), n->in(2));
2370 Node* pair2 = new BinaryNode(n->in(3), n->in(4));
2371 n->set_req(1, pair1);
2372 n->set_req(2, pair2);
2373 n->del_req(4);
2374 n->del_req(3);
2375 break;
2376 }
2377 case Op_VectorCmpMasked:
2378 case Op_CopySignD:
2379 case Op_SignumVF:
2380 case Op_SignumVD:
2381 case Op_SignumF:
2382 case Op_SignumD: {
2383 Node* pair = new BinaryNode(n->in(2), n->in(3));
2384 n->set_req(2, pair);
2385 n->del_req(3);
2386 break;
2387 }
2388 case Op_VectorBlend:
2389 case Op_VectorInsert: {
2390 Node* pair = new BinaryNode(n->in(1), n->in(2));
2391 n->set_req(1, pair);
2392 n->set_req(2, n->in(3));
2393 n->del_req(3);
2394 break;
2395 }
2396 case Op_LoadVectorGatherMasked: // fall-through
2406 n->del_req(MemNode::ValueIn+2);
2407 pair = new BinaryNode(n->in(MemNode::ValueIn), n->in(MemNode::ValueIn+1));
2408 n->set_req(MemNode::ValueIn, pair);
2409 n->del_req(MemNode::ValueIn+1);
2410 break;
2411 }
2412 case Op_VectorMaskCmp: {
2413 n->set_req(1, new BinaryNode(n->in(1), n->in(2)));
2414 n->set_req(2, n->in(3));
2415 n->del_req(3);
2416 break;
2417 }
2418 case Op_PartialSubtypeCheck: {
2419 if (UseSecondarySupersTable && n->in(2)->is_Con()) {
2420 // PartialSubtypeCheck uses both constant and register operands for superclass input.
2421 n->set_req(2, new BinaryNode(n->in(2), n->in(2)));
2422 break;
2423 }
2424 break;
2425 }
2426 default:
2427 break;
2428 }
2429 }
2430
2431 #ifndef PRODUCT
2432 void Matcher::record_new2old(Node* newn, Node* old) {
2433 _new2old_map.map(newn->_idx, old);
2434 if (!_reused.test_set(old->_igv_idx)) {
2435 // Reuse the Ideal-level IGV identifier so that the node can be tracked
2436 // across matching. If there are multiple machine nodes expanded from the
2437 // same Ideal node, only one will reuse its IGV identifier.
2438 newn->_igv_idx = old->_igv_idx;
2439 }
2440 }
2441
2442 // machine-independent root to machine-dependent root
2443 void Matcher::dump_old2new_map() {
2444 _old2new_map.dump();
2445 }
|
149 Node* n = worklist.pop();
150 if (visited.test_set(n->_idx)) {
151 continue;
152 }
153 assert(C->node_arena()->contains(n), "dead node");
154 assert(!n->is_Initialize() || n->as_Initialize()->number_of_projs(TypeFunc::Memory) == 1,
155 "after matching, Initialize should have a single memory projection");
156 for (uint j = 0; j < n->req(); j++) {
157 Node* in = n->in(j);
158 if (in != nullptr) {
159 worklist.push(in);
160 }
161 }
162 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
163 worklist.push(n->fast_out(j));
164 }
165 }
166 }
167 #endif
168
169 // Array of RegMask, one per returned values (inline type instances can
170 // be returned as multiple return values, one per field)
171 RegMask* Matcher::return_values_mask(const TypeFunc* tf) {
172 const TypeTuple* range = tf->range_cc();
173 uint cnt = range->cnt() - TypeFunc::Parms;
174 if (cnt == 0) {
175 return nullptr;
176 }
177 RegMask* mask = NEW_RESOURCE_ARRAY(RegMask, cnt);
178 BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, cnt);
179 VMRegPair* vm_parm_regs = NEW_RESOURCE_ARRAY(VMRegPair, cnt);
180 for (uint i = 0; i < cnt; i++) {
181 sig_bt[i] = range->field_at(i+TypeFunc::Parms)->basic_type();
182 new (mask + i) RegMask();
183 }
184
185 int regs = SharedRuntime::java_return_convention(sig_bt, vm_parm_regs, cnt);
186 if (regs <= 0) {
187 // We ran out of registers to store the null marker for a nullable inline type return.
188 // Since it is only set in the 'call_epilog', we can simply put it on the stack.
189 assert(tf->returns_inline_type_as_fields(), "should have been tested during graph construction");
190 // TODO 8284443 Can we teach the register allocator to reserve a stack slot instead?
191 // mask[--cnt] = STACK_ONLY_mask does not work (test with -XX:+StressGCM)
192 int slot = C->fixed_slots() - 2;
193 if (C->needs_stack_repair()) {
194 slot -= 2; // Account for stack increment value
195 }
196 mask[--cnt].clear();
197 mask[cnt].insert(OptoReg::stack2reg(slot));
198 }
199 for (uint i = 0; i < cnt; i++) {
200 mask[i].clear();
201
202 OptoReg::Name reg1 = OptoReg::as_OptoReg(vm_parm_regs[i].first());
203 if (OptoReg::is_valid(reg1)) {
204 mask[i].insert(reg1);
205 }
206 OptoReg::Name reg2 = OptoReg::as_OptoReg(vm_parm_regs[i].second());
207 if (OptoReg::is_valid(reg2)) {
208 mask[i].insert(reg2);
209 }
210 }
211
212 return mask;
213 }
214
215 //---------------------------match---------------------------------------------
216 void Matcher::match( ) {
217 if( MaxLabelRootDepth < 100 ) { // Too small?
218 assert(false, "invalid MaxLabelRootDepth, increase it to 100 minimum");
219 MaxLabelRootDepth = 100;
220 }
221 // One-time initialization of some register masks.
222 init_spill_mask( C->root()->in(1) );
223 if (C->failing()) {
224 return;
225 }
226 assert(_return_addr_mask.is_empty(),
227 "return address mask must be empty initially");
228 _return_addr_mask.insert(return_addr());
229 #ifdef _LP64
230 // Pointers take 2 slots in 64-bit land
231 _return_addr_mask.insert(OptoReg::add(return_addr(), 1));
232 #endif
233
234 // Map Java-signature return types into return register-value
235 // machine registers.
236 _return_values_mask = return_values_mask(C->tf());
237
238 // ---------------
239 // Frame Layout
240
241 // Need the method signature to determine the incoming argument types,
242 // because the types determine which registers the incoming arguments are
243 // in, and this affects the matched code.
244 const TypeTuple *domain = C->tf()->domain_cc();
245 uint argcnt = domain->cnt() - TypeFunc::Parms;
246 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
247 VMRegPair *vm_parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
248 _parm_regs = NEW_RESOURCE_ARRAY( OptoRegPair, argcnt );
249 _calling_convention_mask = NEW_RESOURCE_ARRAY( RegMask, argcnt );
250 uint i;
251 for( i = 0; i<argcnt; i++ ) {
252 sig_bt[i] = domain->field_at(i+TypeFunc::Parms)->basic_type();
253 new (_calling_convention_mask + i) RegMask(C->comp_arena());
254 }
255
256 // Pass array of ideal registers and length to USER code (from the AD file)
257 // that will convert this to an array of register numbers.
258 const StartNode *start = C->start();
259 start->calling_convention( sig_bt, vm_parm_regs, argcnt );
260 #ifdef ASSERT
261 // Sanity check users' calling convention. Real handy while trying to
262 // get the initial port correct.
263 { for (uint i = 0; i<argcnt; i++) {
264 if( !vm_parm_regs[i].first()->is_valid() && !vm_parm_regs[i].second()->is_valid() ) {
496 // Initialize empty placeholder masks into the newly allocated arena
497 for (int i = 0; i < NOF_STACK_MASKS; i++) {
498 new (rms + i) RegMask(C->comp_arena());
499 }
500
501 int index = 0;
502 for (int i = Op_RegN; i <= Op_RegVectMask; ++i) {
503 idealreg2spillmask[i] = &rms[index++];
504 idealreg2debugmask[i] = &rms[index++];
505 }
506 assert(index == NOF_STACK_MASKS, "wrong size");
507
508 // At first, start with the empty mask
509 C->FIRST_STACK_mask().clear();
510
511 // Add in the incoming argument area
512 OptoReg::Name init_in = OptoReg::add(_old_SP, C->out_preserve_stack_slots());
513 for (OptoReg::Name i = init_in; i < _in_arg_limit; i = OptoReg::add(i, 1)) {
514 C->FIRST_STACK_mask().insert(i);
515 }
516
517 // Add in all bits past the outgoing argument area
518 C->FIRST_STACK_mask().set_all_from(_out_arg_limit);
519
520 // Make spill masks. Registers for their class, plus FIRST_STACK_mask.
521 RegMask aligned_stack_mask(C->FIRST_STACK_mask(), C->comp_arena());
522 // Keep spill masks aligned.
523 aligned_stack_mask.clear_to_pairs();
524 assert(aligned_stack_mask.is_infinite_stack(), "should be infinite stack");
525 RegMask scalable_stack_mask(aligned_stack_mask, C->comp_arena());
526
527 idealreg2spillmask[Op_RegP]->assignFrom(*idealreg2regmask[Op_RegP]);
528 #ifdef _LP64
529 idealreg2spillmask[Op_RegN]->assignFrom(*idealreg2regmask[Op_RegN]);
530 idealreg2spillmask[Op_RegN]->or_with(C->FIRST_STACK_mask());
531 idealreg2spillmask[Op_RegP]->or_with(aligned_stack_mask);
532 #else
533 idealreg2spillmask[Op_RegP]->or_with(C->FIRST_STACK_mask());
534 #endif
535 idealreg2spillmask[Op_RegI]->assignFrom(*idealreg2regmask[Op_RegI]);
536 idealreg2spillmask[Op_RegI]->or_with(C->FIRST_STACK_mask());
717 _register_save_policy[reg] == 'E' ||
718 _register_save_policy[reg] == 'A'; // Save-on-entry register?
719 }
720
721 //---------------------------Fixup_Save_On_Entry-------------------------------
722 void Matcher::Fixup_Save_On_Entry( ) {
723 init_first_stack_mask();
724
725 Node *root = C->root(); // Short name for root
726 // Count number of save-on-entry registers.
727 uint soe_cnt = number_of_saved_registers();
728 uint i;
729
730 // Find the procedure Start Node
731 StartNode *start = C->start();
732 assert( start, "Expect a start node" );
733
734 // Input RegMask array shared by all Returns.
735 // The type for doubles and longs has a count of 2, but
736 // there is only 1 returned value
737 uint ret_edge_cnt = C->tf()->range_cc()->cnt();
738 RegMask *ret_rms = init_input_masks( ret_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
739 for (i = TypeFunc::Parms; i < ret_edge_cnt; i++) {
740 ret_rms[i].assignFrom(_return_values_mask[i-TypeFunc::Parms]);
741 }
742
743 // Input RegMask array shared by all ForwardExceptions
744 uint forw_exc_edge_cnt = TypeFunc::Parms;
745 RegMask* forw_exc_rms = init_input_masks( forw_exc_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
746
747 // Input RegMask array shared by all Rethrows.
748 uint reth_edge_cnt = TypeFunc::Parms+1;
749 RegMask *reth_rms = init_input_masks( reth_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
750 // Rethrow takes exception oop only, but in the argument 0 slot.
751 OptoReg::Name reg = find_receiver();
752 if (reg >= 0) {
753 reth_rms[TypeFunc::Parms].assignFrom(mreg2regmask[reg]);
754 #ifdef _LP64
755 // Need two slots for ptrs in 64-bit land
756 reth_rms[TypeFunc::Parms].insert(OptoReg::add(OptoReg::Name(reg), 1));
757 #endif
758 }
759
760 // Input RegMask array shared by all TailCalls
793
794 // Input RegMask array shared by all Halts
795 uint halt_edge_cnt = TypeFunc::Parms;
796 RegMask *halt_rms = init_input_masks( halt_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
797
798 // Capture the return input masks into each exit flavor
799 for( i=1; i < root->req(); i++ ) {
800 MachReturnNode *exit = root->in(i)->as_MachReturn();
801 switch( exit->ideal_Opcode() ) {
802 case Op_Return : exit->_in_rms = ret_rms; break;
803 case Op_Rethrow : exit->_in_rms = reth_rms; break;
804 case Op_TailCall : exit->_in_rms = tail_call_rms; break;
805 case Op_TailJump : exit->_in_rms = tail_jump_rms; break;
806 case Op_ForwardException: exit->_in_rms = forw_exc_rms; break;
807 case Op_Halt : exit->_in_rms = halt_rms; break;
808 default : ShouldNotReachHere();
809 }
810 }
811
812 // Next unused projection number from Start.
813 int proj_cnt = C->tf()->domain_cc()->cnt();
814
815 // Do all the save-on-entry registers. Make projections from Start for
816 // them, and give them a use at the exit points. To the allocator, they
817 // look like incoming register arguments.
818 for( i = 0; i < _last_Mach_Reg; i++ ) {
819 if( is_save_on_entry(i) ) {
820
821 // Add the save-on-entry to the mask array
822 ret_rms [ ret_edge_cnt].assignFrom(mreg2regmask[i]);
823 reth_rms [ reth_edge_cnt].assignFrom(mreg2regmask[i]);
824 tail_call_rms[tail_call_edge_cnt].assignFrom(mreg2regmask[i]);
825 tail_jump_rms[tail_jump_edge_cnt].assignFrom(mreg2regmask[i]);
826 forw_exc_rms [ forw_exc_edge_cnt].assignFrom(mreg2regmask[i]);
827 // Halts need the SOE registers, but only in the stack as debug info.
828 // A just-prior uncommon-trap or deoptimization will use the SOE regs.
829 halt_rms [ halt_edge_cnt].assignFrom(*idealreg2spillmask[_register_save_type[i]]);
830
831 Node *mproj;
832
833 // Is this a RegF low half of a RegD? Double up 2 adjacent RegF's
1073 m = n->is_SafePoint() ? match_sfpt(n->as_SafePoint()):match_tree(n);
1074 if (C->failing()) return nullptr;
1075 if (m == nullptr) { Matcher::soft_match_failure(); return nullptr; }
1076 if (n->is_MemBar()) {
1077 m->as_MachMemBar()->set_adr_type(n->adr_type());
1078 }
1079 } else { // Nothing the matcher cares about
1080 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Multi()) { // Projections?
1081 if (n->in(0)->is_Initialize() && n->as_Proj()->_con == TypeFunc::Memory) {
1082 // Initialize may have multiple NarrowMem projections. They would all match to identical raw mem MachProjs.
1083 // We don't need multiple MachProjs. Create one if none already exist, otherwise use existing one.
1084 m = n->in(0)->as_Initialize()->mem_mach_proj();
1085 if (m == nullptr && has_new_node(n->in(0))) {
1086 InitializeNode* new_init = new_node(n->in(0))->as_Initialize();
1087 m = new_init->mem_mach_proj();
1088 }
1089 assert(m == nullptr || m->is_MachProj(), "no mem projection yet or a MachProj created during matching");
1090 }
1091 if (m == nullptr) {
1092 // Convert to machine-dependent projection
1093 RegMask* mask = nullptr;
1094 if (n->in(0)->is_Call() && n->in(0)->as_Call()->tf()->returns_inline_type_as_fields()) {
1095 mask = return_values_mask(n->in(0)->as_Call()->tf());
1096 }
1097 m = n->in(0)->as_Multi()->match(n->as_Proj(), this, mask);
1098 NOT_PRODUCT(record_new2old(m, n);)
1099 }
1100 if (m->in(0) != nullptr) // m might be top
1101 collect_null_checks(m, n);
1102 } else { // Else just a regular 'ol guy
1103 m = n->clone(); // So just clone into new-space
1104 NOT_PRODUCT(record_new2old(m, n);)
1105 // Def-Use edges will be added incrementally as Uses
1106 // of this node are matched.
1107 assert(m->outcnt() == 0, "no Uses of this clone yet");
1108 }
1109 }
1110
1111 set_new_node(n, m); // Map old to new
1112 if (_old_node_note_array != nullptr) {
1113 Node_Notes* nn = C->locate_node_notes(_old_node_note_array,
1114 n->_idx);
1115 C->set_node_notes_at(m->_idx, nn);
1116 }
1117 DEBUG_ONLY(match_alias_type(C, n, m));
1213 return warped;
1214 }
1215 return OptoReg::as_OptoReg(reg);
1216 }
1217
1218
1219 //------------------------------match_sfpt-------------------------------------
1220 // Helper function to match call instructions. Calls match special.
1221 // They match alone with no children. Their children, the incoming
1222 // arguments, match normally.
1223 MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) {
1224 MachSafePointNode *msfpt = nullptr;
1225 MachCallNode *mcall = nullptr;
1226 uint cnt;
1227 // Split out case for SafePoint vs Call
1228 CallNode *call;
1229 const TypeTuple *domain;
1230 ciMethod* method = nullptr;
1231 if( sfpt->is_Call() ) {
1232 call = sfpt->as_Call();
1233 domain = call->tf()->domain_cc();
1234 cnt = domain->cnt();
1235
1236 // Match just the call, nothing else
1237 MachNode *m = match_tree(call);
1238 if (C->failing()) return nullptr;
1239 if( m == nullptr ) { Matcher::soft_match_failure(); return nullptr; }
1240
1241 // Copy data from the Ideal SafePoint to the machine version
1242 mcall = m->as_MachCall();
1243
1244 mcall->set_tf( call->tf());
1245 mcall->set_entry_point( call->entry_point());
1246 mcall->set_cnt( call->cnt());
1247 mcall->set_guaranteed_safepoint(call->guaranteed_safepoint());
1248
1249 if( mcall->is_MachCallJava() ) {
1250 MachCallJavaNode *mcall_java = mcall->as_MachCallJava();
1251 const CallJavaNode *call_java = call->as_CallJava();
1252 assert(call_java->validate_symbolic_info(), "inconsistent info");
1253 method = call_java->method();
1289 for (uint i = 0; i < cnt; i++) {
1290 ::new (msfpt->_in_rms + i) RegMask(C->comp_arena());
1291 }
1292
1293 // Do all the pre-defined non-Empty register masks
1294 msfpt->_in_rms[TypeFunc::ReturnAdr].assignFrom(_return_addr_mask);
1295 msfpt->_in_rms[TypeFunc::FramePtr ].assignFrom(c_frame_ptr_mask);
1296
1297 // Place first outgoing argument can possibly be put.
1298 OptoReg::Name begin_out_arg_area = OptoReg::add(_new_SP, C->out_preserve_stack_slots());
1299 assert( is_even(begin_out_arg_area), "" );
1300 // Compute max outgoing register number per call site.
1301 OptoReg::Name out_arg_limit_per_call = begin_out_arg_area;
1302 // Calls to C may hammer extra stack slots above and beyond any arguments.
1303 // These are usually backing store for register arguments for varargs.
1304 if( call != nullptr && call->is_CallRuntime() )
1305 out_arg_limit_per_call = OptoReg::add(out_arg_limit_per_call,C->varargs_C_out_slots_killed());
1306
1307
1308 // Do the normal argument list (parameters) register masks
1309 // Null entry point is a special cast where the target of the call
1310 // is in a register.
1311 int adj = (call != nullptr && call->entry_point() == nullptr) ? 1 : 0;
1312 int argcnt = cnt - TypeFunc::Parms - adj;
1313 if( argcnt > 0 ) { // Skip it all if we have no args
1314 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
1315 VMRegPair *parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
1316 int i;
1317 for( i = 0; i < argcnt; i++ ) {
1318 sig_bt[i] = domain->field_at(i+TypeFunc::Parms+adj)->basic_type();
1319 }
1320 // V-call to pick proper calling convention
1321 call->calling_convention( sig_bt, parm_regs, argcnt );
1322
1323 #ifdef ASSERT
1324 // Sanity check users' calling convention. Really handy during
1325 // the initial porting effort. Fairly expensive otherwise.
1326 { for (int i = 0; i<argcnt; i++) {
1327 if( !parm_regs[i].first()->is_valid() &&
1328 !parm_regs[i].second()->is_valid() ) continue;
1329 VMReg reg1 = parm_regs[i].first();
1330 VMReg reg2 = parm_regs[i].second();
1331 for (int j = 0; j < i; j++) {
1332 if( !parm_regs[j].first()->is_valid() &&
1333 !parm_regs[j].second()->is_valid() ) continue;
1334 VMReg reg3 = parm_regs[j].first();
1335 VMReg reg4 = parm_regs[j].second();
1336 if( !reg1->is_valid() ) {
1337 assert( !reg2->is_valid(), "valid halvsies" );
1338 } else if( !reg3->is_valid() ) {
1339 assert( !reg4->is_valid(), "valid halvsies" );
1340 } else {
1341 assert( reg1 != reg2, "calling conv. must produce distinct regs");
1342 assert( reg1 != reg3, "calling conv. must produce distinct regs");
1343 assert( reg1 != reg4, "calling conv. must produce distinct regs");
1344 assert( reg2 != reg3, "calling conv. must produce distinct regs");
1345 assert( reg2 != reg4 || !reg2->is_valid(), "calling conv. must produce distinct regs");
1346 assert( reg3 != reg4, "calling conv. must produce distinct regs");
1347 }
1348 }
1349 }
1350 }
1351 #endif
1352
1353 // Visit each argument. Compute its outgoing register mask.
1354 // Return results now can have 2 bits returned.
1355 // Compute max over all outgoing arguments both per call-site
1356 // and over the entire method.
1357 for( i = 0; i < argcnt; i++ ) {
1358 // Address of incoming argument mask to fill in
1359 RegMask *rm = &mcall->_in_rms[i+TypeFunc::Parms+adj];
1360 VMReg first = parm_regs[i].first();
1361 VMReg second = parm_regs[i].second();
1362 if(!first->is_valid() &&
1363 !second->is_valid()) {
1364 continue; // Avoid Halves
1365 }
1366 // Handle case where arguments are in vector registers.
1367 if(call->in(TypeFunc::Parms + i)->bottom_type()->isa_vect()) {
1368 OptoReg::Name reg_fst = OptoReg::as_OptoReg(first);
1369 OptoReg::Name reg_snd = OptoReg::as_OptoReg(second);
1370 assert (reg_fst <= reg_snd, "fst=%d snd=%d", reg_fst, reg_snd);
1371 for (OptoReg::Name r = reg_fst; r <= reg_snd; r++) {
1372 rm->insert(r);
1373 }
1374 }
1375 // Grab first register, adjust stack slots and insert in mask.
1376 OptoReg::Name reg1 = warp_outgoing_stk_arg(first, begin_out_arg_area, out_arg_limit_per_call );
1377 if (OptoReg::is_valid(reg1)) {
1378 rm->insert( reg1 );
1379 }
1380 // Grab second register (if any), adjust stack slots and insert in mask.
1381 OptoReg::Name reg2 = warp_outgoing_stk_arg(second, begin_out_arg_area, out_arg_limit_per_call );
1382 if (OptoReg::is_valid(reg2)) {
1383 rm->insert( reg2 );
1384 }
1385 } // End of for all arguments
1386 }
1387
1388 // Compute the max stack slot killed by any call. These will not be
1389 // available for debug info, and will be used to adjust FIRST_STACK_mask
1390 // after all call sites have been visited.
1391 if( _out_arg_limit < out_arg_limit_per_call)
1392 _out_arg_limit = out_arg_limit_per_call;
1393
1394 if (mcall) {
1395 // Kill the outgoing argument area, including any non-argument holes and
1396 // any legacy C-killed slots. Use Fat-Projections to do the killing.
1397 // Since the max-per-method covers the max-per-call-site and debug info
1398 // is excluded on the max-per-method basis, debug info cannot land in
1399 // this killed area.
1400 uint r_cnt = mcall->tf()->range_sig()->cnt();
1401 MachProjNode *proj = new MachProjNode( mcall, r_cnt+10000, RegMask::EMPTY, MachProjNode::fat_proj );
1402 for (int i = begin_out_arg_area; i < out_arg_limit_per_call; i++) {
1403 proj->_rout.insert(OptoReg::Name(i));
1404 }
1405 if (!proj->_rout.is_empty()) {
1406 push_projection(proj);
1407 }
1408 }
1409 // Transfer the safepoint information from the call to the mcall
1410 // Move the JVMState list
1411 msfpt->set_jvms(sfpt->jvms());
1412 for (JVMState* jvms = msfpt->jvms(); jvms; jvms = jvms->caller()) {
1413 jvms->set_map(sfpt);
1414 }
1415
1416 // Debug inputs begin just after the last incoming parameter
1417 assert((mcall == nullptr) || (mcall->jvms() == nullptr) ||
1418 (mcall->jvms()->debug_start() + mcall->_jvmadj == mcall->tf()->domain_cc()->cnt()), "");
1419
1420 // Add additional edges.
1421 if (msfpt->mach_constant_base_node_input() != (uint)-1 && !msfpt->is_MachCallLeaf()) {
1422 // For these calls we can not add MachConstantBase in expand(), as the
1423 // ins are not complete then.
1424 msfpt->ins_req(msfpt->mach_constant_base_node_input(), C->mach_constant_base_node());
1425 if (msfpt->jvms() &&
1426 msfpt->mach_constant_base_node_input() <= msfpt->jvms()->debug_start() + msfpt->_jvmadj) {
1427 // We added an edge before jvms, so we must adapt the position of the ins.
1428 msfpt->jvms()->adapt_position(+1);
1429 }
1430 }
1431
1432 // Registers killed by the call are set in the local scheduling pass
1433 // of Global Code Motion.
1434 return msfpt;
1435 }
1436
1437 //---------------------------match_tree----------------------------------------
1438 // Match a Ideal Node DAG - turn it into a tree; Label & Reduce. Used as part
2087 set_shared(n); // Flag as shared and
2088 if (n->is_DecodeNarrowPtr()) {
2089 // Oop field/array element loads must be shared but since
2090 // they are shared through a DecodeN they may appear to have
2091 // a single use so force sharing here.
2092 set_shared(n->in(1));
2093 }
2094 mstack.pop(); // remove node from stack
2095 continue;
2096 }
2097 nstate = Visit; // Not already visited; so visit now
2098 }
2099 if (nstate == Visit) {
2100 mstack.set_state(Post_Visit);
2101 set_visited(n); // Flag as visited now
2102 bool mem_op = false;
2103 int mem_addr_idx = MemNode::Address;
2104 if (find_shared_visit(mstack, n, nop, mem_op, mem_addr_idx)) {
2105 continue;
2106 }
2107 for (int i = n->len() - 1; i >= 0; --i) { // For my children
2108 Node* m = n->in(i); // Get ith input
2109 if (m == nullptr) {
2110 continue; // Ignore nulls
2111 }
2112 if (clone_node(n, m, mstack)) {
2113 continue;
2114 }
2115
2116 // Clone addressing expressions as they are "free" in memory access instructions
2117 if (mem_op && i == mem_addr_idx && m->is_AddP() &&
2118 // When there are other uses besides address expressions
2119 // put it on stack and mark as shared.
2120 !is_visited(m)) {
2121 // Some inputs for address expression are not put on stack
2122 // to avoid marking them as shared and forcing them into register
2123 // if they are used only in address expressions.
2124 // But they should be marked as shared if there are other uses
2125 // besides address expressions.
2126
2127 if (pd_clone_address_expressions(m->as_AddP(), mstack, address_visited)) {
2398 case Op_FmaHF:
2399 case Op_FmaVD:
2400 case Op_FmaVF:
2401 case Op_FmaVHF: {
2402 // Restructure into a binary tree for Matching.
2403 Node* pair = new BinaryNode(n->in(1), n->in(2));
2404 n->set_req(2, pair);
2405 n->set_req(1, n->in(3));
2406 n->del_req(3);
2407 break;
2408 }
2409 case Op_MulAddS2I: {
2410 Node* pair1 = new BinaryNode(n->in(1), n->in(2));
2411 Node* pair2 = new BinaryNode(n->in(3), n->in(4));
2412 n->set_req(1, pair1);
2413 n->set_req(2, pair2);
2414 n->del_req(4);
2415 n->del_req(3);
2416 break;
2417 }
2418 case Op_ClearArray: {
2419 Node* pair = new BinaryNode(n->in(2), n->in(3));
2420 n->set_req(2, pair);
2421 n->set_req(3, n->in(4));
2422 n->del_req(4);
2423 break;
2424 }
2425 case Op_VectorCmpMasked:
2426 case Op_CopySignD:
2427 case Op_SignumVF:
2428 case Op_SignumVD:
2429 case Op_SignumF:
2430 case Op_SignumD: {
2431 Node* pair = new BinaryNode(n->in(2), n->in(3));
2432 n->set_req(2, pair);
2433 n->del_req(3);
2434 break;
2435 }
2436 case Op_VectorBlend:
2437 case Op_VectorInsert: {
2438 Node* pair = new BinaryNode(n->in(1), n->in(2));
2439 n->set_req(1, pair);
2440 n->set_req(2, n->in(3));
2441 n->del_req(3);
2442 break;
2443 }
2444 case Op_LoadVectorGatherMasked: // fall-through
2454 n->del_req(MemNode::ValueIn+2);
2455 pair = new BinaryNode(n->in(MemNode::ValueIn), n->in(MemNode::ValueIn+1));
2456 n->set_req(MemNode::ValueIn, pair);
2457 n->del_req(MemNode::ValueIn+1);
2458 break;
2459 }
2460 case Op_VectorMaskCmp: {
2461 n->set_req(1, new BinaryNode(n->in(1), n->in(2)));
2462 n->set_req(2, n->in(3));
2463 n->del_req(3);
2464 break;
2465 }
2466 case Op_PartialSubtypeCheck: {
2467 if (UseSecondarySupersTable && n->in(2)->is_Con()) {
2468 // PartialSubtypeCheck uses both constant and register operands for superclass input.
2469 n->set_req(2, new BinaryNode(n->in(2), n->in(2)));
2470 break;
2471 }
2472 break;
2473 }
2474 case Op_StoreLSpecial: {
2475 if (n->req() > (MemNode::ValueIn + 1) && n->in(MemNode::ValueIn + 1) != nullptr) {
2476 Node* pair = new BinaryNode(n->in(MemNode::ValueIn), n->in(MemNode::ValueIn + 1));
2477 n->set_req(MemNode::ValueIn, pair);
2478 n->del_req(MemNode::ValueIn + 1);
2479 }
2480 break;
2481 }
2482 default:
2483 break;
2484 }
2485 }
2486
2487 #ifndef PRODUCT
2488 void Matcher::record_new2old(Node* newn, Node* old) {
2489 _new2old_map.map(newn->_idx, old);
2490 if (!_reused.test_set(old->_igv_idx)) {
2491 // Reuse the Ideal-level IGV identifier so that the node can be tracked
2492 // across matching. If there are multiple machine nodes expanded from the
2493 // same Ideal node, only one will reuse its IGV identifier.
2494 newn->_igv_idx = old->_igv_idx;
2495 }
2496 }
2497
2498 // machine-independent root to machine-dependent root
2499 void Matcher::dump_old2new_map() {
2500 _old2new_map.dump();
2501 }
|