149 Node* n = worklist.pop();
150 if (visited.test_set(n->_idx)) {
151 continue;
152 }
153 assert(C->node_arena()->contains(n), "dead node");
154 assert(!n->is_Initialize() || n->as_Initialize()->number_of_projs(TypeFunc::Memory) == 1,
155 "after matching, Initialize should have a single memory projection");
156 for (uint j = 0; j < n->req(); j++) {
157 Node* in = n->in(j);
158 if (in != nullptr) {
159 worklist.push(in);
160 }
161 }
162 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
163 worklist.push(n->fast_out(j));
164 }
165 }
166 }
167 #endif
168
169
170 //---------------------------match---------------------------------------------
171 void Matcher::match( ) {
172 if( MaxLabelRootDepth < 100 ) { // Too small?
173 assert(false, "invalid MaxLabelRootDepth, increase it to 100 minimum");
174 MaxLabelRootDepth = 100;
175 }
176 // One-time initialization of some register masks.
177 init_spill_mask( C->root()->in(1) );
178 if (C->failing()) {
179 return;
180 }
181 assert(_return_addr_mask.is_empty(),
182 "return address mask must be empty initially");
183 _return_addr_mask.insert(return_addr());
184 #ifdef _LP64
185 // Pointers take 2 slots in 64-bit land
186 _return_addr_mask.insert(OptoReg::add(return_addr(), 1));
187 #endif
188
189 // Map a Java-signature return type into return register-value
190 // machine registers for 0, 1 and 2 returned values.
191 const TypeTuple *range = C->tf()->range();
192 if( range->cnt() > TypeFunc::Parms ) { // If not a void function
193 // Get ideal-register return type
194 uint ireg = range->field_at(TypeFunc::Parms)->ideal_reg();
195 // Get machine return register
196 uint sop = C->start()->Opcode();
197 OptoRegPair regs = return_value(ireg);
198
199 // And mask for same
200 _return_value_mask.assignFrom(RegMask(regs.first()));
201 if( OptoReg::is_valid(regs.second()) )
202 _return_value_mask.insert(regs.second());
203 }
204
205 // ---------------
206 // Frame Layout
207
208 // Need the method signature to determine the incoming argument types,
209 // because the types determine which registers the incoming arguments are
210 // in, and this affects the matched code.
211 const TypeTuple *domain = C->tf()->domain();
212 uint argcnt = domain->cnt() - TypeFunc::Parms;
213 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
214 VMRegPair *vm_parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
215 _parm_regs = NEW_RESOURCE_ARRAY( OptoRegPair, argcnt );
216 _calling_convention_mask = NEW_RESOURCE_ARRAY( RegMask, argcnt );
217 uint i;
218 for( i = 0; i<argcnt; i++ ) {
219 sig_bt[i] = domain->field_at(i+TypeFunc::Parms)->basic_type();
220 new (_calling_convention_mask + i) RegMask(C->comp_arena());
221 }
222
223 // Pass array of ideal registers and length to USER code (from the AD file)
224 // that will convert this to an array of register numbers.
225 const StartNode *start = C->start();
226 start->calling_convention( sig_bt, vm_parm_regs, argcnt );
227 #ifdef ASSERT
228 // Sanity check users' calling convention. Real handy while trying to
229 // get the initial port correct.
230 { for (uint i = 0; i<argcnt; i++) {
231 if( !vm_parm_regs[i].first()->is_valid() && !vm_parm_regs[i].second()->is_valid() ) {
462 // Initialize empty placeholder masks into the newly allocated arena
463 for (int i = 0; i < NOF_STACK_MASKS; i++) {
464 new (rms + i) RegMask(C->comp_arena());
465 }
466
467 int index = 0;
468 for (int i = Op_RegN; i <= Op_RegVectMask; ++i) {
469 idealreg2spillmask[i] = &rms[index++];
470 idealreg2debugmask[i] = &rms[index++];
471 }
472 assert(index == NOF_STACK_MASKS, "wrong size");
473
474 // At first, start with the empty mask
475 C->FIRST_STACK_mask().clear();
476
477 // Add in the incoming argument area
478 OptoReg::Name init_in = OptoReg::add(_old_SP, C->out_preserve_stack_slots());
479 for (OptoReg::Name i = init_in; i < _in_arg_limit; i = OptoReg::add(i, 1)) {
480 C->FIRST_STACK_mask().insert(i);
481 }
482 // Add in all bits past the outgoing argument area
483 C->FIRST_STACK_mask().set_all_from(_out_arg_limit);
484
485 // Make spill masks. Registers for their class, plus FIRST_STACK_mask.
486 RegMask aligned_stack_mask(C->FIRST_STACK_mask(), C->comp_arena());
487 // Keep spill masks aligned.
488 aligned_stack_mask.clear_to_pairs();
489 assert(aligned_stack_mask.is_infinite_stack(), "should be infinite stack");
490 RegMask scalable_stack_mask(aligned_stack_mask, C->comp_arena());
491
492 idealreg2spillmask[Op_RegP]->assignFrom(*idealreg2regmask[Op_RegP]);
493 #ifdef _LP64
494 idealreg2spillmask[Op_RegN]->assignFrom(*idealreg2regmask[Op_RegN]);
495 idealreg2spillmask[Op_RegN]->or_with(C->FIRST_STACK_mask());
496 idealreg2spillmask[Op_RegP]->or_with(aligned_stack_mask);
497 #else
498 idealreg2spillmask[Op_RegP]->or_with(C->FIRST_STACK_mask());
499 #endif
500 idealreg2spillmask[Op_RegI]->assignFrom(*idealreg2regmask[Op_RegI]);
501 idealreg2spillmask[Op_RegI]->or_with(C->FIRST_STACK_mask());
682 _register_save_policy[reg] == 'E' ||
683 _register_save_policy[reg] == 'A'; // Save-on-entry register?
684 }
685
686 //---------------------------Fixup_Save_On_Entry-------------------------------
687 void Matcher::Fixup_Save_On_Entry( ) {
688 init_first_stack_mask();
689
690 Node *root = C->root(); // Short name for root
691 // Count number of save-on-entry registers.
692 uint soe_cnt = number_of_saved_registers();
693 uint i;
694
695 // Find the procedure Start Node
696 StartNode *start = C->start();
697 assert( start, "Expect a start node" );
698
699 // Input RegMask array shared by all Returns.
700 // The type for doubles and longs has a count of 2, but
701 // there is only 1 returned value
702 uint ret_edge_cnt = TypeFunc::Parms + ((C->tf()->range()->cnt() == TypeFunc::Parms) ? 0 : 1);
703 RegMask *ret_rms = init_input_masks( ret_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
704 // Returns have 0 or 1 returned values depending on call signature.
705 // Return register is specified by return_value in the AD file.
706 if (ret_edge_cnt > TypeFunc::Parms) {
707 ret_rms[TypeFunc::Parms + 0].assignFrom(_return_value_mask);
708 }
709
710 // Input RegMask array shared by all ForwardExceptions
711 uint forw_exc_edge_cnt = TypeFunc::Parms;
712 RegMask* forw_exc_rms = init_input_masks( forw_exc_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
713
714 // Input RegMask array shared by all Rethrows.
715 uint reth_edge_cnt = TypeFunc::Parms+1;
716 RegMask *reth_rms = init_input_masks( reth_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
717 // Rethrow takes exception oop only, but in the argument 0 slot.
718 OptoReg::Name reg = find_receiver();
719 if (reg >= 0) {
720 reth_rms[TypeFunc::Parms].assignFrom(mreg2regmask[reg]);
721 #ifdef _LP64
722 // Need two slots for ptrs in 64-bit land
723 reth_rms[TypeFunc::Parms].insert(OptoReg::add(OptoReg::Name(reg), 1));
724 #endif
725 }
726
727 // Input RegMask array shared by all TailCalls
760
761 // Input RegMask array shared by all Halts
762 uint halt_edge_cnt = TypeFunc::Parms;
763 RegMask *halt_rms = init_input_masks( halt_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
764
765 // Capture the return input masks into each exit flavor
766 for( i=1; i < root->req(); i++ ) {
767 MachReturnNode *exit = root->in(i)->as_MachReturn();
768 switch( exit->ideal_Opcode() ) {
769 case Op_Return : exit->_in_rms = ret_rms; break;
770 case Op_Rethrow : exit->_in_rms = reth_rms; break;
771 case Op_TailCall : exit->_in_rms = tail_call_rms; break;
772 case Op_TailJump : exit->_in_rms = tail_jump_rms; break;
773 case Op_ForwardException: exit->_in_rms = forw_exc_rms; break;
774 case Op_Halt : exit->_in_rms = halt_rms; break;
775 default : ShouldNotReachHere();
776 }
777 }
778
779 // Next unused projection number from Start.
780 int proj_cnt = C->tf()->domain()->cnt();
781
782 // Do all the save-on-entry registers. Make projections from Start for
783 // them, and give them a use at the exit points. To the allocator, they
784 // look like incoming register arguments.
785 for( i = 0; i < _last_Mach_Reg; i++ ) {
786 if( is_save_on_entry(i) ) {
787
788 // Add the save-on-entry to the mask array
789 ret_rms [ ret_edge_cnt].assignFrom(mreg2regmask[i]);
790 reth_rms [ reth_edge_cnt].assignFrom(mreg2regmask[i]);
791 tail_call_rms[tail_call_edge_cnt].assignFrom(mreg2regmask[i]);
792 tail_jump_rms[tail_jump_edge_cnt].assignFrom(mreg2regmask[i]);
793 forw_exc_rms [ forw_exc_edge_cnt].assignFrom(mreg2regmask[i]);
794 // Halts need the SOE registers, but only in the stack as debug info.
795 // A just-prior uncommon-trap or deoptimization will use the SOE regs.
796 halt_rms [ halt_edge_cnt].assignFrom(*idealreg2spillmask[_register_save_type[i]]);
797
798 Node *mproj;
799
800 // Is this a RegF low half of a RegD? Double up 2 adjacent RegF's
1040 m = n->is_SafePoint() ? match_sfpt(n->as_SafePoint()):match_tree(n);
1041 if (C->failing()) return nullptr;
1042 if (m == nullptr) { Matcher::soft_match_failure(); return nullptr; }
1043 if (n->is_MemBar()) {
1044 m->as_MachMemBar()->set_adr_type(n->adr_type());
1045 }
1046 } else { // Nothing the matcher cares about
1047 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Multi()) { // Projections?
1048 if (n->in(0)->is_Initialize() && n->as_Proj()->_con == TypeFunc::Memory) {
1049 // Initialize may have multiple NarrowMem projections. They would all match to identical raw mem MachProjs.
1050 // We don't need multiple MachProjs. Create one if none already exist, otherwise use existing one.
1051 m = n->in(0)->as_Initialize()->mem_mach_proj();
1052 if (m == nullptr && has_new_node(n->in(0))) {
1053 InitializeNode* new_init = new_node(n->in(0))->as_Initialize();
1054 m = new_init->mem_mach_proj();
1055 }
1056 assert(m == nullptr || m->is_MachProj(), "no mem projection yet or a MachProj created during matching");
1057 }
1058 if (m == nullptr) {
1059 // Convert to machine-dependent projection
1060 m = n->in(0)->as_Multi()->match( n->as_Proj(), this );
1061 NOT_PRODUCT(record_new2old(m, n);)
1062 }
1063 if (m->in(0) != nullptr) // m might be top
1064 collect_null_checks(m, n);
1065 } else { // Else just a regular 'ol guy
1066 m = n->clone(); // So just clone into new-space
1067 NOT_PRODUCT(record_new2old(m, n);)
1068 // Def-Use edges will be added incrementally as Uses
1069 // of this node are matched.
1070 assert(m->outcnt() == 0, "no Uses of this clone yet");
1071 }
1072 }
1073
1074 set_new_node(n, m); // Map old to new
1075 if (_old_node_note_array != nullptr) {
1076 Node_Notes* nn = C->locate_node_notes(_old_node_note_array,
1077 n->_idx);
1078 C->set_node_notes_at(m->_idx, nn);
1079 }
1080 DEBUG_ONLY(match_alias_type(C, n, m));
1176 return warped;
1177 }
1178 return OptoReg::as_OptoReg(reg);
1179 }
1180
1181
1182 //------------------------------match_sfpt-------------------------------------
1183 // Helper function to match call instructions. Calls match special.
1184 // They match alone with no children. Their children, the incoming
1185 // arguments, match normally.
1186 MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) {
1187 MachSafePointNode *msfpt = nullptr;
1188 MachCallNode *mcall = nullptr;
1189 uint cnt;
1190 // Split out case for SafePoint vs Call
1191 CallNode *call;
1192 const TypeTuple *domain;
1193 ciMethod* method = nullptr;
1194 if( sfpt->is_Call() ) {
1195 call = sfpt->as_Call();
1196 domain = call->tf()->domain();
1197 cnt = domain->cnt();
1198
1199 // Match just the call, nothing else
1200 MachNode *m = match_tree(call);
1201 if (C->failing()) return nullptr;
1202 if( m == nullptr ) { Matcher::soft_match_failure(); return nullptr; }
1203
1204 // Copy data from the Ideal SafePoint to the machine version
1205 mcall = m->as_MachCall();
1206
1207 mcall->set_tf( call->tf());
1208 mcall->set_entry_point( call->entry_point());
1209 mcall->set_cnt( call->cnt());
1210 mcall->set_guaranteed_safepoint(call->guaranteed_safepoint());
1211
1212 if( mcall->is_MachCallJava() ) {
1213 MachCallJavaNode *mcall_java = mcall->as_MachCallJava();
1214 const CallJavaNode *call_java = call->as_CallJava();
1215 assert(call_java->validate_symbolic_info(), "inconsistent info");
1216 method = call_java->method();
1252 for (uint i = 0; i < cnt; i++) {
1253 ::new (msfpt->_in_rms + i) RegMask(C->comp_arena());
1254 }
1255
1256 // Do all the pre-defined non-Empty register masks
1257 msfpt->_in_rms[TypeFunc::ReturnAdr].assignFrom(_return_addr_mask);
1258 msfpt->_in_rms[TypeFunc::FramePtr ].assignFrom(c_frame_ptr_mask);
1259
1260 // Place first outgoing argument can possibly be put.
1261 OptoReg::Name begin_out_arg_area = OptoReg::add(_new_SP, C->out_preserve_stack_slots());
1262 assert( is_even(begin_out_arg_area), "" );
1263 // Compute max outgoing register number per call site.
1264 OptoReg::Name out_arg_limit_per_call = begin_out_arg_area;
1265 // Calls to C may hammer extra stack slots above and beyond any arguments.
1266 // These are usually backing store for register arguments for varargs.
1267 if( call != nullptr && call->is_CallRuntime() )
1268 out_arg_limit_per_call = OptoReg::add(out_arg_limit_per_call,C->varargs_C_out_slots_killed());
1269
1270
1271 // Do the normal argument list (parameters) register masks
1272 int argcnt = cnt - TypeFunc::Parms;
1273 if( argcnt > 0 ) { // Skip it all if we have no args
1274 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
1275 VMRegPair *parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
1276 int i;
1277 for( i = 0; i < argcnt; i++ ) {
1278 sig_bt[i] = domain->field_at(i+TypeFunc::Parms)->basic_type();
1279 }
1280 // V-call to pick proper calling convention
1281 call->calling_convention( sig_bt, parm_regs, argcnt );
1282
1283 #ifdef ASSERT
1284 // Sanity check users' calling convention. Really handy during
1285 // the initial porting effort. Fairly expensive otherwise.
1286 { for (int i = 0; i<argcnt; i++) {
1287 if( !parm_regs[i].first()->is_valid() &&
1288 !parm_regs[i].second()->is_valid() ) continue;
1289 VMReg reg1 = parm_regs[i].first();
1290 VMReg reg2 = parm_regs[i].second();
1291 for (int j = 0; j < i; j++) {
1292 if( !parm_regs[j].first()->is_valid() &&
1293 !parm_regs[j].second()->is_valid() ) continue;
1294 VMReg reg3 = parm_regs[j].first();
1295 VMReg reg4 = parm_regs[j].second();
1296 if( !reg1->is_valid() ) {
1297 assert( !reg2->is_valid(), "valid halvsies" );
1298 } else if( !reg3->is_valid() ) {
1299 assert( !reg4->is_valid(), "valid halvsies" );
1300 } else {
1301 assert( reg1 != reg2, "calling conv. must produce distinct regs");
1302 assert( reg1 != reg3, "calling conv. must produce distinct regs");
1303 assert( reg1 != reg4, "calling conv. must produce distinct regs");
1304 assert( reg2 != reg3, "calling conv. must produce distinct regs");
1305 assert( reg2 != reg4 || !reg2->is_valid(), "calling conv. must produce distinct regs");
1306 assert( reg3 != reg4, "calling conv. must produce distinct regs");
1307 }
1308 }
1309 }
1310 }
1311 #endif
1312
1313 // Visit each argument. Compute its outgoing register mask.
1314 // Return results now can have 2 bits returned.
1315 // Compute max over all outgoing arguments both per call-site
1316 // and over the entire method.
1317 for( i = 0; i < argcnt; i++ ) {
1318 // Address of incoming argument mask to fill in
1319 RegMask *rm = &mcall->_in_rms[i+TypeFunc::Parms];
1320 VMReg first = parm_regs[i].first();
1321 VMReg second = parm_regs[i].second();
1322 if(!first->is_valid() &&
1323 !second->is_valid()) {
1324 continue; // Avoid Halves
1325 }
1326 // Handle case where arguments are in vector registers.
1327 if(call->in(TypeFunc::Parms + i)->bottom_type()->isa_vect()) {
1328 OptoReg::Name reg_fst = OptoReg::as_OptoReg(first);
1329 OptoReg::Name reg_snd = OptoReg::as_OptoReg(second);
1330 assert (reg_fst <= reg_snd, "fst=%d snd=%d", reg_fst, reg_snd);
1331 for (OptoReg::Name r = reg_fst; r <= reg_snd; r++) {
1332 rm->insert(r);
1333 }
1334 }
1335 // Grab first register, adjust stack slots and insert in mask.
1336 OptoReg::Name reg1 = warp_outgoing_stk_arg(first, begin_out_arg_area, out_arg_limit_per_call );
1337 if (OptoReg::is_valid(reg1))
1338 rm->insert(reg1);
1339 // Grab second register (if any), adjust stack slots and insert in mask.
1340 OptoReg::Name reg2 = warp_outgoing_stk_arg(second, begin_out_arg_area, out_arg_limit_per_call );
1341 if (OptoReg::is_valid(reg2))
1342 rm->insert(reg2);
1343 } // End of for all arguments
1344 }
1345
1346 // Compute the max stack slot killed by any call. These will not be
1347 // available for debug info, and will be used to adjust FIRST_STACK_mask
1348 // after all call sites have been visited.
1349 if( _out_arg_limit < out_arg_limit_per_call)
1350 _out_arg_limit = out_arg_limit_per_call;
1351
1352 if (mcall) {
1353 // Kill the outgoing argument area, including any non-argument holes and
1354 // any legacy C-killed slots. Use Fat-Projections to do the killing.
1355 // Since the max-per-method covers the max-per-call-site and debug info
1356 // is excluded on the max-per-method basis, debug info cannot land in
1357 // this killed area.
1358 uint r_cnt = mcall->tf()->range()->cnt();
1359 MachProjNode* proj = new MachProjNode(mcall, r_cnt + 10000, RegMask::EMPTY, MachProjNode::fat_proj);
1360 for (int i = begin_out_arg_area; i < out_arg_limit_per_call; i++) {
1361 proj->_rout.insert(OptoReg::Name(i));
1362 }
1363 if (!proj->_rout.is_empty()) {
1364 push_projection(proj);
1365 }
1366 }
1367 // Transfer the safepoint information from the call to the mcall
1368 // Move the JVMState list
1369 msfpt->set_jvms(sfpt->jvms());
1370 for (JVMState* jvms = msfpt->jvms(); jvms; jvms = jvms->caller()) {
1371 jvms->set_map(sfpt);
1372 }
1373
1374 // Debug inputs begin just after the last incoming parameter
1375 assert((mcall == nullptr) || (mcall->jvms() == nullptr) ||
1376 (mcall->jvms()->debug_start() + mcall->_jvmadj == mcall->tf()->domain()->cnt()), "");
1377
1378 // Add additional edges.
1379 if (msfpt->mach_constant_base_node_input() != (uint)-1 && !msfpt->is_MachCallLeaf()) {
1380 // For these calls we can not add MachConstantBase in expand(), as the
1381 // ins are not complete then.
1382 msfpt->ins_req(msfpt->mach_constant_base_node_input(), C->mach_constant_base_node());
1383 if (msfpt->jvms() &&
1384 msfpt->mach_constant_base_node_input() <= msfpt->jvms()->debug_start() + msfpt->_jvmadj) {
1385 // We added an edge before jvms, so we must adapt the position of the ins.
1386 msfpt->jvms()->adapt_position(+1);
1387 }
1388 }
1389
1390 // Registers killed by the call are set in the local scheduling pass
1391 // of Global Code Motion.
1392 return msfpt;
1393 }
1394
1395 //---------------------------match_tree----------------------------------------
1396 // Match a Ideal Node DAG - turn it into a tree; Label & Reduce. Used as part
2045 set_shared(n); // Flag as shared and
2046 if (n->is_DecodeNarrowPtr()) {
2047 // Oop field/array element loads must be shared but since
2048 // they are shared through a DecodeN they may appear to have
2049 // a single use so force sharing here.
2050 set_shared(n->in(1));
2051 }
2052 mstack.pop(); // remove node from stack
2053 continue;
2054 }
2055 nstate = Visit; // Not already visited; so visit now
2056 }
2057 if (nstate == Visit) {
2058 mstack.set_state(Post_Visit);
2059 set_visited(n); // Flag as visited now
2060 bool mem_op = false;
2061 int mem_addr_idx = MemNode::Address;
2062 if (find_shared_visit(mstack, n, nop, mem_op, mem_addr_idx)) {
2063 continue;
2064 }
2065 for (int i = n->req() - 1; i >= 0; --i) { // For my children
2066 Node* m = n->in(i); // Get ith input
2067 if (m == nullptr) {
2068 continue; // Ignore nulls
2069 }
2070 if (clone_node(n, m, mstack)) {
2071 continue;
2072 }
2073
2074 // Clone addressing expressions as they are "free" in memory access instructions
2075 if (mem_op && i == mem_addr_idx && m->is_AddP() &&
2076 // When there are other uses besides address expressions
2077 // put it on stack and mark as shared.
2078 !is_visited(m)) {
2079 // Some inputs for address expression are not put on stack
2080 // to avoid marking them as shared and forcing them into register
2081 // if they are used only in address expressions.
2082 // But they should be marked as shared if there are other uses
2083 // besides address expressions.
2084
2085 if (pd_clone_address_expressions(m->as_AddP(), mstack, address_visited)) {
2356 case Op_FmaHF:
2357 case Op_FmaVD:
2358 case Op_FmaVF:
2359 case Op_FmaVHF: {
2360 // Restructure into a binary tree for Matching.
2361 Node* pair = new BinaryNode(n->in(1), n->in(2));
2362 n->set_req(2, pair);
2363 n->set_req(1, n->in(3));
2364 n->del_req(3);
2365 break;
2366 }
2367 case Op_MulAddS2I: {
2368 Node* pair1 = new BinaryNode(n->in(1), n->in(2));
2369 Node* pair2 = new BinaryNode(n->in(3), n->in(4));
2370 n->set_req(1, pair1);
2371 n->set_req(2, pair2);
2372 n->del_req(4);
2373 n->del_req(3);
2374 break;
2375 }
2376 case Op_VectorCmpMasked:
2377 case Op_CopySignD:
2378 case Op_SignumVF:
2379 case Op_SignumVD:
2380 case Op_SignumF:
2381 case Op_SignumD: {
2382 Node* pair = new BinaryNode(n->in(2), n->in(3));
2383 n->set_req(2, pair);
2384 n->del_req(3);
2385 break;
2386 }
2387 case Op_VectorBlend:
2388 case Op_VectorInsert: {
2389 Node* pair = new BinaryNode(n->in(1), n->in(2));
2390 n->set_req(1, pair);
2391 n->set_req(2, n->in(3));
2392 n->del_req(3);
2393 break;
2394 }
2395 case Op_LoadVectorGatherMasked: // fall-through
2405 n->del_req(MemNode::ValueIn+2);
2406 pair = new BinaryNode(n->in(MemNode::ValueIn), n->in(MemNode::ValueIn+1));
2407 n->set_req(MemNode::ValueIn, pair);
2408 n->del_req(MemNode::ValueIn+1);
2409 break;
2410 }
2411 case Op_VectorMaskCmp: {
2412 n->set_req(1, new BinaryNode(n->in(1), n->in(2)));
2413 n->set_req(2, n->in(3));
2414 n->del_req(3);
2415 break;
2416 }
2417 case Op_PartialSubtypeCheck: {
2418 if (UseSecondarySupersTable && n->in(2)->is_Con()) {
2419 // PartialSubtypeCheck uses both constant and register operands for superclass input.
2420 n->set_req(2, new BinaryNode(n->in(2), n->in(2)));
2421 break;
2422 }
2423 break;
2424 }
2425 default:
2426 break;
2427 }
2428 }
2429
2430 #ifndef PRODUCT
2431 void Matcher::record_new2old(Node* newn, Node* old) {
2432 _new2old_map.map(newn->_idx, old);
2433 if (!_reused.test_set(old->_igv_idx)) {
2434 // Reuse the Ideal-level IGV identifier so that the node can be tracked
2435 // across matching. If there are multiple machine nodes expanded from the
2436 // same Ideal node, only one will reuse its IGV identifier.
2437 newn->_igv_idx = old->_igv_idx;
2438 }
2439 }
2440
2441 // machine-independent root to machine-dependent root
2442 void Matcher::dump_old2new_map() {
2443 _old2new_map.dump();
2444 }
|
149 Node* n = worklist.pop();
150 if (visited.test_set(n->_idx)) {
151 continue;
152 }
153 assert(C->node_arena()->contains(n), "dead node");
154 assert(!n->is_Initialize() || n->as_Initialize()->number_of_projs(TypeFunc::Memory) == 1,
155 "after matching, Initialize should have a single memory projection");
156 for (uint j = 0; j < n->req(); j++) {
157 Node* in = n->in(j);
158 if (in != nullptr) {
159 worklist.push(in);
160 }
161 }
162 for (DUIterator_Fast jmax, j = n->fast_outs(jmax); j < jmax; j++) {
163 worklist.push(n->fast_out(j));
164 }
165 }
166 }
167 #endif
168
169 // Array of RegMask, one per returned values (inline type instances can
170 // be returned as multiple return values, one per field)
171 RegMask* Matcher::return_values_mask(const TypeFunc* tf) {
172 const TypeTuple* range = tf->range_cc();
173 uint cnt = range->cnt() - TypeFunc::Parms;
174 if (cnt == 0) {
175 return nullptr;
176 }
177 RegMask* mask = NEW_RESOURCE_ARRAY(RegMask, cnt);
178 BasicType* sig_bt = NEW_RESOURCE_ARRAY(BasicType, cnt);
179 VMRegPair* vm_parm_regs = NEW_RESOURCE_ARRAY(VMRegPair, cnt);
180 for (uint i = 0; i < cnt; i++) {
181 sig_bt[i] = range->field_at(i+TypeFunc::Parms)->basic_type();
182 new (mask + i) RegMask();
183 }
184
185 int regs = SharedRuntime::java_return_convention(sig_bt, vm_parm_regs, cnt);
186 if (regs <= 0) {
187 // We ran out of registers to store the null marker for a nullable inline type return.
188 // Since it is only set in the 'call_epilog', we can simply put it on the stack.
189 assert(tf->returns_inline_type_as_fields(), "should have been tested during graph construction");
190 // TODO 8284443 Can we teach the register allocator to reserve a stack slot instead?
191 // mask[--cnt] = STACK_ONLY_mask does not work (test with -XX:+StressGCM)
192 int slot = C->fixed_slots() - 2;
193 if (C->needs_stack_repair()) {
194 slot -= 2; // Account for stack increment value
195 }
196 mask[--cnt].clear();
197 mask[cnt].insert(OptoReg::stack2reg(slot));
198 }
199 for (uint i = 0; i < cnt; i++) {
200 mask[i].clear();
201
202 OptoReg::Name reg1 = OptoReg::as_OptoReg(vm_parm_regs[i].first());
203 if (OptoReg::is_valid(reg1)) {
204 mask[i].insert(reg1);
205 }
206 OptoReg::Name reg2 = OptoReg::as_OptoReg(vm_parm_regs[i].second());
207 if (OptoReg::is_valid(reg2)) {
208 mask[i].insert(reg2);
209 }
210 }
211
212 return mask;
213 }
214
215 //---------------------------match---------------------------------------------
216 void Matcher::match( ) {
217 if( MaxLabelRootDepth < 100 ) { // Too small?
218 assert(false, "invalid MaxLabelRootDepth, increase it to 100 minimum");
219 MaxLabelRootDepth = 100;
220 }
221 // One-time initialization of some register masks.
222 init_spill_mask( C->root()->in(1) );
223 if (C->failing()) {
224 return;
225 }
226 assert(_return_addr_mask.is_empty(),
227 "return address mask must be empty initially");
228 _return_addr_mask.insert(return_addr());
229 #ifdef _LP64
230 // Pointers take 2 slots in 64-bit land
231 _return_addr_mask.insert(OptoReg::add(return_addr(), 1));
232 #endif
233
234 // Map Java-signature return types into return register-value
235 // machine registers.
236 _return_values_mask = return_values_mask(C->tf());
237
238 // ---------------
239 // Frame Layout
240
241 // Need the method signature to determine the incoming argument types,
242 // because the types determine which registers the incoming arguments are
243 // in, and this affects the matched code.
244 const TypeTuple *domain = C->tf()->domain_cc();
245 uint argcnt = domain->cnt() - TypeFunc::Parms;
246 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
247 VMRegPair *vm_parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
248 _parm_regs = NEW_RESOURCE_ARRAY( OptoRegPair, argcnt );
249 _calling_convention_mask = NEW_RESOURCE_ARRAY( RegMask, argcnt );
250 uint i;
251 for( i = 0; i<argcnt; i++ ) {
252 sig_bt[i] = domain->field_at(i+TypeFunc::Parms)->basic_type();
253 new (_calling_convention_mask + i) RegMask(C->comp_arena());
254 }
255
256 // Pass array of ideal registers and length to USER code (from the AD file)
257 // that will convert this to an array of register numbers.
258 const StartNode *start = C->start();
259 start->calling_convention( sig_bt, vm_parm_regs, argcnt );
260 #ifdef ASSERT
261 // Sanity check users' calling convention. Real handy while trying to
262 // get the initial port correct.
263 { for (uint i = 0; i<argcnt; i++) {
264 if( !vm_parm_regs[i].first()->is_valid() && !vm_parm_regs[i].second()->is_valid() ) {
495 // Initialize empty placeholder masks into the newly allocated arena
496 for (int i = 0; i < NOF_STACK_MASKS; i++) {
497 new (rms + i) RegMask(C->comp_arena());
498 }
499
500 int index = 0;
501 for (int i = Op_RegN; i <= Op_RegVectMask; ++i) {
502 idealreg2spillmask[i] = &rms[index++];
503 idealreg2debugmask[i] = &rms[index++];
504 }
505 assert(index == NOF_STACK_MASKS, "wrong size");
506
507 // At first, start with the empty mask
508 C->FIRST_STACK_mask().clear();
509
510 // Add in the incoming argument area
511 OptoReg::Name init_in = OptoReg::add(_old_SP, C->out_preserve_stack_slots());
512 for (OptoReg::Name i = init_in; i < _in_arg_limit; i = OptoReg::add(i, 1)) {
513 C->FIRST_STACK_mask().insert(i);
514 }
515
516 // Add in all bits past the outgoing argument area
517 C->FIRST_STACK_mask().set_all_from(_out_arg_limit);
518
519 // Make spill masks. Registers for their class, plus FIRST_STACK_mask.
520 RegMask aligned_stack_mask(C->FIRST_STACK_mask(), C->comp_arena());
521 // Keep spill masks aligned.
522 aligned_stack_mask.clear_to_pairs();
523 assert(aligned_stack_mask.is_infinite_stack(), "should be infinite stack");
524 RegMask scalable_stack_mask(aligned_stack_mask, C->comp_arena());
525
526 idealreg2spillmask[Op_RegP]->assignFrom(*idealreg2regmask[Op_RegP]);
527 #ifdef _LP64
528 idealreg2spillmask[Op_RegN]->assignFrom(*idealreg2regmask[Op_RegN]);
529 idealreg2spillmask[Op_RegN]->or_with(C->FIRST_STACK_mask());
530 idealreg2spillmask[Op_RegP]->or_with(aligned_stack_mask);
531 #else
532 idealreg2spillmask[Op_RegP]->or_with(C->FIRST_STACK_mask());
533 #endif
534 idealreg2spillmask[Op_RegI]->assignFrom(*idealreg2regmask[Op_RegI]);
535 idealreg2spillmask[Op_RegI]->or_with(C->FIRST_STACK_mask());
716 _register_save_policy[reg] == 'E' ||
717 _register_save_policy[reg] == 'A'; // Save-on-entry register?
718 }
719
720 //---------------------------Fixup_Save_On_Entry-------------------------------
721 void Matcher::Fixup_Save_On_Entry( ) {
722 init_first_stack_mask();
723
724 Node *root = C->root(); // Short name for root
725 // Count number of save-on-entry registers.
726 uint soe_cnt = number_of_saved_registers();
727 uint i;
728
729 // Find the procedure Start Node
730 StartNode *start = C->start();
731 assert( start, "Expect a start node" );
732
733 // Input RegMask array shared by all Returns.
734 // The type for doubles and longs has a count of 2, but
735 // there is only 1 returned value
736 uint ret_edge_cnt = C->tf()->range_cc()->cnt();
737 RegMask *ret_rms = init_input_masks( ret_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
738 for (i = TypeFunc::Parms; i < ret_edge_cnt; i++) {
739 ret_rms[i].assignFrom(_return_values_mask[i-TypeFunc::Parms]);
740 }
741
742 // Input RegMask array shared by all ForwardExceptions
743 uint forw_exc_edge_cnt = TypeFunc::Parms;
744 RegMask* forw_exc_rms = init_input_masks( forw_exc_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
745
746 // Input RegMask array shared by all Rethrows.
747 uint reth_edge_cnt = TypeFunc::Parms+1;
748 RegMask *reth_rms = init_input_masks( reth_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
749 // Rethrow takes exception oop only, but in the argument 0 slot.
750 OptoReg::Name reg = find_receiver();
751 if (reg >= 0) {
752 reth_rms[TypeFunc::Parms].assignFrom(mreg2regmask[reg]);
753 #ifdef _LP64
754 // Need two slots for ptrs in 64-bit land
755 reth_rms[TypeFunc::Parms].insert(OptoReg::add(OptoReg::Name(reg), 1));
756 #endif
757 }
758
759 // Input RegMask array shared by all TailCalls
792
793 // Input RegMask array shared by all Halts
794 uint halt_edge_cnt = TypeFunc::Parms;
795 RegMask *halt_rms = init_input_masks( halt_edge_cnt + soe_cnt, _return_addr_mask, c_frame_ptr_mask );
796
797 // Capture the return input masks into each exit flavor
798 for( i=1; i < root->req(); i++ ) {
799 MachReturnNode *exit = root->in(i)->as_MachReturn();
800 switch( exit->ideal_Opcode() ) {
801 case Op_Return : exit->_in_rms = ret_rms; break;
802 case Op_Rethrow : exit->_in_rms = reth_rms; break;
803 case Op_TailCall : exit->_in_rms = tail_call_rms; break;
804 case Op_TailJump : exit->_in_rms = tail_jump_rms; break;
805 case Op_ForwardException: exit->_in_rms = forw_exc_rms; break;
806 case Op_Halt : exit->_in_rms = halt_rms; break;
807 default : ShouldNotReachHere();
808 }
809 }
810
811 // Next unused projection number from Start.
812 int proj_cnt = C->tf()->domain_cc()->cnt();
813
814 // Do all the save-on-entry registers. Make projections from Start for
815 // them, and give them a use at the exit points. To the allocator, they
816 // look like incoming register arguments.
817 for( i = 0; i < _last_Mach_Reg; i++ ) {
818 if( is_save_on_entry(i) ) {
819
820 // Add the save-on-entry to the mask array
821 ret_rms [ ret_edge_cnt].assignFrom(mreg2regmask[i]);
822 reth_rms [ reth_edge_cnt].assignFrom(mreg2regmask[i]);
823 tail_call_rms[tail_call_edge_cnt].assignFrom(mreg2regmask[i]);
824 tail_jump_rms[tail_jump_edge_cnt].assignFrom(mreg2regmask[i]);
825 forw_exc_rms [ forw_exc_edge_cnt].assignFrom(mreg2regmask[i]);
826 // Halts need the SOE registers, but only in the stack as debug info.
827 // A just-prior uncommon-trap or deoptimization will use the SOE regs.
828 halt_rms [ halt_edge_cnt].assignFrom(*idealreg2spillmask[_register_save_type[i]]);
829
830 Node *mproj;
831
832 // Is this a RegF low half of a RegD? Double up 2 adjacent RegF's
1072 m = n->is_SafePoint() ? match_sfpt(n->as_SafePoint()):match_tree(n);
1073 if (C->failing()) return nullptr;
1074 if (m == nullptr) { Matcher::soft_match_failure(); return nullptr; }
1075 if (n->is_MemBar()) {
1076 m->as_MachMemBar()->set_adr_type(n->adr_type());
1077 }
1078 } else { // Nothing the matcher cares about
1079 if (n->is_Proj() && n->in(0) != nullptr && n->in(0)->is_Multi()) { // Projections?
1080 if (n->in(0)->is_Initialize() && n->as_Proj()->_con == TypeFunc::Memory) {
1081 // Initialize may have multiple NarrowMem projections. They would all match to identical raw mem MachProjs.
1082 // We don't need multiple MachProjs. Create one if none already exist, otherwise use existing one.
1083 m = n->in(0)->as_Initialize()->mem_mach_proj();
1084 if (m == nullptr && has_new_node(n->in(0))) {
1085 InitializeNode* new_init = new_node(n->in(0))->as_Initialize();
1086 m = new_init->mem_mach_proj();
1087 }
1088 assert(m == nullptr || m->is_MachProj(), "no mem projection yet or a MachProj created during matching");
1089 }
1090 if (m == nullptr) {
1091 // Convert to machine-dependent projection
1092 RegMask* mask = nullptr;
1093 if (n->in(0)->is_Call() && n->in(0)->as_Call()->tf()->returns_inline_type_as_fields()) {
1094 mask = return_values_mask(n->in(0)->as_Call()->tf());
1095 }
1096 m = n->in(0)->as_Multi()->match(n->as_Proj(), this, mask);
1097 NOT_PRODUCT(record_new2old(m, n);)
1098 }
1099 if (m->in(0) != nullptr) // m might be top
1100 collect_null_checks(m, n);
1101 } else { // Else just a regular 'ol guy
1102 m = n->clone(); // So just clone into new-space
1103 NOT_PRODUCT(record_new2old(m, n);)
1104 // Def-Use edges will be added incrementally as Uses
1105 // of this node are matched.
1106 assert(m->outcnt() == 0, "no Uses of this clone yet");
1107 }
1108 }
1109
1110 set_new_node(n, m); // Map old to new
1111 if (_old_node_note_array != nullptr) {
1112 Node_Notes* nn = C->locate_node_notes(_old_node_note_array,
1113 n->_idx);
1114 C->set_node_notes_at(m->_idx, nn);
1115 }
1116 DEBUG_ONLY(match_alias_type(C, n, m));
1212 return warped;
1213 }
1214 return OptoReg::as_OptoReg(reg);
1215 }
1216
1217
1218 //------------------------------match_sfpt-------------------------------------
1219 // Helper function to match call instructions. Calls match special.
1220 // They match alone with no children. Their children, the incoming
1221 // arguments, match normally.
1222 MachNode *Matcher::match_sfpt( SafePointNode *sfpt ) {
1223 MachSafePointNode *msfpt = nullptr;
1224 MachCallNode *mcall = nullptr;
1225 uint cnt;
1226 // Split out case for SafePoint vs Call
1227 CallNode *call;
1228 const TypeTuple *domain;
1229 ciMethod* method = nullptr;
1230 if( sfpt->is_Call() ) {
1231 call = sfpt->as_Call();
1232 domain = call->tf()->domain_cc();
1233 cnt = domain->cnt();
1234
1235 // Match just the call, nothing else
1236 MachNode *m = match_tree(call);
1237 if (C->failing()) return nullptr;
1238 if( m == nullptr ) { Matcher::soft_match_failure(); return nullptr; }
1239
1240 // Copy data from the Ideal SafePoint to the machine version
1241 mcall = m->as_MachCall();
1242
1243 mcall->set_tf( call->tf());
1244 mcall->set_entry_point( call->entry_point());
1245 mcall->set_cnt( call->cnt());
1246 mcall->set_guaranteed_safepoint(call->guaranteed_safepoint());
1247
1248 if( mcall->is_MachCallJava() ) {
1249 MachCallJavaNode *mcall_java = mcall->as_MachCallJava();
1250 const CallJavaNode *call_java = call->as_CallJava();
1251 assert(call_java->validate_symbolic_info(), "inconsistent info");
1252 method = call_java->method();
1288 for (uint i = 0; i < cnt; i++) {
1289 ::new (msfpt->_in_rms + i) RegMask(C->comp_arena());
1290 }
1291
1292 // Do all the pre-defined non-Empty register masks
1293 msfpt->_in_rms[TypeFunc::ReturnAdr].assignFrom(_return_addr_mask);
1294 msfpt->_in_rms[TypeFunc::FramePtr ].assignFrom(c_frame_ptr_mask);
1295
1296 // Place first outgoing argument can possibly be put.
1297 OptoReg::Name begin_out_arg_area = OptoReg::add(_new_SP, C->out_preserve_stack_slots());
1298 assert( is_even(begin_out_arg_area), "" );
1299 // Compute max outgoing register number per call site.
1300 OptoReg::Name out_arg_limit_per_call = begin_out_arg_area;
1301 // Calls to C may hammer extra stack slots above and beyond any arguments.
1302 // These are usually backing store for register arguments for varargs.
1303 if( call != nullptr && call->is_CallRuntime() )
1304 out_arg_limit_per_call = OptoReg::add(out_arg_limit_per_call,C->varargs_C_out_slots_killed());
1305
1306
1307 // Do the normal argument list (parameters) register masks
1308 // Null entry point is a special cast where the target of the call
1309 // is in a register.
1310 int adj = (call != nullptr && call->entry_point() == nullptr) ? 1 : 0;
1311 int argcnt = cnt - TypeFunc::Parms - adj;
1312 if( argcnt > 0 ) { // Skip it all if we have no args
1313 BasicType *sig_bt = NEW_RESOURCE_ARRAY( BasicType, argcnt );
1314 VMRegPair *parm_regs = NEW_RESOURCE_ARRAY( VMRegPair, argcnt );
1315 int i;
1316 for( i = 0; i < argcnt; i++ ) {
1317 sig_bt[i] = domain->field_at(i+TypeFunc::Parms+adj)->basic_type();
1318 }
1319 // V-call to pick proper calling convention
1320 call->calling_convention( sig_bt, parm_regs, argcnt );
1321
1322 #ifdef ASSERT
1323 // Sanity check users' calling convention. Really handy during
1324 // the initial porting effort. Fairly expensive otherwise.
1325 { for (int i = 0; i<argcnt; i++) {
1326 if( !parm_regs[i].first()->is_valid() &&
1327 !parm_regs[i].second()->is_valid() ) continue;
1328 VMReg reg1 = parm_regs[i].first();
1329 VMReg reg2 = parm_regs[i].second();
1330 for (int j = 0; j < i; j++) {
1331 if( !parm_regs[j].first()->is_valid() &&
1332 !parm_regs[j].second()->is_valid() ) continue;
1333 VMReg reg3 = parm_regs[j].first();
1334 VMReg reg4 = parm_regs[j].second();
1335 if( !reg1->is_valid() ) {
1336 assert( !reg2->is_valid(), "valid halvsies" );
1337 } else if( !reg3->is_valid() ) {
1338 assert( !reg4->is_valid(), "valid halvsies" );
1339 } else {
1340 assert( reg1 != reg2, "calling conv. must produce distinct regs");
1341 assert( reg1 != reg3, "calling conv. must produce distinct regs");
1342 assert( reg1 != reg4, "calling conv. must produce distinct regs");
1343 assert( reg2 != reg3, "calling conv. must produce distinct regs");
1344 assert( reg2 != reg4 || !reg2->is_valid(), "calling conv. must produce distinct regs");
1345 assert( reg3 != reg4, "calling conv. must produce distinct regs");
1346 }
1347 }
1348 }
1349 }
1350 #endif
1351
1352 // Visit each argument. Compute its outgoing register mask.
1353 // Return results now can have 2 bits returned.
1354 // Compute max over all outgoing arguments both per call-site
1355 // and over the entire method.
1356 for( i = 0; i < argcnt; i++ ) {
1357 // Address of incoming argument mask to fill in
1358 RegMask *rm = &mcall->_in_rms[i+TypeFunc::Parms+adj];
1359 VMReg first = parm_regs[i].first();
1360 VMReg second = parm_regs[i].second();
1361 if(!first->is_valid() &&
1362 !second->is_valid()) {
1363 continue; // Avoid Halves
1364 }
1365 // Handle case where arguments are in vector registers.
1366 if(call->in(TypeFunc::Parms + i)->bottom_type()->isa_vect()) {
1367 OptoReg::Name reg_fst = OptoReg::as_OptoReg(first);
1368 OptoReg::Name reg_snd = OptoReg::as_OptoReg(second);
1369 assert (reg_fst <= reg_snd, "fst=%d snd=%d", reg_fst, reg_snd);
1370 for (OptoReg::Name r = reg_fst; r <= reg_snd; r++) {
1371 rm->insert(r);
1372 }
1373 }
1374 // Grab first register, adjust stack slots and insert in mask.
1375 OptoReg::Name reg1 = warp_outgoing_stk_arg(first, begin_out_arg_area, out_arg_limit_per_call );
1376 if (OptoReg::is_valid(reg1)) {
1377 rm->insert( reg1 );
1378 }
1379 // Grab second register (if any), adjust stack slots and insert in mask.
1380 OptoReg::Name reg2 = warp_outgoing_stk_arg(second, begin_out_arg_area, out_arg_limit_per_call );
1381 if (OptoReg::is_valid(reg2)) {
1382 rm->insert( reg2 );
1383 }
1384 } // End of for all arguments
1385 }
1386
1387 // Compute the max stack slot killed by any call. These will not be
1388 // available for debug info, and will be used to adjust FIRST_STACK_mask
1389 // after all call sites have been visited.
1390 if( _out_arg_limit < out_arg_limit_per_call)
1391 _out_arg_limit = out_arg_limit_per_call;
1392
1393 if (mcall) {
1394 // Kill the outgoing argument area, including any non-argument holes and
1395 // any legacy C-killed slots. Use Fat-Projections to do the killing.
1396 // Since the max-per-method covers the max-per-call-site and debug info
1397 // is excluded on the max-per-method basis, debug info cannot land in
1398 // this killed area.
1399 uint r_cnt = mcall->tf()->range_sig()->cnt();
1400 MachProjNode *proj = new MachProjNode( mcall, r_cnt+10000, RegMask::EMPTY, MachProjNode::fat_proj );
1401 for (int i = begin_out_arg_area; i < out_arg_limit_per_call; i++) {
1402 proj->_rout.insert(OptoReg::Name(i));
1403 }
1404 if (!proj->_rout.is_empty()) {
1405 push_projection(proj);
1406 }
1407 }
1408 // Transfer the safepoint information from the call to the mcall
1409 // Move the JVMState list
1410 msfpt->set_jvms(sfpt->jvms());
1411 for (JVMState* jvms = msfpt->jvms(); jvms; jvms = jvms->caller()) {
1412 jvms->set_map(sfpt);
1413 }
1414
1415 // Debug inputs begin just after the last incoming parameter
1416 assert((mcall == nullptr) || (mcall->jvms() == nullptr) ||
1417 (mcall->jvms()->debug_start() + mcall->_jvmadj == mcall->tf()->domain_cc()->cnt()), "");
1418
1419 // Add additional edges.
1420 if (msfpt->mach_constant_base_node_input() != (uint)-1 && !msfpt->is_MachCallLeaf()) {
1421 // For these calls we can not add MachConstantBase in expand(), as the
1422 // ins are not complete then.
1423 msfpt->ins_req(msfpt->mach_constant_base_node_input(), C->mach_constant_base_node());
1424 if (msfpt->jvms() &&
1425 msfpt->mach_constant_base_node_input() <= msfpt->jvms()->debug_start() + msfpt->_jvmadj) {
1426 // We added an edge before jvms, so we must adapt the position of the ins.
1427 msfpt->jvms()->adapt_position(+1);
1428 }
1429 }
1430
1431 // Registers killed by the call are set in the local scheduling pass
1432 // of Global Code Motion.
1433 return msfpt;
1434 }
1435
1436 //---------------------------match_tree----------------------------------------
1437 // Match a Ideal Node DAG - turn it into a tree; Label & Reduce. Used as part
2086 set_shared(n); // Flag as shared and
2087 if (n->is_DecodeNarrowPtr()) {
2088 // Oop field/array element loads must be shared but since
2089 // they are shared through a DecodeN they may appear to have
2090 // a single use so force sharing here.
2091 set_shared(n->in(1));
2092 }
2093 mstack.pop(); // remove node from stack
2094 continue;
2095 }
2096 nstate = Visit; // Not already visited; so visit now
2097 }
2098 if (nstate == Visit) {
2099 mstack.set_state(Post_Visit);
2100 set_visited(n); // Flag as visited now
2101 bool mem_op = false;
2102 int mem_addr_idx = MemNode::Address;
2103 if (find_shared_visit(mstack, n, nop, mem_op, mem_addr_idx)) {
2104 continue;
2105 }
2106 for (int i = n->len() - 1; i >= 0; --i) { // For my children
2107 Node* m = n->in(i); // Get ith input
2108 if (m == nullptr) {
2109 continue; // Ignore nulls
2110 }
2111 if (clone_node(n, m, mstack)) {
2112 continue;
2113 }
2114
2115 // Clone addressing expressions as they are "free" in memory access instructions
2116 if (mem_op && i == mem_addr_idx && m->is_AddP() &&
2117 // When there are other uses besides address expressions
2118 // put it on stack and mark as shared.
2119 !is_visited(m)) {
2120 // Some inputs for address expression are not put on stack
2121 // to avoid marking them as shared and forcing them into register
2122 // if they are used only in address expressions.
2123 // But they should be marked as shared if there are other uses
2124 // besides address expressions.
2125
2126 if (pd_clone_address_expressions(m->as_AddP(), mstack, address_visited)) {
2397 case Op_FmaHF:
2398 case Op_FmaVD:
2399 case Op_FmaVF:
2400 case Op_FmaVHF: {
2401 // Restructure into a binary tree for Matching.
2402 Node* pair = new BinaryNode(n->in(1), n->in(2));
2403 n->set_req(2, pair);
2404 n->set_req(1, n->in(3));
2405 n->del_req(3);
2406 break;
2407 }
2408 case Op_MulAddS2I: {
2409 Node* pair1 = new BinaryNode(n->in(1), n->in(2));
2410 Node* pair2 = new BinaryNode(n->in(3), n->in(4));
2411 n->set_req(1, pair1);
2412 n->set_req(2, pair2);
2413 n->del_req(4);
2414 n->del_req(3);
2415 break;
2416 }
2417 case Op_ClearArray: {
2418 Node* pair = new BinaryNode(n->in(2), n->in(3));
2419 n->set_req(2, pair);
2420 n->set_req(3, n->in(4));
2421 n->del_req(4);
2422 break;
2423 }
2424 case Op_VectorCmpMasked:
2425 case Op_CopySignD:
2426 case Op_SignumVF:
2427 case Op_SignumVD:
2428 case Op_SignumF:
2429 case Op_SignumD: {
2430 Node* pair = new BinaryNode(n->in(2), n->in(3));
2431 n->set_req(2, pair);
2432 n->del_req(3);
2433 break;
2434 }
2435 case Op_VectorBlend:
2436 case Op_VectorInsert: {
2437 Node* pair = new BinaryNode(n->in(1), n->in(2));
2438 n->set_req(1, pair);
2439 n->set_req(2, n->in(3));
2440 n->del_req(3);
2441 break;
2442 }
2443 case Op_LoadVectorGatherMasked: // fall-through
2453 n->del_req(MemNode::ValueIn+2);
2454 pair = new BinaryNode(n->in(MemNode::ValueIn), n->in(MemNode::ValueIn+1));
2455 n->set_req(MemNode::ValueIn, pair);
2456 n->del_req(MemNode::ValueIn+1);
2457 break;
2458 }
2459 case Op_VectorMaskCmp: {
2460 n->set_req(1, new BinaryNode(n->in(1), n->in(2)));
2461 n->set_req(2, n->in(3));
2462 n->del_req(3);
2463 break;
2464 }
2465 case Op_PartialSubtypeCheck: {
2466 if (UseSecondarySupersTable && n->in(2)->is_Con()) {
2467 // PartialSubtypeCheck uses both constant and register operands for superclass input.
2468 n->set_req(2, new BinaryNode(n->in(2), n->in(2)));
2469 break;
2470 }
2471 break;
2472 }
2473 case Op_StoreLSpecial: {
2474 if (n->req() > (MemNode::ValueIn + 1) && n->in(MemNode::ValueIn + 1) != nullptr) {
2475 Node* pair = new BinaryNode(n->in(MemNode::ValueIn), n->in(MemNode::ValueIn + 1));
2476 n->set_req(MemNode::ValueIn, pair);
2477 n->del_req(MemNode::ValueIn + 1);
2478 }
2479 break;
2480 }
2481 default:
2482 break;
2483 }
2484 }
2485
2486 #ifndef PRODUCT
2487 void Matcher::record_new2old(Node* newn, Node* old) {
2488 _new2old_map.map(newn->_idx, old);
2489 if (!_reused.test_set(old->_igv_idx)) {
2490 // Reuse the Ideal-level IGV identifier so that the node can be tracked
2491 // across matching. If there are multiple machine nodes expanded from the
2492 // same Ideal node, only one will reuse its IGV identifier.
2493 newn->_igv_idx = old->_igv_idx;
2494 }
2495 }
2496
2497 // machine-independent root to machine-dependent root
2498 void Matcher::dump_old2new_map() {
2499 _old2new_map.dump();
2500 }
|