1 /* 2 * Copyright (c) 2007, 2022, Oracle and/or its affiliates. All rights reserved. 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 4 * 5 * This code is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License version 2 only, as 7 * published by the Free Software Foundation. 8 * 9 * This code is distributed in the hope that it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 12 * version 2 for more details (a copy is included in the LICENSE file that 13 * accompanied this code). 14 * 15 * You should have received a copy of the GNU General Public License version 16 * 2 along with this work; if not, write to the Free Software Foundation, 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 18 * 19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 20 * or visit www.oracle.com if you need additional information or have any 21 * questions. 22 */ 23 24 #include "precompiled.hpp" 25 #include "memory/allocation.inline.hpp" 26 #include "opto/connode.hpp" 27 #include "opto/mulnode.hpp" 28 #include "opto/subnode.hpp" 29 #include "opto/vectornode.hpp" 30 #include "opto/convertnode.hpp" 31 #include "utilities/powerOfTwo.hpp" 32 #include "utilities/globalDefinitions.hpp" 33 34 //------------------------------VectorNode-------------------------------------- 35 36 // Return the vector operator for the specified scalar operation 37 // and vector length. 38 int VectorNode::opcode(int sopc, BasicType bt) { 39 switch (sopc) { 40 case Op_AddI: 41 switch (bt) { 42 case T_BOOLEAN: 43 case T_BYTE: return Op_AddVB; 44 case T_CHAR: 45 case T_SHORT: return Op_AddVS; 46 case T_INT: return Op_AddVI; 47 default: return 0; 48 } 49 case Op_AddL: return (bt == T_LONG ? Op_AddVL : 0); 50 case Op_AddF: return (bt == T_FLOAT ? Op_AddVF : 0); 51 case Op_AddD: return (bt == T_DOUBLE ? Op_AddVD : 0); 52 53 case Op_SubI: 54 switch (bt) { 55 case T_BOOLEAN: 56 case T_BYTE: return Op_SubVB; 57 case T_CHAR: 58 case T_SHORT: return Op_SubVS; 59 case T_INT: return Op_SubVI; 60 default: return 0; 61 } 62 case Op_SubL: return (bt == T_LONG ? Op_SubVL : 0); 63 case Op_SubF: return (bt == T_FLOAT ? Op_SubVF : 0); 64 case Op_SubD: return (bt == T_DOUBLE ? Op_SubVD : 0); 65 66 case Op_MulI: 67 switch (bt) { 68 case T_BOOLEAN:return 0; 69 case T_BYTE: return Op_MulVB; 70 case T_CHAR: 71 case T_SHORT: return Op_MulVS; 72 case T_INT: return Op_MulVI; 73 default: return 0; 74 } 75 case Op_MulL: return (bt == T_LONG ? Op_MulVL : 0); 76 case Op_MulF: 77 return (bt == T_FLOAT ? Op_MulVF : 0); 78 case Op_MulD: 79 return (bt == T_DOUBLE ? Op_MulVD : 0); 80 case Op_FmaD: 81 return (bt == T_DOUBLE ? Op_FmaVD : 0); 82 case Op_FmaF: 83 return (bt == T_FLOAT ? Op_FmaVF : 0); 84 case Op_CMoveF: 85 return (bt == T_FLOAT ? Op_CMoveVF : 0); 86 case Op_CMoveD: 87 return (bt == T_DOUBLE ? Op_CMoveVD : 0); 88 case Op_DivF: 89 return (bt == T_FLOAT ? Op_DivVF : 0); 90 case Op_DivD: 91 return (bt == T_DOUBLE ? Op_DivVD : 0); 92 case Op_AbsI: 93 switch (bt) { 94 case T_BOOLEAN: 95 case T_CHAR: return 0; // abs does not make sense for unsigned 96 case T_BYTE: return Op_AbsVB; 97 case T_SHORT: return Op_AbsVS; 98 case T_INT: return Op_AbsVI; 99 default: return 0; 100 } 101 case Op_AbsL: 102 return (bt == T_LONG ? Op_AbsVL : 0); 103 case Op_MinI: 104 switch (bt) { 105 case T_BOOLEAN: 106 case T_CHAR: return 0; 107 case T_BYTE: 108 case T_SHORT: 109 case T_INT: return Op_MinV; 110 default: return 0; 111 } 112 case Op_MinL: 113 return (bt == T_LONG ? Op_MinV : 0); 114 case Op_MinF: 115 return (bt == T_FLOAT ? Op_MinV : 0); 116 case Op_MinD: 117 return (bt == T_DOUBLE ? Op_MinV : 0); 118 case Op_MaxI: 119 switch (bt) { 120 case T_BOOLEAN: 121 case T_CHAR: return 0; 122 case T_BYTE: 123 case T_SHORT: 124 case T_INT: return Op_MaxV; 125 default: return 0; 126 } 127 case Op_MaxL: 128 return (bt == T_LONG ? Op_MaxV : 0); 129 case Op_MaxF: 130 return (bt == T_FLOAT ? Op_MaxV : 0); 131 case Op_MaxD: 132 return (bt == T_DOUBLE ? Op_MaxV : 0); 133 case Op_AbsF: 134 return (bt == T_FLOAT ? Op_AbsVF : 0); 135 case Op_AbsD: 136 return (bt == T_DOUBLE ? Op_AbsVD : 0); 137 case Op_NegI: 138 switch (bt) { 139 case T_BYTE: 140 case T_SHORT: 141 case T_INT: return Op_NegVI; 142 default: return 0; 143 } 144 case Op_NegL: 145 return (bt == T_LONG ? Op_NegVL : 0); 146 case Op_NegF: 147 return (bt == T_FLOAT ? Op_NegVF : 0); 148 case Op_NegD: 149 return (bt == T_DOUBLE ? Op_NegVD : 0); 150 case Op_RoundDoubleMode: 151 return (bt == T_DOUBLE ? Op_RoundDoubleModeV : 0); 152 case Op_RotateLeft: 153 return (is_integral_type(bt) ? Op_RotateLeftV : 0); 154 case Op_RotateRight: 155 return (is_integral_type(bt) ? Op_RotateRightV : 0); 156 case Op_SqrtF: 157 return (bt == T_FLOAT ? Op_SqrtVF : 0); 158 case Op_SqrtD: 159 return (bt == T_DOUBLE ? Op_SqrtVD : 0); 160 case Op_RoundF: 161 return (bt == T_INT ? Op_RoundVF : 0); 162 case Op_RoundD: 163 return (bt == T_LONG ? Op_RoundVD : 0); 164 case Op_PopCountI: 165 return Op_PopCountVI; 166 case Op_PopCountL: 167 return Op_PopCountVL; 168 case Op_ReverseI: 169 case Op_ReverseL: 170 return (is_integral_type(bt) ? Op_ReverseV : 0); 171 case Op_ReverseBytesS: 172 case Op_ReverseBytesI: 173 case Op_ReverseBytesL: 174 return (is_integral_type(bt) ? Op_ReverseBytesV : 0); 175 case Op_CompressBits: 176 // Not implemented. Returning 0 temporarily 177 return 0; 178 case Op_ExpandBits: 179 // Not implemented. Returning 0 temporarily 180 return 0; 181 case Op_LShiftI: 182 switch (bt) { 183 case T_BOOLEAN: 184 case T_BYTE: return Op_LShiftVB; 185 case T_CHAR: 186 case T_SHORT: return Op_LShiftVS; 187 case T_INT: return Op_LShiftVI; 188 default: return 0; 189 } 190 case Op_LShiftL: 191 return (bt == T_LONG ? Op_LShiftVL : 0); 192 case Op_RShiftI: 193 switch (bt) { 194 case T_BOOLEAN:return Op_URShiftVB; // boolean is unsigned value 195 case T_CHAR: return Op_URShiftVS; // char is unsigned value 196 case T_BYTE: return Op_RShiftVB; 197 case T_SHORT: return Op_RShiftVS; 198 case T_INT: return Op_RShiftVI; 199 default: return 0; 200 } 201 case Op_RShiftL: 202 return (bt == T_LONG ? Op_RShiftVL : 0); 203 case Op_URShiftB: 204 return (bt == T_BYTE ? Op_URShiftVB : 0); 205 case Op_URShiftS: 206 return (bt == T_SHORT ? Op_URShiftVS : 0); 207 case Op_URShiftI: 208 switch (bt) { 209 case T_BOOLEAN:return Op_URShiftVB; 210 case T_CHAR: return Op_URShiftVS; 211 case T_BYTE: 212 case T_SHORT: return 0; // Vector logical right shift for signed short 213 // values produces incorrect Java result for 214 // negative data because java code should convert 215 // a short value into int value with sign 216 // extension before a shift. 217 case T_INT: return Op_URShiftVI; 218 default: return 0; 219 } 220 case Op_URShiftL: 221 return (bt == T_LONG ? Op_URShiftVL : 0); 222 case Op_AndI: 223 case Op_AndL: 224 return Op_AndV; 225 case Op_OrI: 226 case Op_OrL: 227 return Op_OrV; 228 case Op_XorI: 229 case Op_XorL: 230 return Op_XorV; 231 232 case Op_LoadB: 233 case Op_LoadUB: 234 case Op_LoadUS: 235 case Op_LoadS: 236 case Op_LoadI: 237 case Op_LoadL: 238 case Op_LoadF: 239 case Op_LoadD: 240 return Op_LoadVector; 241 242 case Op_StoreB: 243 case Op_StoreC: 244 case Op_StoreI: 245 case Op_StoreL: 246 case Op_StoreF: 247 case Op_StoreD: 248 return Op_StoreVector; 249 case Op_MulAddS2I: 250 return Op_MulAddVS2VI; 251 case Op_ConvI2F: 252 return Op_VectorCastI2X; 253 case Op_ConvL2D: 254 return Op_VectorCastL2X; 255 case Op_ConvF2I: 256 return Op_VectorCastF2X; 257 case Op_ConvD2L: 258 return Op_VectorCastD2X; 259 case Op_CountLeadingZerosI: 260 case Op_CountLeadingZerosL: 261 return Op_CountLeadingZerosV; 262 case Op_CountTrailingZerosI: 263 case Op_CountTrailingZerosL: 264 return Op_CountTrailingZerosV; 265 266 default: 267 return 0; // Unimplemented 268 } 269 } 270 271 int VectorNode::replicate_opcode(BasicType bt) { 272 switch(bt) { 273 case T_BOOLEAN: 274 case T_BYTE: 275 return Op_ReplicateB; 276 case T_SHORT: 277 case T_CHAR: 278 return Op_ReplicateS; 279 case T_INT: 280 return Op_ReplicateI; 281 case T_LONG: 282 return Op_ReplicateL; 283 case T_FLOAT: 284 return Op_ReplicateF; 285 case T_DOUBLE: 286 return Op_ReplicateD; 287 default: 288 assert(false, "wrong type: %s", type2name(bt)); 289 return 0; 290 } 291 } 292 293 // Also used to check if the code generator 294 // supports the vector operation. 295 bool VectorNode::implemented(int opc, uint vlen, BasicType bt) { 296 if (is_java_primitive(bt) && 297 (vlen > 1) && is_power_of_2(vlen) && 298 Matcher::vector_size_supported(bt, vlen)) { 299 int vopc = VectorNode::opcode(opc, bt); 300 // For rotate operation we will do a lazy de-generation into 301 // OrV/LShiftV/URShiftV pattern if the target does not support 302 // vector rotation instruction. 303 if (VectorNode::is_vector_rotate(vopc)) { 304 return is_vector_rotate_supported(vopc, vlen, bt); 305 } 306 if (VectorNode::is_vector_integral_negate(vopc)) { 307 return is_vector_integral_negate_supported(vopc, vlen, bt, false); 308 } 309 return vopc > 0 && Matcher::match_rule_supported_vector(vopc, vlen, bt); 310 } 311 return false; 312 } 313 314 bool VectorNode::is_type_transition_short_to_int(Node* n) { 315 switch (n->Opcode()) { 316 case Op_MulAddS2I: 317 return true; 318 } 319 return false; 320 } 321 322 bool VectorNode::is_type_transition_to_int(Node* n) { 323 return is_type_transition_short_to_int(n); 324 } 325 326 bool VectorNode::is_muladds2i(Node* n) { 327 if (n->Opcode() == Op_MulAddS2I) { 328 return true; 329 } 330 return false; 331 } 332 333 bool VectorNode::is_type_transition_long_to_int(Node* n) { 334 switch(n->Opcode()) { 335 case Op_PopCountL: 336 case Op_CountLeadingZerosL: 337 case Op_CountTrailingZerosL: 338 return true; 339 default: 340 return false; 341 } 342 } 343 344 bool VectorNode::is_roundopD(Node* n) { 345 if (n->Opcode() == Op_RoundDoubleMode) { 346 return true; 347 } 348 return false; 349 } 350 351 bool VectorNode::is_vector_rotate_supported(int vopc, uint vlen, BasicType bt) { 352 assert(VectorNode::is_vector_rotate(vopc), "wrong opcode"); 353 354 // If target defines vector rotation patterns then no 355 // need for degeneration. 356 if (Matcher::match_rule_supported_vector(vopc, vlen, bt)) { 357 return true; 358 } 359 360 // If target does not support variable shift operations then no point 361 // in creating a rotate vector node since it will not be disintegratable. 362 // Adding a pessimistic check to avoid complex pattern matching which 363 // may not be full proof. 364 if (!Matcher::supports_vector_variable_shifts()) { 365 return false; 366 } 367 368 // Validate existence of nodes created in case of rotate degeneration. 369 switch (bt) { 370 case T_INT: 371 return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) && 372 Matcher::match_rule_supported_vector(Op_LShiftVI, vlen, bt) && 373 Matcher::match_rule_supported_vector(Op_URShiftVI, vlen, bt); 374 case T_LONG: 375 return Matcher::match_rule_supported_vector(Op_OrV, vlen, bt) && 376 Matcher::match_rule_supported_vector(Op_LShiftVL, vlen, bt) && 377 Matcher::match_rule_supported_vector(Op_URShiftVL, vlen, bt); 378 default: 379 assert(false, "not supported: %s", type2name(bt)); 380 return false; 381 } 382 } 383 384 // Check whether the architecture supports the vector negate instructions. If not, then check 385 // whether the alternative vector nodes used to implement vector negation are supported. 386 // Return false if neither of them is supported. 387 bool VectorNode::is_vector_integral_negate_supported(int opc, uint vlen, BasicType bt, bool use_predicate) { 388 if (!use_predicate) { 389 // Check whether the NegVI/L is supported by the architecture. 390 if (Matcher::match_rule_supported_vector(opc, vlen, bt)) { 391 return true; 392 } 393 // Negate is implemented with "(SubVI/L (ReplicateI/L 0) src)", if NegVI/L is not supported. 394 int sub_opc = (bt == T_LONG) ? Op_SubL : Op_SubI; 395 if (Matcher::match_rule_supported_vector(VectorNode::opcode(sub_opc, bt), vlen, bt) && 396 Matcher::match_rule_supported_vector(VectorNode::replicate_opcode(bt), vlen, bt)) { 397 return true; 398 } 399 } else { 400 // Check whether the predicated NegVI/L is supported by the architecture. 401 if (Matcher::match_rule_supported_vector_masked(opc, vlen, bt)) { 402 return true; 403 } 404 // Predicated negate is implemented with "(AddVI/L (XorV src (ReplicateI/L -1)) (ReplicateI/L 1))", 405 // if predicated NegVI/L is not supported. 406 int add_opc = (bt == T_LONG) ? Op_AddL : Op_AddI; 407 if (Matcher::match_rule_supported_vector_masked(Op_XorV, vlen, bt) && 408 Matcher::match_rule_supported_vector_masked(VectorNode::opcode(add_opc, bt), vlen, bt) && 409 Matcher::match_rule_supported_vector(VectorNode::replicate_opcode(bt), vlen, bt)) { 410 return true; 411 } 412 } 413 return false; 414 } 415 416 bool VectorNode::is_shift_opcode(int opc) { 417 switch (opc) { 418 case Op_LShiftI: 419 case Op_LShiftL: 420 case Op_RShiftI: 421 case Op_RShiftL: 422 case Op_URShiftB: 423 case Op_URShiftS: 424 case Op_URShiftI: 425 case Op_URShiftL: 426 return true; 427 default: 428 return false; 429 } 430 } 431 432 bool VectorNode::is_shift(Node* n) { 433 return is_shift_opcode(n->Opcode()); 434 } 435 436 bool VectorNode::is_rotate_opcode(int opc) { 437 switch (opc) { 438 case Op_RotateRight: 439 case Op_RotateLeft: 440 return true; 441 default: 442 return false; 443 } 444 } 445 446 bool VectorNode::is_scalar_rotate(Node* n) { 447 if (is_rotate_opcode(n->Opcode())) { 448 return true; 449 } 450 return false; 451 } 452 453 bool VectorNode::is_vshift_cnt_opcode(int opc) { 454 switch (opc) { 455 case Op_LShiftCntV: 456 case Op_RShiftCntV: 457 return true; 458 default: 459 return false; 460 } 461 } 462 463 bool VectorNode::is_vshift_cnt(Node* n) { 464 return is_vshift_cnt_opcode(n->Opcode()); 465 } 466 467 // Check if input is loop invariant vector. 468 bool VectorNode::is_invariant_vector(Node* n) { 469 // Only Replicate vector nodes are loop invariant for now. 470 switch (n->Opcode()) { 471 case Op_ReplicateB: 472 case Op_ReplicateS: 473 case Op_ReplicateI: 474 case Op_ReplicateL: 475 case Op_ReplicateF: 476 case Op_ReplicateD: 477 return true; 478 default: 479 return false; 480 } 481 } 482 483 // [Start, end) half-open range defining which operands are vectors 484 void VectorNode::vector_operands(Node* n, uint* start, uint* end) { 485 switch (n->Opcode()) { 486 case Op_LoadB: case Op_LoadUB: 487 case Op_LoadS: case Op_LoadUS: 488 case Op_LoadI: case Op_LoadL: 489 case Op_LoadF: case Op_LoadD: 490 case Op_LoadP: case Op_LoadN: 491 *start = 0; 492 *end = 0; // no vector operands 493 break; 494 case Op_StoreB: case Op_StoreC: 495 case Op_StoreI: case Op_StoreL: 496 case Op_StoreF: case Op_StoreD: 497 case Op_StoreP: case Op_StoreN: 498 *start = MemNode::ValueIn; 499 *end = MemNode::ValueIn + 1; // 1 vector operand 500 break; 501 case Op_LShiftI: case Op_LShiftL: 502 case Op_RShiftI: case Op_RShiftL: 503 case Op_URShiftI: case Op_URShiftL: 504 *start = 1; 505 *end = 2; // 1 vector operand 506 break; 507 case Op_AddI: case Op_AddL: case Op_AddF: case Op_AddD: 508 case Op_SubI: case Op_SubL: case Op_SubF: case Op_SubD: 509 case Op_MulI: case Op_MulL: case Op_MulF: case Op_MulD: 510 case Op_DivF: case Op_DivD: 511 case Op_AndI: case Op_AndL: 512 case Op_OrI: case Op_OrL: 513 case Op_XorI: case Op_XorL: 514 case Op_MulAddS2I: 515 *start = 1; 516 *end = 3; // 2 vector operands 517 break; 518 case Op_CMoveI: case Op_CMoveL: case Op_CMoveF: case Op_CMoveD: 519 *start = 2; 520 *end = n->req(); 521 break; 522 case Op_FmaD: 523 case Op_FmaF: 524 *start = 1; 525 *end = 4; // 3 vector operands 526 break; 527 default: 528 *start = 1; 529 *end = n->req(); // default is all operands 530 } 531 } 532 533 VectorNode* VectorNode::make_mask_node(int vopc, Node* n1, Node* n2, uint vlen, BasicType bt) { 534 guarantee(vopc > 0, "vopc must be > 0"); 535 const TypeVect* vmask_type = TypeVect::makemask(bt, vlen); 536 switch (vopc) { 537 case Op_AndV: 538 if (Matcher::match_rule_supported_vector_masked(Op_AndVMask, vlen, bt)) { 539 return new AndVMaskNode(n1, n2, vmask_type); 540 } 541 return new AndVNode(n1, n2, vmask_type); 542 case Op_OrV: 543 if (Matcher::match_rule_supported_vector_masked(Op_OrVMask, vlen, bt)) { 544 return new OrVMaskNode(n1, n2, vmask_type); 545 } 546 return new OrVNode(n1, n2, vmask_type); 547 case Op_XorV: 548 if (Matcher::match_rule_supported_vector_masked(Op_XorVMask, vlen, bt)) { 549 return new XorVMaskNode(n1, n2, vmask_type); 550 } 551 return new XorVNode(n1, n2, vmask_type); 552 default: 553 fatal("Unsupported mask vector creation for '%s'", NodeClassNames[vopc]); 554 return NULL; 555 } 556 } 557 558 // Make a vector node for binary operation 559 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, const TypeVect* vt, bool is_mask, bool is_var_shift) { 560 // This method should not be called for unimplemented vectors. 561 guarantee(vopc > 0, "vopc must be > 0"); 562 563 if (is_mask) { 564 return make_mask_node(vopc, n1, n2, vt->length(), vt->element_basic_type()); 565 } 566 567 switch (vopc) { 568 case Op_AddVB: return new AddVBNode(n1, n2, vt); 569 case Op_AddVS: return new AddVSNode(n1, n2, vt); 570 case Op_AddVI: return new AddVINode(n1, n2, vt); 571 case Op_AddVL: return new AddVLNode(n1, n2, vt); 572 case Op_AddVF: return new AddVFNode(n1, n2, vt); 573 case Op_AddVD: return new AddVDNode(n1, n2, vt); 574 575 case Op_SubVB: return new SubVBNode(n1, n2, vt); 576 case Op_SubVS: return new SubVSNode(n1, n2, vt); 577 case Op_SubVI: return new SubVINode(n1, n2, vt); 578 case Op_SubVL: return new SubVLNode(n1, n2, vt); 579 case Op_SubVF: return new SubVFNode(n1, n2, vt); 580 case Op_SubVD: return new SubVDNode(n1, n2, vt); 581 582 case Op_MulVB: return new MulVBNode(n1, n2, vt); 583 case Op_MulVS: return new MulVSNode(n1, n2, vt); 584 case Op_MulVI: return new MulVINode(n1, n2, vt); 585 case Op_MulVL: return new MulVLNode(n1, n2, vt); 586 case Op_MulVF: return new MulVFNode(n1, n2, vt); 587 case Op_MulVD: return new MulVDNode(n1, n2, vt); 588 589 case Op_DivVF: return new DivVFNode(n1, n2, vt); 590 case Op_DivVD: return new DivVDNode(n1, n2, vt); 591 592 case Op_MinV: return new MinVNode(n1, n2, vt); 593 case Op_MaxV: return new MaxVNode(n1, n2, vt); 594 595 case Op_AbsVF: return new AbsVFNode(n1, vt); 596 case Op_AbsVD: return new AbsVDNode(n1, vt); 597 case Op_AbsVB: return new AbsVBNode(n1, vt); 598 case Op_AbsVS: return new AbsVSNode(n1, vt); 599 case Op_AbsVI: return new AbsVINode(n1, vt); 600 case Op_AbsVL: return new AbsVLNode(n1, vt); 601 602 case Op_NegVI: return new NegVINode(n1, vt); 603 case Op_NegVL: return new NegVLNode(n1, vt); 604 case Op_NegVF: return new NegVFNode(n1, vt); 605 case Op_NegVD: return new NegVDNode(n1, vt); 606 607 case Op_ReverseV: return new ReverseVNode(n1, vt); 608 case Op_ReverseBytesV: return new ReverseBytesVNode(n1, vt); 609 610 case Op_SqrtVF: return new SqrtVFNode(n1, vt); 611 case Op_SqrtVD: return new SqrtVDNode(n1, vt); 612 613 case Op_RoundVF: return new RoundVFNode(n1, vt); 614 case Op_RoundVD: return new RoundVDNode(n1, vt); 615 616 case Op_PopCountVI: return new PopCountVINode(n1, vt); 617 case Op_PopCountVL: return new PopCountVLNode(n1, vt); 618 case Op_RotateLeftV: return new RotateLeftVNode(n1, n2, vt); 619 case Op_RotateRightV: return new RotateRightVNode(n1, n2, vt); 620 621 case Op_LShiftVB: return new LShiftVBNode(n1, n2, vt, is_var_shift); 622 case Op_LShiftVS: return new LShiftVSNode(n1, n2, vt, is_var_shift); 623 case Op_LShiftVI: return new LShiftVINode(n1, n2, vt, is_var_shift); 624 case Op_LShiftVL: return new LShiftVLNode(n1, n2, vt, is_var_shift); 625 626 case Op_RShiftVB: return new RShiftVBNode(n1, n2, vt, is_var_shift); 627 case Op_RShiftVS: return new RShiftVSNode(n1, n2, vt, is_var_shift); 628 case Op_RShiftVI: return new RShiftVINode(n1, n2, vt, is_var_shift); 629 case Op_RShiftVL: return new RShiftVLNode(n1, n2, vt, is_var_shift); 630 631 case Op_URShiftVB: return new URShiftVBNode(n1, n2, vt, is_var_shift); 632 case Op_URShiftVS: return new URShiftVSNode(n1, n2, vt, is_var_shift); 633 case Op_URShiftVI: return new URShiftVINode(n1, n2, vt, is_var_shift); 634 case Op_URShiftVL: return new URShiftVLNode(n1, n2, vt, is_var_shift); 635 636 case Op_AndV: return new AndVNode(n1, n2, vt); 637 case Op_OrV: return new OrVNode (n1, n2, vt); 638 case Op_XorV: return new XorVNode(n1, n2, vt); 639 640 case Op_RoundDoubleModeV: return new RoundDoubleModeVNode(n1, n2, vt); 641 642 case Op_MulAddVS2VI: return new MulAddVS2VINode(n1, n2, vt); 643 644 case Op_ExpandV: return new ExpandVNode(n1, n2, vt); 645 case Op_CompressV: return new CompressVNode(n1, n2, vt); 646 case Op_CompressM: assert(n1 == NULL, ""); return new CompressMNode(n2, vt); 647 case Op_CountLeadingZerosV: return new CountLeadingZerosVNode(n1, vt); 648 case Op_CountTrailingZerosV: return new CountTrailingZerosVNode(n1, vt); 649 default: 650 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); 651 return NULL; 652 } 653 } 654 655 // Return the vector version of a scalar binary operation node. 656 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, uint vlen, BasicType bt, bool is_var_shift) { 657 const TypeVect* vt = TypeVect::make(bt, vlen); 658 int vopc = VectorNode::opcode(opc, bt); 659 // This method should not be called for unimplemented vectors. 660 guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]); 661 return make(vopc, n1, n2, vt, false, is_var_shift); 662 } 663 664 // Make a vector node for ternary operation 665 VectorNode* VectorNode::make(int vopc, Node* n1, Node* n2, Node* n3, const TypeVect* vt) { 666 // This method should not be called for unimplemented vectors. 667 guarantee(vopc > 0, "vopc must be > 0"); 668 switch (vopc) { 669 case Op_FmaVD: return new FmaVDNode(n1, n2, n3, vt); 670 case Op_FmaVF: return new FmaVFNode(n1, n2, n3, vt); 671 default: 672 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); 673 return NULL; 674 } 675 } 676 677 // Return the vector version of a scalar ternary operation node. 678 VectorNode* VectorNode::make(int opc, Node* n1, Node* n2, Node* n3, uint vlen, BasicType bt) { 679 const TypeVect* vt = TypeVect::make(bt, vlen); 680 int vopc = VectorNode::opcode(opc, bt); 681 // This method should not be called for unimplemented vectors. 682 guarantee(vopc > 0, "Vector for '%s' is not implemented", NodeClassNames[opc]); 683 return make(vopc, n1, n2, n3, vt); 684 } 685 686 // Scalar promotion 687 VectorNode* VectorNode::scalar2vector(Node* s, uint vlen, const Type* opd_t, bool is_mask) { 688 BasicType bt = opd_t->array_element_basic_type(); 689 if (is_mask && Matcher::match_rule_supported_vector(Op_MaskAll, vlen, bt)) { 690 const TypeVect* vt = TypeVect::make(opd_t, vlen, true); 691 return new MaskAllNode(s, vt); 692 } 693 694 const TypeVect* vt = opd_t->singleton() ? TypeVect::make(opd_t, vlen) 695 : TypeVect::make(bt, vlen); 696 switch (bt) { 697 case T_BOOLEAN: 698 case T_BYTE: 699 return new ReplicateBNode(s, vt); 700 case T_CHAR: 701 case T_SHORT: 702 return new ReplicateSNode(s, vt); 703 case T_INT: 704 return new ReplicateINode(s, vt); 705 case T_LONG: 706 return new ReplicateLNode(s, vt); 707 case T_FLOAT: 708 return new ReplicateFNode(s, vt); 709 case T_DOUBLE: 710 return new ReplicateDNode(s, vt); 711 default: 712 fatal("Type '%s' is not supported for vectors", type2name(bt)); 713 return NULL; 714 } 715 } 716 717 VectorNode* VectorNode::shift_count(int opc, Node* cnt, uint vlen, BasicType bt) { 718 // Match shift count type with shift vector type. 719 const TypeVect* vt = TypeVect::make(bt, vlen); 720 switch (opc) { 721 case Op_LShiftI: 722 case Op_LShiftL: 723 return new LShiftCntVNode(cnt, vt); 724 case Op_RShiftI: 725 case Op_RShiftL: 726 case Op_URShiftB: 727 case Op_URShiftS: 728 case Op_URShiftI: 729 case Op_URShiftL: 730 return new RShiftCntVNode(cnt, vt); 731 default: 732 fatal("Missed vector creation for '%s'", NodeClassNames[opc]); 733 return NULL; 734 } 735 } 736 737 bool VectorNode::is_vector_rotate(int opc) { 738 switch (opc) { 739 case Op_RotateLeftV: 740 case Op_RotateRightV: 741 return true; 742 default: 743 return false; 744 } 745 } 746 747 bool VectorNode::is_vector_integral_negate(int opc) { 748 return opc == Op_NegVI || opc == Op_NegVL; 749 } 750 751 bool VectorNode::is_vector_shift(int opc) { 752 assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode"); 753 switch (opc) { 754 case Op_LShiftVB: 755 case Op_LShiftVS: 756 case Op_LShiftVI: 757 case Op_LShiftVL: 758 case Op_RShiftVB: 759 case Op_RShiftVS: 760 case Op_RShiftVI: 761 case Op_RShiftVL: 762 case Op_URShiftVB: 763 case Op_URShiftVS: 764 case Op_URShiftVI: 765 case Op_URShiftVL: 766 return true; 767 default: 768 return false; 769 } 770 } 771 772 bool VectorNode::is_vector_shift_count(int opc) { 773 assert(opc > _last_machine_leaf && opc < _last_opcode, "invalid opcode"); 774 switch (opc) { 775 case Op_RShiftCntV: 776 case Op_LShiftCntV: 777 return true; 778 default: 779 return false; 780 } 781 } 782 783 static bool is_con_M1(Node* n) { 784 if (n->is_Con()) { 785 const Type* t = n->bottom_type(); 786 if (t->isa_int() && t->is_int()->get_con() == -1) { 787 return true; 788 } 789 if (t->isa_long() && t->is_long()->get_con() == -1) { 790 return true; 791 } 792 } 793 return false; 794 } 795 796 bool VectorNode::is_all_ones_vector(Node* n) { 797 switch (n->Opcode()) { 798 case Op_ReplicateB: 799 case Op_ReplicateS: 800 case Op_ReplicateI: 801 case Op_ReplicateL: 802 return is_con_M1(n->in(1)); 803 default: 804 return false; 805 } 806 } 807 808 bool VectorNode::is_vector_bitwise_not_pattern(Node* n) { 809 if (n->Opcode() == Op_XorV) { 810 return is_all_ones_vector(n->in(1)) || 811 is_all_ones_vector(n->in(2)); 812 } 813 return false; 814 } 815 816 // Return initial Pack node. Additional operands added with add_opd() calls. 817 PackNode* PackNode::make(Node* s, uint vlen, BasicType bt) { 818 const TypeVect* vt = TypeVect::make(bt, vlen); 819 switch (bt) { 820 case T_BOOLEAN: 821 case T_BYTE: 822 return new PackBNode(s, vt); 823 case T_CHAR: 824 case T_SHORT: 825 return new PackSNode(s, vt); 826 case T_INT: 827 return new PackINode(s, vt); 828 case T_LONG: 829 return new PackLNode(s, vt); 830 case T_FLOAT: 831 return new PackFNode(s, vt); 832 case T_DOUBLE: 833 return new PackDNode(s, vt); 834 default: 835 fatal("Type '%s' is not supported for vectors", type2name(bt)); 836 return NULL; 837 } 838 } 839 840 // Create a binary tree form for Packs. [lo, hi) (half-open) range 841 PackNode* PackNode::binary_tree_pack(int lo, int hi) { 842 int ct = hi - lo; 843 assert(is_power_of_2(ct), "power of 2"); 844 if (ct == 2) { 845 PackNode* pk = PackNode::make(in(lo), 2, vect_type()->element_basic_type()); 846 pk->add_opd(in(lo+1)); 847 return pk; 848 } else { 849 int mid = lo + ct/2; 850 PackNode* n1 = binary_tree_pack(lo, mid); 851 PackNode* n2 = binary_tree_pack(mid, hi ); 852 853 BasicType bt = n1->vect_type()->element_basic_type(); 854 assert(bt == n2->vect_type()->element_basic_type(), "should be the same"); 855 switch (bt) { 856 case T_BOOLEAN: 857 case T_BYTE: 858 return new PackSNode(n1, n2, TypeVect::make(T_SHORT, 2)); 859 case T_CHAR: 860 case T_SHORT: 861 return new PackINode(n1, n2, TypeVect::make(T_INT, 2)); 862 case T_INT: 863 return new PackLNode(n1, n2, TypeVect::make(T_LONG, 2)); 864 case T_LONG: 865 return new Pack2LNode(n1, n2, TypeVect::make(T_LONG, 2)); 866 case T_FLOAT: 867 return new PackDNode(n1, n2, TypeVect::make(T_DOUBLE, 2)); 868 case T_DOUBLE: 869 return new Pack2DNode(n1, n2, TypeVect::make(T_DOUBLE, 2)); 870 default: 871 fatal("Type '%s' is not supported for vectors", type2name(bt)); 872 return NULL; 873 } 874 } 875 } 876 877 // Return the vector version of a scalar load node. 878 LoadVectorNode* LoadVectorNode::make(int opc, Node* ctl, Node* mem, 879 Node* adr, const TypePtr* atyp, 880 uint vlen, BasicType bt, 881 ControlDependency control_dependency) { 882 const TypeVect* vt = TypeVect::make(bt, vlen); 883 return new LoadVectorNode(ctl, mem, adr, atyp, vt, control_dependency); 884 } 885 886 // Return the vector version of a scalar store node. 887 StoreVectorNode* StoreVectorNode::make(int opc, Node* ctl, Node* mem, 888 Node* adr, const TypePtr* atyp, Node* val, 889 uint vlen) { 890 return new StoreVectorNode(ctl, mem, adr, atyp, val); 891 } 892 893 Node* LoadVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) { 894 if (!in(3)->is_top() && in(3)->Opcode() == Op_VectorMaskGen) { 895 Node* mask_len = in(3)->in(1); 896 const TypeLong* ty = phase->type(mask_len)->isa_long(); 897 if (ty && ty->is_con()) { 898 BasicType mask_bt = Matcher::vector_element_basic_type(in(3)); 899 int load_sz = type2aelembytes(mask_bt) * ty->get_con(); 900 assert(load_sz <= MaxVectorSize, "Unexpected load size"); 901 if (load_sz == MaxVectorSize) { 902 Node* ctr = in(MemNode::Control); 903 Node* mem = in(MemNode::Memory); 904 Node* adr = in(MemNode::Address); 905 return phase->transform(new LoadVectorNode(ctr, mem, adr, adr_type(), vect_type())); 906 } 907 } 908 } 909 return NULL; 910 } 911 912 Node* StoreVectorMaskedNode::Ideal(PhaseGVN* phase, bool can_reshape) { 913 if (!in(4)->is_top() && in(4)->Opcode() == Op_VectorMaskGen) { 914 Node* mask_len = in(4)->in(1); 915 const TypeLong* ty = phase->type(mask_len)->isa_long(); 916 if (ty && ty->is_con()) { 917 BasicType mask_bt = Matcher::vector_element_basic_type(in(4)); 918 int load_sz = type2aelembytes(mask_bt) * ty->get_con(); 919 assert(load_sz <= MaxVectorSize, "Unexpected store size"); 920 if (load_sz == MaxVectorSize) { 921 Node* ctr = in(MemNode::Control); 922 Node* mem = in(MemNode::Memory); 923 Node* adr = in(MemNode::Address); 924 Node* val = in(MemNode::ValueIn); 925 return phase->transform(new StoreVectorNode(ctr, mem, adr, adr_type(), val)); 926 } 927 } 928 } 929 return NULL; 930 } 931 932 int ExtractNode::opcode(BasicType bt) { 933 switch (bt) { 934 case T_BOOLEAN: return Op_ExtractUB; 935 case T_BYTE: return Op_ExtractB; 936 case T_CHAR: return Op_ExtractC; 937 case T_SHORT: return Op_ExtractS; 938 case T_INT: return Op_ExtractI; 939 case T_LONG: return Op_ExtractL; 940 case T_FLOAT: return Op_ExtractF; 941 case T_DOUBLE: return Op_ExtractD; 942 default: 943 assert(false, "wrong type: %s", type2name(bt)); 944 return 0; 945 } 946 } 947 948 // Extract a scalar element of vector. 949 Node* ExtractNode::make(Node* v, uint position, BasicType bt) { 950 assert((int)position < Matcher::max_vector_size(bt), "pos in range"); 951 ConINode* pos = ConINode::make((int)position); 952 switch (bt) { 953 case T_BOOLEAN: return new ExtractUBNode(v, pos); 954 case T_BYTE: return new ExtractBNode(v, pos); 955 case T_CHAR: return new ExtractCNode(v, pos); 956 case T_SHORT: return new ExtractSNode(v, pos); 957 case T_INT: return new ExtractINode(v, pos); 958 case T_LONG: return new ExtractLNode(v, pos); 959 case T_FLOAT: return new ExtractFNode(v, pos); 960 case T_DOUBLE: return new ExtractDNode(v, pos); 961 default: 962 assert(false, "wrong type: %s", type2name(bt)); 963 return NULL; 964 } 965 } 966 967 int ReductionNode::opcode(int opc, BasicType bt) { 968 int vopc = opc; 969 switch (opc) { 970 case Op_AddI: 971 switch (bt) { 972 case T_BOOLEAN: 973 case T_CHAR: return 0; 974 case T_BYTE: 975 case T_SHORT: 976 case T_INT: 977 vopc = Op_AddReductionVI; 978 break; 979 default: ShouldNotReachHere(); return 0; 980 } 981 break; 982 case Op_AddL: 983 assert(bt == T_LONG, "must be"); 984 vopc = Op_AddReductionVL; 985 break; 986 case Op_AddF: 987 assert(bt == T_FLOAT, "must be"); 988 vopc = Op_AddReductionVF; 989 break; 990 case Op_AddD: 991 assert(bt == T_DOUBLE, "must be"); 992 vopc = Op_AddReductionVD; 993 break; 994 case Op_MulI: 995 switch (bt) { 996 case T_BOOLEAN: 997 case T_CHAR: return 0; 998 case T_BYTE: 999 case T_SHORT: 1000 case T_INT: 1001 vopc = Op_MulReductionVI; 1002 break; 1003 default: ShouldNotReachHere(); return 0; 1004 } 1005 break; 1006 case Op_MulL: 1007 assert(bt == T_LONG, "must be"); 1008 vopc = Op_MulReductionVL; 1009 break; 1010 case Op_MulF: 1011 assert(bt == T_FLOAT, "must be"); 1012 vopc = Op_MulReductionVF; 1013 break; 1014 case Op_MulD: 1015 assert(bt == T_DOUBLE, "must be"); 1016 vopc = Op_MulReductionVD; 1017 break; 1018 case Op_MinI: 1019 switch (bt) { 1020 case T_BOOLEAN: 1021 case T_CHAR: return 0; 1022 case T_BYTE: 1023 case T_SHORT: 1024 case T_INT: 1025 vopc = Op_MinReductionV; 1026 break; 1027 default: ShouldNotReachHere(); return 0; 1028 } 1029 break; 1030 case Op_MinL: 1031 assert(bt == T_LONG, "must be"); 1032 vopc = Op_MinReductionV; 1033 break; 1034 case Op_MinF: 1035 assert(bt == T_FLOAT, "must be"); 1036 vopc = Op_MinReductionV; 1037 break; 1038 case Op_MinD: 1039 assert(bt == T_DOUBLE, "must be"); 1040 vopc = Op_MinReductionV; 1041 break; 1042 case Op_MaxI: 1043 switch (bt) { 1044 case T_BOOLEAN: 1045 case T_CHAR: return 0; 1046 case T_BYTE: 1047 case T_SHORT: 1048 case T_INT: 1049 vopc = Op_MaxReductionV; 1050 break; 1051 default: ShouldNotReachHere(); return 0; 1052 } 1053 break; 1054 case Op_MaxL: 1055 assert(bt == T_LONG, "must be"); 1056 vopc = Op_MaxReductionV; 1057 break; 1058 case Op_MaxF: 1059 assert(bt == T_FLOAT, "must be"); 1060 vopc = Op_MaxReductionV; 1061 break; 1062 case Op_MaxD: 1063 assert(bt == T_DOUBLE, "must be"); 1064 vopc = Op_MaxReductionV; 1065 break; 1066 case Op_AndI: 1067 switch (bt) { 1068 case T_BOOLEAN: 1069 case T_CHAR: return 0; 1070 case T_BYTE: 1071 case T_SHORT: 1072 case T_INT: 1073 vopc = Op_AndReductionV; 1074 break; 1075 default: ShouldNotReachHere(); return 0; 1076 } 1077 break; 1078 case Op_AndL: 1079 assert(bt == T_LONG, "must be"); 1080 vopc = Op_AndReductionV; 1081 break; 1082 case Op_OrI: 1083 switch(bt) { 1084 case T_BOOLEAN: 1085 case T_CHAR: return 0; 1086 case T_BYTE: 1087 case T_SHORT: 1088 case T_INT: 1089 vopc = Op_OrReductionV; 1090 break; 1091 default: ShouldNotReachHere(); return 0; 1092 } 1093 break; 1094 case Op_OrL: 1095 assert(bt == T_LONG, "must be"); 1096 vopc = Op_OrReductionV; 1097 break; 1098 case Op_XorI: 1099 switch(bt) { 1100 case T_BOOLEAN: 1101 case T_CHAR: return 0; 1102 case T_BYTE: 1103 case T_SHORT: 1104 case T_INT: 1105 vopc = Op_XorReductionV; 1106 break; 1107 default: ShouldNotReachHere(); return 0; 1108 } 1109 break; 1110 case Op_XorL: 1111 assert(bt == T_LONG, "must be"); 1112 vopc = Op_XorReductionV; 1113 break; 1114 default: 1115 break; 1116 } 1117 return vopc; 1118 } 1119 1120 // Return the appropriate reduction node. 1121 ReductionNode* ReductionNode::make(int opc, Node *ctrl, Node* n1, Node* n2, BasicType bt) { 1122 1123 int vopc = opcode(opc, bt); 1124 1125 // This method should not be called for unimplemented vectors. 1126 guarantee(vopc != opc, "Vector for '%s' is not implemented", NodeClassNames[opc]); 1127 1128 switch (vopc) { 1129 case Op_AddReductionVI: return new AddReductionVINode(ctrl, n1, n2); 1130 case Op_AddReductionVL: return new AddReductionVLNode(ctrl, n1, n2); 1131 case Op_AddReductionVF: return new AddReductionVFNode(ctrl, n1, n2); 1132 case Op_AddReductionVD: return new AddReductionVDNode(ctrl, n1, n2); 1133 case Op_MulReductionVI: return new MulReductionVINode(ctrl, n1, n2); 1134 case Op_MulReductionVL: return new MulReductionVLNode(ctrl, n1, n2); 1135 case Op_MulReductionVF: return new MulReductionVFNode(ctrl, n1, n2); 1136 case Op_MulReductionVD: return new MulReductionVDNode(ctrl, n1, n2); 1137 case Op_MinReductionV: return new MinReductionVNode(ctrl, n1, n2); 1138 case Op_MaxReductionV: return new MaxReductionVNode(ctrl, n1, n2); 1139 case Op_AndReductionV: return new AndReductionVNode(ctrl, n1, n2); 1140 case Op_OrReductionV: return new OrReductionVNode(ctrl, n1, n2); 1141 case Op_XorReductionV: return new XorReductionVNode(ctrl, n1, n2); 1142 default: 1143 assert(false, "unknown node: %s", NodeClassNames[vopc]); 1144 return NULL; 1145 } 1146 } 1147 1148 Node* VectorLoadMaskNode::Identity(PhaseGVN* phase) { 1149 BasicType out_bt = type()->is_vect()->element_basic_type(); 1150 if (!Matcher::has_predicated_vectors() && out_bt == T_BOOLEAN) { 1151 return in(1); // redundant conversion 1152 } 1153 1154 return this; 1155 } 1156 1157 Node* VectorStoreMaskNode::Identity(PhaseGVN* phase) { 1158 // Identity transformation on boolean vectors. 1159 // VectorStoreMask (VectorLoadMask bv) elem_size ==> bv 1160 // vector[n]{bool} => vector[n]{t} => vector[n]{bool} 1161 if (in(1)->Opcode() == Op_VectorLoadMask) { 1162 return in(1)->in(1); 1163 } 1164 return this; 1165 } 1166 1167 VectorStoreMaskNode* VectorStoreMaskNode::make(PhaseGVN& gvn, Node* in, BasicType in_type, uint num_elem) { 1168 assert(in->bottom_type()->isa_vect(), "sanity"); 1169 const TypeVect* vt = TypeVect::make(T_BOOLEAN, num_elem); 1170 int elem_size = type2aelembytes(in_type); 1171 return new VectorStoreMaskNode(in, gvn.intcon(elem_size), vt); 1172 } 1173 1174 VectorCastNode* VectorCastNode::make(int vopc, Node* n1, BasicType bt, uint vlen) { 1175 const TypeVect* vt = TypeVect::make(bt, vlen); 1176 switch (vopc) { 1177 case Op_VectorCastB2X: return new VectorCastB2XNode(n1, vt); 1178 case Op_VectorCastS2X: return new VectorCastS2XNode(n1, vt); 1179 case Op_VectorCastI2X: return new VectorCastI2XNode(n1, vt); 1180 case Op_VectorCastL2X: return new VectorCastL2XNode(n1, vt); 1181 case Op_VectorCastF2X: return new VectorCastF2XNode(n1, vt); 1182 case Op_VectorCastD2X: return new VectorCastD2XNode(n1, vt); 1183 case Op_VectorUCastB2X: return new VectorUCastB2XNode(n1, vt); 1184 case Op_VectorUCastS2X: return new VectorUCastS2XNode(n1, vt); 1185 case Op_VectorUCastI2X: return new VectorUCastI2XNode(n1, vt); 1186 default: 1187 assert(false, "unknown node: %s", NodeClassNames[vopc]); 1188 return NULL; 1189 } 1190 } 1191 1192 int VectorCastNode::opcode(BasicType bt, bool is_signed) { 1193 assert((is_integral_type(bt) && bt != T_LONG) || is_signed, ""); 1194 switch (bt) { 1195 case T_BYTE: return is_signed ? Op_VectorCastB2X : Op_VectorUCastB2X; 1196 case T_SHORT: return is_signed ? Op_VectorCastS2X : Op_VectorUCastS2X; 1197 case T_INT: return is_signed ? Op_VectorCastI2X : Op_VectorUCastI2X; 1198 case T_LONG: return Op_VectorCastL2X; 1199 case T_FLOAT: return Op_VectorCastF2X; 1200 case T_DOUBLE: return Op_VectorCastD2X; 1201 default: 1202 assert(false, "unknown type: %s", type2name(bt)); 1203 return 0; 1204 } 1205 } 1206 1207 Node* VectorCastNode::Identity(PhaseGVN* phase) { 1208 if (!in(1)->is_top()) { 1209 BasicType in_bt = in(1)->bottom_type()->is_vect()->element_basic_type(); 1210 BasicType out_bt = vect_type()->element_basic_type(); 1211 if (in_bt == out_bt) { 1212 return in(1); // redundant cast 1213 } 1214 } 1215 return this; 1216 } 1217 1218 Node* ReductionNode::make_reduction_input(PhaseGVN& gvn, int opc, BasicType bt) { 1219 int vopc = opcode(opc, bt); 1220 guarantee(vopc != opc, "Vector reduction for '%s' is not implemented", NodeClassNames[opc]); 1221 1222 switch (vopc) { 1223 case Op_AndReductionV: 1224 switch (bt) { 1225 case T_BYTE: 1226 case T_SHORT: 1227 case T_INT: 1228 return gvn.makecon(TypeInt::MINUS_1); 1229 case T_LONG: 1230 return gvn.makecon(TypeLong::MINUS_1); 1231 default: 1232 fatal("Missed vector creation for '%s' as the basic type is not correct.", NodeClassNames[vopc]); 1233 return NULL; 1234 } 1235 break; 1236 case Op_AddReductionVI: // fallthrough 1237 case Op_AddReductionVL: // fallthrough 1238 case Op_AddReductionVF: // fallthrough 1239 case Op_AddReductionVD: 1240 case Op_OrReductionV: 1241 case Op_XorReductionV: 1242 return gvn.zerocon(bt); 1243 case Op_MulReductionVI: 1244 return gvn.makecon(TypeInt::ONE); 1245 case Op_MulReductionVL: 1246 return gvn.makecon(TypeLong::ONE); 1247 case Op_MulReductionVF: 1248 return gvn.makecon(TypeF::ONE); 1249 case Op_MulReductionVD: 1250 return gvn.makecon(TypeD::ONE); 1251 case Op_MinReductionV: 1252 switch (bt) { 1253 case T_BYTE: 1254 return gvn.makecon(TypeInt::make(max_jbyte)); 1255 case T_SHORT: 1256 return gvn.makecon(TypeInt::make(max_jshort)); 1257 case T_INT: 1258 return gvn.makecon(TypeInt::MAX); 1259 case T_LONG: 1260 return gvn.makecon(TypeLong::MAX); 1261 case T_FLOAT: 1262 return gvn.makecon(TypeF::POS_INF); 1263 case T_DOUBLE: 1264 return gvn.makecon(TypeD::POS_INF); 1265 default: Unimplemented(); return NULL; 1266 } 1267 break; 1268 case Op_MaxReductionV: 1269 switch (bt) { 1270 case T_BYTE: 1271 return gvn.makecon(TypeInt::make(min_jbyte)); 1272 case T_SHORT: 1273 return gvn.makecon(TypeInt::make(min_jshort)); 1274 case T_INT: 1275 return gvn.makecon(TypeInt::MIN); 1276 case T_LONG: 1277 return gvn.makecon(TypeLong::MIN); 1278 case T_FLOAT: 1279 return gvn.makecon(TypeF::NEG_INF); 1280 case T_DOUBLE: 1281 return gvn.makecon(TypeD::NEG_INF); 1282 default: Unimplemented(); return NULL; 1283 } 1284 break; 1285 default: 1286 fatal("Missed vector creation for '%s'", NodeClassNames[vopc]); 1287 return NULL; 1288 } 1289 } 1290 1291 bool ReductionNode::implemented(int opc, uint vlen, BasicType bt) { 1292 if (is_java_primitive(bt) && 1293 (vlen > 1) && is_power_of_2(vlen) && 1294 Matcher::vector_size_supported(bt, vlen)) { 1295 int vopc = ReductionNode::opcode(opc, bt); 1296 return vopc != opc && Matcher::match_rule_supported_vector(vopc, vlen, bt); 1297 } 1298 return false; 1299 } 1300 1301 MacroLogicVNode* MacroLogicVNode::make(PhaseGVN& gvn, Node* in1, Node* in2, Node* in3, 1302 Node* mask, uint truth_table, const TypeVect* vt) { 1303 assert(truth_table <= 0xFF, "invalid"); 1304 assert(in1->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch"); 1305 assert(in2->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch"); 1306 assert(in3->bottom_type()->is_vect()->length_in_bytes() == vt->length_in_bytes(), "mismatch"); 1307 assert(!mask || mask->bottom_type()->isa_vectmask(), "predicated register type expected"); 1308 Node* fn = gvn.intcon(truth_table); 1309 return new MacroLogicVNode(in1, in2, in3, fn, mask, vt); 1310 } 1311 1312 Node* VectorNode::degenerate_vector_rotate(Node* src, Node* cnt, bool is_rotate_left, 1313 int vlen, BasicType bt, PhaseGVN* phase) { 1314 assert(is_integral_type(bt), "sanity"); 1315 const TypeVect* vt = TypeVect::make(bt, vlen); 1316 1317 int shift_mask = (type2aelembytes(bt) * 8) - 1; 1318 int shiftLOpc = (bt == T_LONG) ? Op_LShiftL : Op_LShiftI; 1319 auto urshiftopc = [=]() { 1320 switch(bt) { 1321 case T_INT: return Op_URShiftI; 1322 case T_LONG: return Op_URShiftL; 1323 case T_BYTE: return Op_URShiftB; 1324 case T_SHORT: return Op_URShiftS; 1325 default: return (Opcodes)0; 1326 } 1327 }; 1328 int shiftROpc = urshiftopc(); 1329 1330 // Compute shift values for right rotation and 1331 // later swap them in case of left rotation. 1332 Node* shiftRCnt = NULL; 1333 Node* shiftLCnt = NULL; 1334 const TypeInt* cnt_type = cnt->bottom_type()->isa_int(); 1335 bool is_binary_vector_op = false; 1336 if (cnt_type && cnt_type->is_con()) { 1337 // Constant shift. 1338 int shift = cnt_type->get_con() & shift_mask; 1339 shiftRCnt = phase->intcon(shift); 1340 shiftLCnt = phase->intcon(shift_mask + 1 - shift); 1341 } else if (VectorNode::is_invariant_vector(cnt)) { 1342 // Scalar variable shift, handle replicates generated by auto vectorizer. 1343 cnt = cnt->in(1); 1344 if (bt == T_LONG) { 1345 // Shift count vector for Rotate vector has long elements too. 1346 if (cnt->Opcode() == Op_ConvI2L) { 1347 cnt = cnt->in(1); 1348 } else { 1349 assert(cnt->bottom_type()->isa_long() && 1350 cnt->bottom_type()->is_long()->is_con(), "Long constant expected"); 1351 cnt = phase->transform(new ConvL2INode(cnt)); 1352 } 1353 } 1354 shiftRCnt = phase->transform(new AndINode(cnt, phase->intcon(shift_mask))); 1355 shiftLCnt = phase->transform(new SubINode(phase->intcon(shift_mask + 1), shiftRCnt)); 1356 } else { 1357 // Variable vector rotate count. 1358 assert(Matcher::supports_vector_variable_shifts(), ""); 1359 1360 int subVopc = 0; 1361 int addVopc = 0; 1362 Node* shift_mask_node = NULL; 1363 Node* const_one_node = NULL; 1364 1365 assert(cnt->bottom_type()->isa_vect(), "Unexpected shift"); 1366 const Type* elem_ty = Type::get_const_basic_type(bt); 1367 1368 if (bt == T_LONG) { 1369 shift_mask_node = phase->longcon(shift_mask); 1370 const_one_node = phase->longcon(1L); 1371 subVopc = VectorNode::opcode(Op_SubL, bt); 1372 addVopc = VectorNode::opcode(Op_AddL, bt); 1373 } else { 1374 shift_mask_node = phase->intcon(shift_mask); 1375 const_one_node = phase->intcon(1); 1376 subVopc = VectorNode::opcode(Op_SubI, bt); 1377 addVopc = VectorNode::opcode(Op_AddI, bt); 1378 } 1379 Node* vector_mask = phase->transform(VectorNode::scalar2vector(shift_mask_node, vlen, elem_ty)); 1380 Node* vector_one = phase->transform(VectorNode::scalar2vector(const_one_node, vlen, elem_ty)); 1381 1382 shiftRCnt = cnt; 1383 shiftRCnt = phase->transform(VectorNode::make(Op_AndV, shiftRCnt, vector_mask, vt)); 1384 vector_mask = phase->transform(VectorNode::make(addVopc, vector_one, vector_mask, vt)); 1385 shiftLCnt = phase->transform(VectorNode::make(subVopc, vector_mask, shiftRCnt, vt)); 1386 is_binary_vector_op = true; 1387 } 1388 1389 // Swap the computed left and right shift counts. 1390 if (is_rotate_left) { 1391 swap(shiftRCnt,shiftLCnt); 1392 } 1393 1394 if (!is_binary_vector_op) { 1395 shiftLCnt = phase->transform(new LShiftCntVNode(shiftLCnt, vt)); 1396 shiftRCnt = phase->transform(new RShiftCntVNode(shiftRCnt, vt)); 1397 } 1398 1399 return new OrVNode(phase->transform(VectorNode::make(shiftLOpc, src, shiftLCnt, vlen, bt, is_binary_vector_op)), 1400 phase->transform(VectorNode::make(shiftROpc, src, shiftRCnt, vlen, bt, is_binary_vector_op)), 1401 vt); 1402 } 1403 1404 Node* RotateLeftVNode::Ideal(PhaseGVN* phase, bool can_reshape) { 1405 int vlen = length(); 1406 BasicType bt = vect_type()->element_basic_type(); 1407 if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) || 1408 !Matcher::match_rule_supported_vector(Op_RotateLeftV, vlen, bt)) { 1409 return VectorNode::degenerate_vector_rotate(in(1), in(2), true, vlen, bt, phase); 1410 } 1411 return NULL; 1412 } 1413 1414 Node* RotateRightVNode::Ideal(PhaseGVN* phase, bool can_reshape) { 1415 int vlen = length(); 1416 BasicType bt = vect_type()->element_basic_type(); 1417 if ((!in(2)->is_Con() && !Matcher::supports_vector_variable_rotates()) || 1418 !Matcher::match_rule_supported_vector(Op_RotateRightV, vlen, bt)) { 1419 return VectorNode::degenerate_vector_rotate(in(1), in(2), false, vlen, bt, phase); 1420 } 1421 return NULL; 1422 } 1423 1424 #ifndef PRODUCT 1425 void VectorMaskCmpNode::dump_spec(outputStream *st) const { 1426 st->print(" %d #", _predicate); _type->dump_on(st); 1427 } 1428 #endif // PRODUCT 1429 1430 Node* VectorReinterpretNode::Identity(PhaseGVN *phase) { 1431 Node* n = in(1); 1432 if (n->Opcode() == Op_VectorReinterpret) { 1433 // "VectorReinterpret (VectorReinterpret node) ==> node" if: 1434 // 1) Types of 'node' and 'this' are identical 1435 // 2) Truncations are not introduced by the first VectorReinterpret 1436 if (Type::cmp(bottom_type(), n->in(1)->bottom_type()) == 0 && 1437 length_in_bytes() <= n->bottom_type()->is_vect()->length_in_bytes()) { 1438 return n->in(1); 1439 } 1440 } 1441 return this; 1442 } 1443 1444 Node* VectorInsertNode::make(Node* vec, Node* new_val, int position) { 1445 assert(position < (int)vec->bottom_type()->is_vect()->length(), "pos in range"); 1446 ConINode* pos = ConINode::make(position); 1447 return new VectorInsertNode(vec, new_val, pos, vec->bottom_type()->is_vect()); 1448 } 1449 1450 Node* VectorUnboxNode::Ideal(PhaseGVN* phase, bool can_reshape) { 1451 Node* n = obj()->uncast(); 1452 if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) { 1453 if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) { 1454 // Handled by VectorUnboxNode::Identity() 1455 } else { 1456 VectorBoxNode* vbox = static_cast<VectorBoxNode*>(n); 1457 ciKlass* vbox_klass = vbox->box_type()->klass(); 1458 const TypeVect* in_vt = vbox->vec_type(); 1459 const TypeVect* out_vt = type()->is_vect(); 1460 1461 if (in_vt->length() == out_vt->length()) { 1462 Node* value = vbox->in(VectorBoxNode::Value); 1463 1464 bool is_vector_mask = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorMask_klass()); 1465 bool is_vector_shuffle = vbox_klass->is_subclass_of(ciEnv::current()->vector_VectorShuffle_klass()); 1466 if (is_vector_mask) { 1467 const TypeVect* vmask_type = TypeVect::makemask(out_vt->element_basic_type(), out_vt->length()); 1468 if (in_vt->length_in_bytes() == out_vt->length_in_bytes() && 1469 Matcher::match_rule_supported_vector(Op_VectorMaskCast, out_vt->length(), out_vt->element_basic_type())) { 1470 // Apply "VectorUnbox (VectorBox vmask) ==> VectorMaskCast (vmask)" 1471 // directly. This could avoid the transformation ordering issue from 1472 // "VectorStoreMask (VectorLoadMask vmask) => vmask". 1473 return new VectorMaskCastNode(value, vmask_type); 1474 } 1475 // VectorUnbox (VectorBox vmask) ==> VectorLoadMask (VectorStoreMask vmask) 1476 value = phase->transform(VectorStoreMaskNode::make(*phase, value, in_vt->element_basic_type(), in_vt->length())); 1477 return new VectorLoadMaskNode(value, vmask_type); 1478 } else if (is_vector_shuffle) { 1479 if (!is_shuffle_to_vector()) { 1480 // VectorUnbox (VectorBox vshuffle) ==> VectorLoadShuffle vshuffle 1481 return new VectorLoadShuffleNode(value, out_vt); 1482 } 1483 } else { 1484 // Vector type mismatch is only supported for masks and shuffles, but sometimes it happens in pathological cases. 1485 } 1486 } else { 1487 // Vector length mismatch. 1488 // Sometimes happen in pathological cases (e.g., when unboxing happens in effectively dead code). 1489 } 1490 } 1491 } 1492 return NULL; 1493 } 1494 1495 Node* VectorUnboxNode::Identity(PhaseGVN* phase) { 1496 Node* n = obj()->uncast(); 1497 if (EnableVectorReboxing && n->Opcode() == Op_VectorBox) { 1498 if (Type::cmp(bottom_type(), n->in(VectorBoxNode::Value)->bottom_type()) == 0) { 1499 return n->in(VectorBoxNode::Value); // VectorUnbox (VectorBox v) ==> v 1500 } else { 1501 // Handled by VectorUnboxNode::Ideal(). 1502 } 1503 } 1504 return this; 1505 } 1506 1507 const TypeFunc* VectorBoxNode::vec_box_type(const TypeInstPtr* box_type) { 1508 const Type** fields = TypeTuple::fields(0); 1509 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields); 1510 1511 fields = TypeTuple::fields(1); 1512 fields[TypeFunc::Parms+0] = box_type; 1513 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields); 1514 1515 return TypeFunc::make(domain, range); 1516 } 1517 1518 Node* ShiftVNode::Identity(PhaseGVN* phase) { 1519 Node* in2 = in(2); 1520 // Shift by ZERO does nothing 1521 if (is_vshift_cnt(in2) && phase->find_int_type(in2->in(1)) == TypeInt::ZERO) { 1522 return in(1); 1523 } 1524 return this; 1525 } 1526 1527 Node* VectorMaskGenNode::make(Node* length, BasicType mask_bt) { 1528 int max_vector = Matcher::max_vector_size(mask_bt); 1529 const TypeVectMask* t_vmask = TypeVectMask::make(mask_bt, max_vector); 1530 return new VectorMaskGenNode(length, t_vmask); 1531 } 1532 1533 Node* VectorMaskOpNode::make(Node* mask, const Type* ty, int mopc) { 1534 switch(mopc) { 1535 case Op_VectorMaskTrueCount: 1536 return new VectorMaskTrueCountNode(mask, ty); 1537 case Op_VectorMaskLastTrue: 1538 return new VectorMaskLastTrueNode(mask, ty); 1539 case Op_VectorMaskFirstTrue: 1540 return new VectorMaskFirstTrueNode(mask, ty); 1541 case Op_VectorMaskToLong: 1542 return new VectorMaskToLongNode(mask, ty); 1543 default: 1544 assert(false, "Unhandled operation"); 1545 } 1546 return NULL; 1547 } 1548 1549 Node* VectorMaskToLongNode::Identity(PhaseGVN* phase) { 1550 if (in(1)->Opcode() == Op_VectorLongToMask) { 1551 return in(1)->in(1); 1552 } 1553 return this; 1554 } 1555 1556 1557 Node* VectorMaskCastNode::makeCastNode(PhaseGVN* phase, Node* src, const TypeVect* dst_type) { 1558 const TypeVect* src_type = src->bottom_type()->is_vect(); 1559 assert(src_type->length() == dst_type->length(), ""); 1560 1561 int num_elem = src_type->length(); 1562 BasicType elem_bt_from = src_type->element_basic_type(); 1563 BasicType elem_bt_to = dst_type->element_basic_type(); 1564 1565 if (dst_type->isa_vectmask() == NULL && src_type->isa_vectmask() == NULL && 1566 type2aelembytes(elem_bt_from) != type2aelembytes(elem_bt_to)) { 1567 1568 Node* op = src; 1569 BasicType new_elem_bt_from = elem_bt_from; 1570 BasicType new_elem_bt_to = elem_bt_to; 1571 if (is_floating_point_type(elem_bt_from)) { 1572 new_elem_bt_from = elem_bt_from == T_FLOAT ? T_INT : T_LONG; 1573 } 1574 if (is_floating_point_type(elem_bt_to)) { 1575 new_elem_bt_to = elem_bt_to == T_FLOAT ? T_INT : T_LONG; 1576 } 1577 1578 // Special handling for casting operation involving floating point types. 1579 // Case A) F -> X := F -> VectorMaskCast (F->I/L [NOP]) -> VectorCast[I/L]2X 1580 // Case B) X -> F := X -> VectorCastX2[I/L] -> VectorMaskCast ([I/L]->F [NOP]) 1581 // Case C) F -> F := VectorMaskCast (F->I/L [NOP]) -> VectorCast[I/L]2[L/I] -> VectotMaskCast (L/I->F [NOP]) 1582 1583 if (new_elem_bt_from != elem_bt_from) { 1584 const TypeVect* new_src_type = TypeVect::makemask(new_elem_bt_from, num_elem); 1585 op = phase->transform(new VectorMaskCastNode(op, new_src_type)); 1586 } 1587 1588 op = phase->transform(VectorCastNode::make(VectorCastNode::opcode(new_elem_bt_from), op, new_elem_bt_to, num_elem)); 1589 1590 if (new_elem_bt_to != elem_bt_to) { 1591 op = phase->transform(new VectorMaskCastNode(op, dst_type)); 1592 } 1593 return op; 1594 } else { 1595 return new VectorMaskCastNode(src, dst_type); 1596 } 1597 } 1598 1599 Node* VectorLongToMaskNode::Ideal(PhaseGVN* phase, bool can_reshape) { 1600 const TypeVect* dst_type = bottom_type()->is_vect(); 1601 if (in(1)->Opcode() == Op_AndL && 1602 in(1)->in(1)->Opcode() == Op_VectorMaskToLong && 1603 in(1)->in(2)->bottom_type()->isa_long() && 1604 in(1)->in(2)->bottom_type()->is_long()->is_con() && 1605 in(1)->in(2)->bottom_type()->is_long()->get_con() == ((1L << dst_type->length()) - 1)) { 1606 // Different src/dst mask length represents a re-interpretation operation, 1607 // we can however generate a mask casting operation if length matches. 1608 Node* src = in(1)->in(1)->in(1); 1609 if (dst_type->isa_vectmask() == NULL) { 1610 if (src->Opcode() != Op_VectorStoreMask) { 1611 return NULL; 1612 } 1613 src = src->in(1); 1614 } 1615 const TypeVect* src_type = src->bottom_type()->is_vect(); 1616 if (src_type->length() == dst_type->length() && 1617 ((src_type->isa_vectmask() == NULL && dst_type->isa_vectmask() == NULL) || 1618 (src_type->isa_vectmask() && dst_type->isa_vectmask()))) { 1619 return VectorMaskCastNode::makeCastNode(phase, src, dst_type); 1620 } 1621 } 1622 return NULL; 1623 } 1624 1625 // Generate other vector nodes to implement the masked/non-masked vector negation. 1626 Node* NegVNode::degenerate_integral_negate(PhaseGVN* phase, bool is_predicated) { 1627 const TypeVect* vt = vect_type(); 1628 BasicType bt = vt->element_basic_type(); 1629 uint vlen = length(); 1630 1631 // Transformation for predicated NegVI/L 1632 if (is_predicated) { 1633 // (NegVI/L src m) ==> (AddVI/L (XorV src (ReplicateI/L -1) m) (ReplicateI/L 1) m) 1634 Node* const_minus_one = NULL; 1635 Node* const_one = NULL; 1636 int add_opc; 1637 if (bt == T_LONG) { 1638 const_minus_one = phase->longcon(-1L); 1639 const_one = phase->longcon(1L); 1640 add_opc = Op_AddL; 1641 } else { 1642 const_minus_one = phase->intcon(-1); 1643 const_one = phase->intcon(1); 1644 add_opc = Op_AddI; 1645 } 1646 const_minus_one = phase->transform(VectorNode::scalar2vector(const_minus_one, vlen, Type::get_const_basic_type(bt))); 1647 Node* xorv = VectorNode::make(Op_XorV, in(1), const_minus_one, vt); 1648 xorv->add_req(in(2)); 1649 xorv->add_flag(Node::Flag_is_predicated_vector); 1650 phase->transform(xorv); 1651 const_one = phase->transform(VectorNode::scalar2vector(const_one, vlen, Type::get_const_basic_type(bt))); 1652 Node* addv = VectorNode::make(VectorNode::opcode(add_opc, bt), xorv, const_one, vt); 1653 addv->add_req(in(2)); 1654 addv->add_flag(Node::Flag_is_predicated_vector); 1655 return addv; 1656 } 1657 1658 // NegVI/L ==> (SubVI/L (ReplicateI/L 0) src) 1659 Node* const_zero = NULL; 1660 int sub_opc; 1661 if (bt == T_LONG) { 1662 const_zero = phase->longcon(0L); 1663 sub_opc = Op_SubL; 1664 } else { 1665 const_zero = phase->intcon(0); 1666 sub_opc = Op_SubI; 1667 } 1668 const_zero = phase->transform(VectorNode::scalar2vector(const_zero, vlen, Type::get_const_basic_type(bt))); 1669 return VectorNode::make(VectorNode::opcode(sub_opc, bt), const_zero, in(1), vt); 1670 } 1671 1672 Node* NegVNode::Ideal(PhaseGVN* phase, bool can_reshape) { 1673 BasicType bt = vect_type()->element_basic_type(); 1674 uint vlen = length(); 1675 int opc = Opcode(); 1676 if (is_vector_integral_negate(opc)) { 1677 if (is_predicated_vector()) { 1678 if (!Matcher::match_rule_supported_vector_masked(opc, vlen, bt)) { 1679 return degenerate_integral_negate(phase, true); 1680 } 1681 } else if (!Matcher::match_rule_supported_vector(opc, vlen, bt)) { 1682 return degenerate_integral_negate(phase, false); 1683 } 1684 } 1685 return NULL; 1686 } 1687 1688 #ifndef PRODUCT 1689 void VectorBoxAllocateNode::dump_spec(outputStream *st) const { 1690 CallStaticJavaNode::dump_spec(st); 1691 } 1692 1693 #endif // !PRODUCT