< prev index next >

src/hotspot/cpu/aarch64/c2_MacroAssembler_aarch64.cpp

Print this page

 955     case BoolTest::eq: condition = Assembler::EQ; break;
 956     case BoolTest::ne: condition = Assembler::NE; break;
 957     case BoolTest::le: z1 = zm; z2 = zn; condition = Assembler::GE; break;
 958     case BoolTest::ge: condition = Assembler::GE; break;
 959     case BoolTest::lt: z1 = zm; z2 = zn; condition = Assembler::GT; break;
 960     case BoolTest::gt: condition = Assembler::GT; break;
 961     default:
 962       assert(false, "unsupported compare condition");
 963       ShouldNotReachHere();
 964   }
 965 
 966   SIMD_RegVariant size = elemType_to_regVariant(bt);
 967   if (bt == T_FLOAT || bt == T_DOUBLE) {
 968     sve_fcm(condition, pd, size, pg, z1, z2);
 969   } else {
 970     assert(is_integral_type(bt), "unsupported element type");
 971     sve_cmp(condition, pd, size, pg, z1, z2);
 972   }
 973 }
 974 
 975 void C2_MacroAssembler::sve_vmask_reduction(int opc, Register dst, SIMD_RegVariant size, FloatRegister src,
 976                                             PRegister pg, PRegister pn, int length) {























































































































 977   assert(pg->is_governing(), "This register has to be a governing predicate register");
 978   // The conditional flags will be clobbered by this function
 979   sve_cmp(Assembler::NE, pn, size, pg, src, 0);

 980   switch (opc) {
 981     case Op_VectorMaskTrueCount:
 982       sve_cntp(dst, size, ptrue, pn);




































 983       break;
 984     case Op_VectorMaskFirstTrue:
 985       sve_brkb(pn, pg, pn, false);
 986       sve_cntp(dst, size, ptrue, pn);




















 987       break;
 988     case Op_VectorMaskLastTrue:
 989       sve_rev(pn, size, pn);
 990       sve_brkb(pn, ptrue, pn, false);
 991       sve_cntp(dst, size, ptrue, pn);
 992       movw(rscratch1, length - 1);
 993       subw(dst, rscratch1, dst);






 994       break;

 995     default:
 996       assert(false, "unsupported");
 997       ShouldNotReachHere();
 998   }








 999 }

 955     case BoolTest::eq: condition = Assembler::EQ; break;
 956     case BoolTest::ne: condition = Assembler::NE; break;
 957     case BoolTest::le: z1 = zm; z2 = zn; condition = Assembler::GE; break;
 958     case BoolTest::ge: condition = Assembler::GE; break;
 959     case BoolTest::lt: z1 = zm; z2 = zn; condition = Assembler::GT; break;
 960     case BoolTest::gt: condition = Assembler::GT; break;
 961     default:
 962       assert(false, "unsupported compare condition");
 963       ShouldNotReachHere();
 964   }
 965 
 966   SIMD_RegVariant size = elemType_to_regVariant(bt);
 967   if (bt == T_FLOAT || bt == T_DOUBLE) {
 968     sve_fcm(condition, pd, size, pg, z1, z2);
 969   } else {
 970     assert(is_integral_type(bt), "unsupported element type");
 971     sve_cmp(condition, pd, size, pg, z1, z2);
 972   }
 973 }
 974 
 975 // Get index of the last mask lane that is set
 976 void C2_MacroAssembler::sve_vmask_lasttrue(Register dst, BasicType bt, PRegister src, PRegister ptmp) {
 977   SIMD_RegVariant size = elemType_to_regVariant(bt);
 978   sve_rev(ptmp, size, src);
 979   sve_brkb(ptmp, ptrue, ptmp, false);
 980   sve_cntp(dst, size, ptrue, ptmp);
 981   movw(rscratch1, MaxVectorSize / type2aelembytes(bt) - 1);
 982   subw(dst, rscratch1, dst);
 983 }
 984 
 985 void C2_MacroAssembler::sve_vector_extend(FloatRegister dst, SIMD_RegVariant dst_size,
 986                                           FloatRegister src, SIMD_RegVariant src_size) {
 987   assert(dst_size > src_size && dst_size <= D && src_size <= S, "invalid element size");
 988   if (src_size == B) {
 989     switch (dst_size) {
 990     case H:
 991       sve_sunpklo(dst, H, src);
 992       break;
 993     case S:
 994       sve_sunpklo(dst, H, src);
 995       sve_sunpklo(dst, S, dst);
 996       break;
 997     case D:
 998       sve_sunpklo(dst, H, src);
 999       sve_sunpklo(dst, S, dst);
1000       sve_sunpklo(dst, D, dst);
1001       break;
1002     default:
1003       ShouldNotReachHere();
1004     }
1005   } else if (src_size == H) {
1006     if (dst_size == S) {
1007       sve_sunpklo(dst, S, src);
1008     } else { // D
1009       sve_sunpklo(dst, S, src);
1010       sve_sunpklo(dst, D, dst);
1011     }
1012   } else if (src_size == S) {
1013     sve_sunpklo(dst, D, src);
1014   }
1015 }
1016 
1017 // Vector narrow from src to dst with specified element sizes.
1018 // High part of dst vector will be filled with zero.
1019 void C2_MacroAssembler::sve_vector_narrow(FloatRegister dst, SIMD_RegVariant dst_size,
1020                                           FloatRegister src, SIMD_RegVariant src_size,
1021                                           FloatRegister tmp) {
1022   assert(dst_size < src_size && dst_size <= S && src_size <= D, "invalid element size");
1023   sve_dup(tmp, src_size, 0);
1024   if (src_size == D) {
1025     switch (dst_size) {
1026     case S:
1027       sve_uzp1(dst, S, src, tmp);
1028       break;
1029     case H:
1030       sve_uzp1(dst, S, src, tmp);
1031       sve_uzp1(dst, H, dst, tmp);
1032       break;
1033     case B:
1034       sve_uzp1(dst, S, src, tmp);
1035       sve_uzp1(dst, H, dst, tmp);
1036       sve_uzp1(dst, B, dst, tmp);
1037       break;
1038     default:
1039       ShouldNotReachHere();
1040     }
1041   } else if (src_size == S) {
1042     if (dst_size == H) {
1043       sve_uzp1(dst, H, src, tmp);
1044     } else { // B
1045       sve_uzp1(dst, H, src, tmp);
1046       sve_uzp1(dst, B, dst, tmp);
1047     }
1048   } else if (src_size == H) {
1049     sve_uzp1(dst, B, src, tmp);
1050   }
1051 }
1052 
1053 // Extend src predicate to dst predicate with the same lane count but larger
1054 // element size, e.g. 64Byte -> 512Long
1055 void C2_MacroAssembler::sve_vmaskcast_extend(PRegister dst, PRegister src,
1056                                              uint dst_element_length_in_bytes,
1057                                              uint src_element_length_in_bytes) {
1058   if (dst_element_length_in_bytes == 2 * src_element_length_in_bytes) {
1059     sve_punpklo(dst, src);
1060   } else if (dst_element_length_in_bytes == 4 * src_element_length_in_bytes) {
1061     sve_punpklo(dst, src);
1062     sve_punpklo(dst, dst);
1063   } else if (dst_element_length_in_bytes == 8 * src_element_length_in_bytes) {
1064     sve_punpklo(dst, src);
1065     sve_punpklo(dst, dst);
1066     sve_punpklo(dst, dst);
1067   } else {
1068     assert(false, "unsupported");
1069     ShouldNotReachHere();
1070   }
1071 }
1072 
1073 // Narrow src predicate to dst predicate with the same lane count but
1074 // smaller element size, e.g. 512Long -> 64Byte
1075 void C2_MacroAssembler::sve_vmaskcast_narrow(PRegister dst, PRegister src,
1076                                              uint dst_element_length_in_bytes, uint src_element_length_in_bytes) {
1077   // The insignificant bits in src predicate are expected to be zero.
1078   if (dst_element_length_in_bytes * 2 == src_element_length_in_bytes) {
1079     sve_uzp1(dst, B, src, src);
1080   } else if (dst_element_length_in_bytes * 4 == src_element_length_in_bytes) {
1081     sve_uzp1(dst, H, src, src);
1082     sve_uzp1(dst, B, dst, dst);
1083   } else if (dst_element_length_in_bytes * 8 == src_element_length_in_bytes) {
1084     sve_uzp1(dst, S, src, src);
1085     sve_uzp1(dst, H, dst, dst);
1086     sve_uzp1(dst, B, dst, dst);
1087   } else {
1088     assert(false, "unsupported");
1089     ShouldNotReachHere();
1090   }
1091 }
1092 
1093 void C2_MacroAssembler::sve_reduce_integral(int opc, Register dst, BasicType bt, Register src1,
1094                                             FloatRegister src2, PRegister pg, FloatRegister tmp) {
1095   assert(bt == T_BYTE || bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported element type");
1096   assert(pg->is_governing(), "This register has to be a governing predicate register");
1097   assert_different_registers(src1, dst);
1098   // Register "dst" and "tmp" are to be clobbered, and "src1" and "src2" should be preserved.
1099   Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
1100   switch (opc) {
1101     case Op_AddReductionVI: {
1102       sve_uaddv(tmp, size, pg, src2);
1103       smov(dst, tmp, size, 0);
1104       if (bt == T_BYTE) {
1105         addw(dst, src1, dst, ext::sxtb);
1106       } else if (bt == T_SHORT) {
1107         addw(dst, src1, dst, ext::sxth);
1108       } else {
1109         addw(dst, dst, src1);
1110       }
1111       break;
1112     }
1113     case Op_AddReductionVL: {
1114       sve_uaddv(tmp, size, pg, src2);
1115       umov(dst, tmp, size, 0);
1116       add(dst, dst, src1);
1117       break;
1118     }
1119     case Op_AndReductionV: {
1120       sve_andv(tmp, size, pg, src2);
1121       if (bt == T_LONG) {
1122         umov(dst, tmp, size, 0);
1123         andr(dst, dst, src1);
1124       } else {
1125         smov(dst, tmp, size, 0);
1126         andw(dst, dst, src1);
1127       }
1128       break;
1129     }
1130     case Op_OrReductionV: {
1131       sve_orv(tmp, size, pg, src2);
1132       if (bt == T_LONG) {
1133         umov(dst, tmp, size, 0);
1134         orr(dst, dst, src1);
1135       } else {
1136         smov(dst, tmp, size, 0);
1137         orrw(dst, dst, src1);
1138       }
1139       break;
1140     }
1141     case Op_XorReductionV: {
1142       sve_eorv(tmp, size, pg, src2);
1143       if (bt == T_LONG) {
1144         umov(dst, tmp, size, 0);
1145         eor(dst, dst, src1);
1146       } else {
1147         smov(dst, tmp, size, 0);
1148         eorw(dst, dst, src1);
1149       }
1150       break;
1151     }
1152     case Op_MaxReductionV: {
1153       sve_smaxv(tmp, size, pg, src2);
1154       if (bt == T_LONG) {
1155         umov(dst, tmp, size, 0);
1156         cmp(dst, src1);
1157         csel(dst, dst, src1, Assembler::GT);
1158       } else {
1159         smov(dst, tmp, size, 0);
1160         cmpw(dst, src1);
1161         cselw(dst, dst, src1, Assembler::GT);
1162       }
1163       break;
1164     }
1165     case Op_MinReductionV: {
1166       sve_sminv(tmp, size, pg, src2);
1167       if (bt == T_LONG) {
1168         umov(dst, tmp, size, 0);
1169         cmp(dst, src1);
1170         csel(dst, dst, src1, Assembler::LT);
1171       } else {
1172         smov(dst, tmp, size, 0);
1173         cmpw(dst, src1);
1174         cselw(dst, dst, src1, Assembler::LT);
1175       }
1176       break;
1177     }
1178     default:
1179       assert(false, "unsupported");
1180       ShouldNotReachHere();
1181   }
1182 
1183   if (opc == Op_AndReductionV || opc == Op_OrReductionV || opc == Op_XorReductionV) {
1184     if (bt == T_BYTE) {
1185       sxtb(dst, dst);
1186     } else if (bt == T_SHORT) {
1187       sxth(dst, dst);
1188     }
1189   }
1190 }
< prev index next >