1 /*
  2  * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #include "precompiled.hpp"
 26 #include "jni.h"
 27 #include "jvm.h"
 28 #include "classfile/javaClasses.inline.hpp"
 29 #include "classfile/vmClasses.hpp"
 30 #include "classfile/vmSymbols.hpp"
 31 #include "code/location.hpp"
 32 #include "oops/klass.inline.hpp"
 33 #include "oops/typeArrayOop.inline.hpp"
 34 #include "prims/vectorSupport.hpp"
 35 #include "runtime/fieldDescriptor.inline.hpp"
 36 #include "runtime/frame.inline.hpp"
 37 #include "runtime/handles.inline.hpp"
 38 #include "runtime/interfaceSupport.inline.hpp"
 39 #include "runtime/jniHandles.inline.hpp"
 40 #include "runtime/stackValue.hpp"
 41 
 42 #ifdef COMPILER2
 43 #include "opto/matcher.hpp" // Matcher::max_vector_size(BasicType)
 44 #endif // COMPILER2
 45 
 46 #ifdef COMPILER2
 47 const char* VectorSupport::svmlname[VectorSupport::NUM_SVML_OP] = {
 48     "tan",
 49     "tanh",
 50     "sin",
 51     "sinh",
 52     "cos",
 53     "cosh",
 54     "asin",
 55     "acos",
 56     "atan",
 57     "atan2",
 58     "cbrt",
 59     "log",
 60     "log10",
 61     "log1p",
 62     "pow",
 63     "exp",
 64     "expm1",
 65     "hypot",
 66 };
 67 #endif
 68 
 69 bool VectorSupport::is_vector(Klass* klass) {
 70   return klass->is_subclass_of(vmClasses::vector_VectorPayload_klass());
 71 }
 72 
 73 bool VectorSupport::is_vector_mask(Klass* klass) {
 74   return klass->is_subclass_of(vmClasses::vector_VectorMask_klass());
 75 }
 76 
 77 bool VectorSupport::is_vector_shuffle(Klass* klass) {
 78   return klass->is_subclass_of(vmClasses::vector_VectorShuffle_klass());
 79 }
 80 
 81 BasicType VectorSupport::klass2bt(InstanceKlass* ik) {
 82   assert(ik->is_subclass_of(vmClasses::vector_VectorPayload_klass()), "%s not a VectorPayload", ik->name()->as_C_string());
 83   fieldDescriptor fd; // find_field initializes fd if found
 84   // static final Class<?> ETYPE;
 85   Klass* holder = ik->find_field(vmSymbols::ETYPE_name(), vmSymbols::class_signature(), &fd);
 86 
 87   assert(holder != NULL, "sanity");
 88   assert(fd.is_static(), "");
 89   assert(fd.offset() > 0, "");
 90 
 91   if (is_vector_shuffle(ik)) {
 92     return T_BYTE;
 93   } else if (is_vector_mask(ik)) {
 94     return T_BOOLEAN;
 95   } else { // vector and mask
 96     oop value = ik->java_mirror()->obj_field(fd.offset());
 97     BasicType elem_bt = java_lang_Class::as_BasicType(value);
 98     return elem_bt;
 99   }
100 }
101 
102 jint VectorSupport::klass2length(InstanceKlass* ik) {
103   fieldDescriptor fd; // find_field initializes fd if found
104   // static final int VLENGTH;
105   Klass* holder = ik->find_field(vmSymbols::VLENGTH_name(), vmSymbols::int_signature(), &fd);
106 
107   assert(holder != NULL, "sanity");
108   assert(fd.is_static(), "");
109   assert(fd.offset() > 0, "");
110 
111   jint vlen = ik->java_mirror()->int_field(fd.offset());
112   assert(vlen > 0, "");
113   return vlen;
114 }
115 
116 // Masks require special handling: when boxed they are packed and stored in boolean
117 // arrays, but in scalarized form they have the same size as corresponding vectors.
118 // For example, Int512Mask is represented in memory as boolean[16], but
119 // occupies the whole 512-bit vector register when scalarized.
120 // During scalarization inserting a VectorStoreMask node between mask
121 // and safepoint node always ensures the existence of masks in a boolean array.
122 
123 void VectorSupport::init_payload_element(typeArrayOop arr, BasicType elem_bt, int index, address addr) {
124   switch (elem_bt) {
125     case T_BOOLEAN: arr->bool_at_put(index, *(jboolean*)addr); break;
126     case T_BYTE:    arr->byte_at_put(index, *(jbyte*)addr); break;
127     case T_SHORT:   arr->short_at_put(index, *(jshort*)addr); break;
128     case T_INT:     arr->int_at_put(index, *(jint*)addr); break;
129     case T_FLOAT:   arr->float_at_put(index, *(jfloat*)addr); break;
130     case T_LONG:    arr->long_at_put(index, *(jlong*)addr); break;
131     case T_DOUBLE:  arr->double_at_put(index, *(jdouble*)addr); break;
132     default: fatal("unsupported: %s", type2name(elem_bt));
133   }
134 }
135 
136 Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS) {
137   int num_elem = klass2length(ik);
138   BasicType elem_bt = klass2bt(ik);
139   int elem_size = type2aelembytes(elem_bt);
140 
141   // On-heap vector values are represented as primitive arrays.
142   TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(elem_bt));
143 
144   typeArrayOop arr = tak->allocate(num_elem, CHECK_NH); // safepoint
145 
146   if (location.is_register()) {
147     // Value was in a callee-saved register.
148     VMReg vreg = VMRegImpl::as_VMReg(location.register_number());
149 
150     for (int i = 0; i < num_elem; i++) {
151       int vslot = (i * elem_size) / VMRegImpl::stack_slot_size;
152       int off   = (i * elem_size) % VMRegImpl::stack_slot_size;
153 
154       address elem_addr = reg_map->location(vreg, vslot) + off; // assumes little endian element order
155       init_payload_element(arr, elem_bt, i, elem_addr);
156     }
157   } else {
158     // Value was directly saved on the stack.
159     address base_addr = ((address)fr->unextended_sp()) + location.stack_offset();
160     for (int i = 0; i < num_elem; i++) {
161       init_payload_element(arr, elem_bt, i, base_addr + i * elem_size);
162     }
163   }
164   return Handle(THREAD, arr);
165 }
166 
167 Handle VectorSupport::allocate_vector_payload(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ScopeValue* payload, TRAPS) {
168   if (payload->is_location()) {
169     Location location = payload->as_LocationValue()->location();
170     if (location.type() == Location::vector) {
171       // Vector value in an aligned adjacent tuple (1, 2, 4, 8, or 16 slots).
172       return allocate_vector_payload_helper(ik, fr, reg_map, location, THREAD); // safepoint
173     }
174 #ifdef ASSERT
175     // Other payload values are: 'oop' type location and Scalar-replaced boxed vector representation.
176     // They will be processed in Deoptimization::reassign_fields() after all objects are reallocated.
177     else {
178       Location::Type loc_type = location.type();
179       assert(loc_type == Location::oop || loc_type == Location::narrowoop,
180              "expected 'oop'(%d) or 'narrowoop'(%d) types location but got: %d", Location::oop, Location::narrowoop, loc_type);
181     }
182   } else if (!payload->is_object()) {
183     stringStream ss;
184     payload->print_on(&ss);
185     assert(payload->is_object(), "expected 'object' value for scalar-replaced boxed vector but got: %s", ss.as_string());
186 #endif
187   }
188   return Handle(THREAD, nullptr);
189 }
190 
191 instanceOop VectorSupport::allocate_vector(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ObjectValue* ov, TRAPS) {
192   assert(is_vector(ik), "%s not a vector", ik->name()->as_C_string());
193   assert(ov->field_size() == 1, "%s not a vector", ik->name()->as_C_string());
194 
195   ScopeValue* payload_value = ov->field_at(0);
196   Handle payload_instance = VectorSupport::allocate_vector_payload(ik, fr, reg_map, payload_value, CHECK_NULL);
197   instanceOop vbox = ik->allocate_instance(CHECK_NULL);
198   vector_VectorPayload::set_payload(vbox, payload_instance());
199   return vbox;
200 }
201 
202 #ifdef COMPILER2
203 int VectorSupport::vop2ideal(jint id, BasicType bt) {
204   VectorOperation vop = (VectorOperation)id;
205   switch (vop) {
206     case VECTOR_OP_ADD: {
207       switch (bt) {
208         case T_BYTE:   // fall-through
209         case T_SHORT:  // fall-through
210         case T_INT:    return Op_AddI;
211         case T_LONG:   return Op_AddL;
212         case T_FLOAT:  return Op_AddF;
213         case T_DOUBLE: return Op_AddD;
214         default: fatal("ADD: %s", type2name(bt));
215       }
216       break;
217     }
218     case VECTOR_OP_SUB: {
219       switch (bt) {
220         case T_BYTE:   // fall-through
221         case T_SHORT:  // fall-through
222         case T_INT:    return Op_SubI;
223         case T_LONG:   return Op_SubL;
224         case T_FLOAT:  return Op_SubF;
225         case T_DOUBLE: return Op_SubD;
226         default: fatal("SUB: %s", type2name(bt));
227       }
228       break;
229     }
230     case VECTOR_OP_MUL: {
231       switch (bt) {
232         case T_BYTE:   // fall-through
233         case T_SHORT:  // fall-through
234         case T_INT:    return Op_MulI;
235         case T_LONG:   return Op_MulL;
236         case T_FLOAT:  return Op_MulF;
237         case T_DOUBLE: return Op_MulD;
238         default: fatal("MUL: %s", type2name(bt));
239       }
240       break;
241     }
242     case VECTOR_OP_DIV: {
243       switch (bt) {
244         case T_BYTE:   // fall-through
245         case T_SHORT:  // fall-through
246         case T_INT:    return Op_DivI;
247         case T_LONG:   return Op_DivL;
248         case T_FLOAT:  return Op_DivF;
249         case T_DOUBLE: return Op_DivD;
250         default: fatal("DIV: %s", type2name(bt));
251       }
252       break;
253     }
254     case VECTOR_OP_MIN: {
255       switch (bt) {
256         case T_BYTE:
257         case T_SHORT:
258         case T_INT:    return Op_MinI;
259         case T_LONG:   return Op_MinL;
260         case T_FLOAT:  return Op_MinF;
261         case T_DOUBLE: return Op_MinD;
262         default: fatal("MIN: %s", type2name(bt));
263       }
264       break;
265     }
266     case VECTOR_OP_MAX: {
267       switch (bt) {
268         case T_BYTE:
269         case T_SHORT:
270         case T_INT:    return Op_MaxI;
271         case T_LONG:   return Op_MaxL;
272         case T_FLOAT:  return Op_MaxF;
273         case T_DOUBLE: return Op_MaxD;
274         default: fatal("MAX: %s", type2name(bt));
275       }
276       break;
277     }
278     case VECTOR_OP_ABS: {
279       switch (bt) {
280         case T_BYTE:   // fall-through
281         case T_SHORT:  // fall-through
282         case T_INT:    return Op_AbsI;
283         case T_LONG:   return Op_AbsL;
284         case T_FLOAT:  return Op_AbsF;
285         case T_DOUBLE: return Op_AbsD;
286         default: fatal("ABS: %s", type2name(bt));
287       }
288       break;
289     }
290     case VECTOR_OP_NEG: {
291       switch (bt) {
292         case T_BYTE:   // fall-through
293         case T_SHORT:  // fall-through
294         case T_INT:    return Op_NegI;
295         case T_FLOAT:  return Op_NegF;
296         case T_DOUBLE: return Op_NegD;
297         default: fatal("NEG: %s", type2name(bt));
298       }
299       break;
300     }
301     case VECTOR_OP_AND: {
302       switch (bt) {
303         case T_BYTE:   // fall-through
304         case T_SHORT:  // fall-through
305         case T_INT:    return Op_AndI;
306         case T_LONG:   return Op_AndL;
307         default: fatal("AND: %s", type2name(bt));
308       }
309       break;
310     }
311     case VECTOR_OP_OR: {
312       switch (bt) {
313         case T_BYTE:   // fall-through
314         case T_SHORT:  // fall-through
315         case T_INT:    return Op_OrI;
316         case T_LONG:   return Op_OrL;
317         default: fatal("OR: %s", type2name(bt));
318       }
319       break;
320     }
321     case VECTOR_OP_XOR: {
322       switch (bt) {
323         case T_BYTE:   // fall-through
324         case T_SHORT:  // fall-through
325         case T_INT:    return Op_XorI;
326         case T_LONG:   return Op_XorL;
327         default: fatal("XOR: %s", type2name(bt));
328       }
329       break;
330     }
331     case VECTOR_OP_SQRT: {
332       switch (bt) {
333         case T_FLOAT:  return Op_SqrtF;
334         case T_DOUBLE: return Op_SqrtD;
335         default: fatal("SQRT: %s", type2name(bt));
336       }
337       break;
338     }
339     case VECTOR_OP_FMA: {
340       switch (bt) {
341         case T_FLOAT:  return Op_FmaF;
342         case T_DOUBLE: return Op_FmaD;
343         default: fatal("FMA: %s", type2name(bt));
344       }
345       break;
346     }
347     case VECTOR_OP_LSHIFT: {
348       switch (bt) {
349         case T_BYTE:   // fall-through
350         case T_SHORT:  // fall-through
351         case T_INT:  return Op_LShiftI;
352         case T_LONG: return Op_LShiftL;
353         default: fatal("LSHIFT: %s", type2name(bt));
354       }
355       break;
356     }
357     case VECTOR_OP_RSHIFT: {
358       switch (bt) {
359         case T_BYTE:   // fall-through
360         case T_SHORT:  // fall-through
361         case T_INT:  return Op_RShiftI;
362         case T_LONG: return Op_RShiftL;
363         default: fatal("RSHIFT: %s", type2name(bt));
364       }
365       break;
366     }
367     case VECTOR_OP_URSHIFT: {
368       switch (bt) {
369         case T_BYTE:  return Op_URShiftB;
370         case T_SHORT: return Op_URShiftS;
371         case T_INT:   return Op_URShiftI;
372         case T_LONG:  return Op_URShiftL;
373         default: fatal("URSHIFT: %s", type2name(bt));
374       }
375       break;
376     }
377     case VECTOR_OP_LROTATE: {
378       switch (bt) {
379         case T_BYTE:   // fall-through
380         case T_SHORT:  // fall-through
381         case T_INT:    // fall-through
382         case T_LONG:  return Op_RotateLeft;
383         default: fatal("LROTATE: %s", type2name(bt));
384       }
385       break;
386     }
387     case VECTOR_OP_RROTATE: {
388       switch (bt) {
389         case T_BYTE:   // fall-through
390         case T_SHORT:  // fall-through
391         case T_INT:    // fall-through
392         case T_LONG:  return Op_RotateRight;
393         default: fatal("RROTATE: %s", type2name(bt));
394       }
395       break;
396     }
397     case VECTOR_OP_MASK_LASTTRUE: {
398       switch (bt) {
399         case T_BYTE:  // fall-through
400         case T_SHORT: // fall-through
401         case T_INT:   // fall-through
402         case T_LONG:  // fall-through
403         case T_FLOAT: // fall-through
404         case T_DOUBLE: return Op_VectorMaskLastTrue;
405         default: fatal("MASK_LASTTRUE: %s", type2name(bt));
406       }
407       break;
408     }
409     case VECTOR_OP_MASK_FIRSTTRUE: {
410       switch (bt) {
411         case T_BYTE:  // fall-through
412         case T_SHORT: // fall-through
413         case T_INT:   // fall-through
414         case T_LONG:  // fall-through
415         case T_FLOAT: // fall-through
416         case T_DOUBLE: return Op_VectorMaskFirstTrue;
417         default: fatal("MASK_FIRSTTRUE: %s", type2name(bt));
418       }
419       break;
420     }
421     case VECTOR_OP_MASK_TRUECOUNT: {
422       switch (bt) {
423         case T_BYTE:  // fall-through
424         case T_SHORT: // fall-through
425         case T_INT:   // fall-through
426         case T_LONG:  // fall-through
427         case T_FLOAT: // fall-through
428         case T_DOUBLE: return Op_VectorMaskTrueCount;
429         default: fatal("MASK_TRUECOUNT: %s", type2name(bt));
430       }
431       break;
432     }
433     case VECTOR_OP_MASK_TOLONG: {
434       switch (bt) {
435         case T_BYTE:  // fall-through
436         case T_SHORT: // fall-through
437         case T_INT:   // fall-through
438         case T_LONG:  // fall-through
439         case T_FLOAT: // fall-through
440         case T_DOUBLE: return Op_VectorMaskToLong;
441         default: fatal("MASK_TOLONG: %s", type2name(bt));
442       }
443       break;
444     }
445     case VECTOR_OP_TAN:
446     case VECTOR_OP_TANH:
447     case VECTOR_OP_SIN:
448     case VECTOR_OP_SINH:
449     case VECTOR_OP_COS:
450     case VECTOR_OP_COSH:
451     case VECTOR_OP_ASIN:
452     case VECTOR_OP_ACOS:
453     case VECTOR_OP_ATAN:
454     case VECTOR_OP_ATAN2:
455     case VECTOR_OP_CBRT:
456     case VECTOR_OP_LOG:
457     case VECTOR_OP_LOG10:
458     case VECTOR_OP_LOG1P:
459     case VECTOR_OP_POW:
460     case VECTOR_OP_EXP:
461     case VECTOR_OP_EXPM1:
462     case VECTOR_OP_HYPOT:
463       return Op_CallLeafVector;
464     default: fatal("unknown op: %d", vop);
465   }
466   return 0; // Unimplemented
467 }
468 #endif // COMPILER2
469 
470 /**
471  * Implementation of the jdk.internal.vm.vector.VectorSupport class
472  */
473 
474 JVM_ENTRY(jint, VectorSupport_GetMaxLaneCount(JNIEnv *env, jclass vsclazz, jobject clazz)) {
475 #ifdef COMPILER2
476   oop mirror = JNIHandles::resolve_non_null(clazz);
477   if (java_lang_Class::is_primitive(mirror)) {
478     BasicType bt = java_lang_Class::primitive_type(mirror);
479     return Matcher::max_vector_size(bt);
480   }
481 #endif // COMPILER2
482   return -1;
483 } JVM_END
484 
485 // JVM_RegisterVectorSupportMethods
486 
487 #define LANG "Ljava/lang/"
488 #define CLS LANG "Class;"
489 
490 #define CC (char*)  /*cast a literal from (const char*)*/
491 #define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f)
492 
493 static JNINativeMethod jdk_internal_vm_vector_VectorSupport_methods[] = {
494     {CC "getMaxLaneCount",   CC "(" CLS ")I", FN_PTR(VectorSupport_GetMaxLaneCount)}
495 };
496 
497 #undef CC
498 #undef FN_PTR
499 
500 #undef LANG
501 #undef CLS
502 
503 // This function is exported, used by NativeLookup.
504 
505 JVM_ENTRY(void, JVM_RegisterVectorSupportMethods(JNIEnv* env, jclass vsclass)) {
506   ThreadToNativeFromVM ttnfv(thread);
507 
508   int ok = env->RegisterNatives(vsclass, jdk_internal_vm_vector_VectorSupport_methods, sizeof(jdk_internal_vm_vector_VectorSupport_methods)/sizeof(JNINativeMethod));
509   guarantee(ok == 0, "register jdk.internal.vm.vector.VectorSupport natives");
510 } JVM_END