1 /*
2 * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "jni.h"
27 #include "jvm.h"
28 #include "classfile/javaClasses.inline.hpp"
29 #include "classfile/vmClasses.hpp"
30 #include "classfile/vmSymbols.hpp"
31 #include "code/location.hpp"
32 #include "oops/klass.inline.hpp"
33 #include "oops/typeArrayOop.inline.hpp"
34 #include "prims/vectorSupport.hpp"
35 #include "runtime/fieldDescriptor.inline.hpp"
36 #include "runtime/frame.inline.hpp"
37 #include "runtime/handles.inline.hpp"
38 #include "runtime/interfaceSupport.inline.hpp"
39 #include "runtime/jniHandles.inline.hpp"
40 #include "runtime/stackValue.hpp"
41
42 #ifdef COMPILER2
43 #include "opto/matcher.hpp" // Matcher::max_vector_size(BasicType)
44 #endif // COMPILER2
45
46 #ifdef COMPILER2
47 const char* VectorSupport::svmlname[VectorSupport::NUM_SVML_OP] = {
48 "tan",
49 "tanh",
50 "sin",
51 "sinh",
52 "cos",
53 "cosh",
54 "asin",
55 "acos",
56 "atan",
57 "atan2",
58 "cbrt",
59 "log",
60 "log10",
61 "log1p",
62 "pow",
63 "exp",
64 "expm1",
65 "hypot",
66 };
67 #endif
68
69 bool VectorSupport::is_vector(Klass* klass) {
70 return klass->is_subclass_of(vmClasses::vector_VectorPayload_klass());
71 }
72
73 bool VectorSupport::is_vector_mask(Klass* klass) {
74 return klass->is_subclass_of(vmClasses::vector_VectorMask_klass());
75 }
76
77 bool VectorSupport::is_vector_shuffle(Klass* klass) {
78 return klass->is_subclass_of(vmClasses::vector_VectorShuffle_klass());
79 }
80
81 BasicType VectorSupport::klass2bt(InstanceKlass* ik) {
82 assert(ik->is_subclass_of(vmClasses::vector_VectorPayload_klass()), "%s not a VectorPayload", ik->name()->as_C_string());
83 fieldDescriptor fd; // find_field initializes fd if found
84 // static final Class<?> ETYPE;
85 Klass* holder = ik->find_field(vmSymbols::ETYPE_name(), vmSymbols::class_signature(), &fd);
86
87 assert(holder != NULL, "sanity");
88 assert(fd.is_static(), "");
89 assert(fd.offset() > 0, "");
90
91 if (is_vector_shuffle(ik)) {
92 return T_BYTE;
93 } else if (is_vector_mask(ik)) {
94 return T_BOOLEAN;
95 } else { // vector and mask
96 oop value = ik->java_mirror()->obj_field(fd.offset());
97 BasicType elem_bt = java_lang_Class::as_BasicType(value);
98 return elem_bt;
99 }
100 }
101
102 jint VectorSupport::klass2length(InstanceKlass* ik) {
103 fieldDescriptor fd; // find_field initializes fd if found
104 // static final int VLENGTH;
105 Klass* holder = ik->find_field(vmSymbols::VLENGTH_name(), vmSymbols::int_signature(), &fd);
106
107 assert(holder != NULL, "sanity");
108 assert(fd.is_static(), "");
109 assert(fd.offset() > 0, "");
110
111 jint vlen = ik->java_mirror()->int_field(fd.offset());
112 assert(vlen > 0, "");
113 return vlen;
114 }
115
116 // Masks require special handling: when boxed they are packed and stored in boolean
117 // arrays, but in scalarized form they have the same size as corresponding vectors.
118 // For example, Int512Mask is represented in memory as boolean[16], but
119 // occupies the whole 512-bit vector register when scalarized.
120 // During scalarization inserting a VectorStoreMask node between mask
121 // and safepoint node always ensures the existence of masks in a boolean array.
122
123 void VectorSupport::init_payload_element(typeArrayOop arr, BasicType elem_bt, int index, address addr) {
124 switch (elem_bt) {
125 case T_BOOLEAN: arr->bool_at_put(index, *(jboolean*)addr); break;
126 case T_BYTE: arr->byte_at_put(index, *(jbyte*)addr); break;
127 case T_SHORT: arr->short_at_put(index, *(jshort*)addr); break;
128 case T_INT: arr->int_at_put(index, *(jint*)addr); break;
129 case T_FLOAT: arr->float_at_put(index, *(jfloat*)addr); break;
130 case T_LONG: arr->long_at_put(index, *(jlong*)addr); break;
131 case T_DOUBLE: arr->double_at_put(index, *(jdouble*)addr); break;
132 default: fatal("unsupported: %s", type2name(elem_bt));
133 }
134 }
135
136 Handle VectorSupport::allocate_vector_payload_helper(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, Location location, TRAPS) {
137 int num_elem = klass2length(ik);
138 BasicType elem_bt = klass2bt(ik);
139 int elem_size = type2aelembytes(elem_bt);
140
141 // On-heap vector values are represented as primitive arrays.
142 TypeArrayKlass* tak = TypeArrayKlass::cast(Universe::typeArrayKlassObj(elem_bt));
143
144 typeArrayOop arr = tak->allocate(num_elem, CHECK_NH); // safepoint
145
146 if (location.is_register()) {
147 // Value was in a callee-saved register.
148 VMReg vreg = VMRegImpl::as_VMReg(location.register_number());
149
150 for (int i = 0; i < num_elem; i++) {
151 int vslot = (i * elem_size) / VMRegImpl::stack_slot_size;
152 int off = (i * elem_size) % VMRegImpl::stack_slot_size;
153
154 address elem_addr = reg_map->location(vreg, vslot) + off; // assumes little endian element order
155 init_payload_element(arr, elem_bt, i, elem_addr);
156 }
157 } else {
158 // Value was directly saved on the stack.
159 address base_addr = ((address)fr->unextended_sp()) + location.stack_offset();
160 for (int i = 0; i < num_elem; i++) {
161 init_payload_element(arr, elem_bt, i, base_addr + i * elem_size);
162 }
163 }
164 return Handle(THREAD, arr);
165 }
166
167 Handle VectorSupport::allocate_vector_payload(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ScopeValue* payload, TRAPS) {
168 if (payload->is_location()) {
169 Location location = payload->as_LocationValue()->location();
170 if (location.type() == Location::vector) {
171 // Vector value in an aligned adjacent tuple (1, 2, 4, 8, or 16 slots).
172 return allocate_vector_payload_helper(ik, fr, reg_map, location, THREAD); // safepoint
173 }
174 #ifdef ASSERT
175 // Other payload values are: 'oop' type location and scalar-replaced boxed vector representation.
176 // They will be processed in Deoptimization::reassign_fields() after all objects are reallocated.
177 else {
178 Location::Type loc_type = location.type();
179 assert(loc_type == Location::oop || loc_type == Location::narrowoop,
180 "expected 'oop'(%d) or 'narrowoop'(%d) types location but got: %d", Location::oop, Location::narrowoop, loc_type);
181 }
182 } else if (!payload->is_object() && !payload->is_constant_oop()) {
183 stringStream ss;
184 payload->print_on(&ss);
185 assert(false, "expected 'object' value for scalar-replaced boxed vector but got: %s", ss.as_string());
186 #endif
187 }
188 return Handle(THREAD, nullptr);
189 }
190
191 instanceOop VectorSupport::allocate_vector(InstanceKlass* ik, frame* fr, RegisterMap* reg_map, ObjectValue* ov, TRAPS) {
192 assert(is_vector(ik), "%s not a vector", ik->name()->as_C_string());
193 assert(ov->field_size() == 1, "%s not a vector", ik->name()->as_C_string());
194
195 ScopeValue* payload_value = ov->field_at(0);
196 Handle payload_instance = VectorSupport::allocate_vector_payload(ik, fr, reg_map, payload_value, CHECK_NULL);
197 instanceOop vbox = ik->allocate_instance(CHECK_NULL);
198 vector_VectorPayload::set_payload(vbox, payload_instance());
199 return vbox;
200 }
201
202 #ifdef COMPILER2
203 int VectorSupport::vop2ideal(jint id, BasicType bt) {
204 VectorOperation vop = (VectorOperation)id;
205 switch (vop) {
206 case VECTOR_OP_ADD: {
207 switch (bt) {
208 case T_BYTE: // fall-through
209 case T_SHORT: // fall-through
210 case T_INT: return Op_AddI;
211 case T_LONG: return Op_AddL;
212 case T_FLOAT: return Op_AddF;
213 case T_DOUBLE: return Op_AddD;
214 default: fatal("ADD: %s", type2name(bt));
215 }
216 break;
217 }
218 case VECTOR_OP_SUB: {
219 switch (bt) {
220 case T_BYTE: // fall-through
221 case T_SHORT: // fall-through
222 case T_INT: return Op_SubI;
223 case T_LONG: return Op_SubL;
224 case T_FLOAT: return Op_SubF;
225 case T_DOUBLE: return Op_SubD;
226 default: fatal("SUB: %s", type2name(bt));
227 }
228 break;
229 }
230 case VECTOR_OP_MUL: {
231 switch (bt) {
232 case T_BYTE: // fall-through
233 case T_SHORT: // fall-through
234 case T_INT: return Op_MulI;
235 case T_LONG: return Op_MulL;
236 case T_FLOAT: return Op_MulF;
237 case T_DOUBLE: return Op_MulD;
238 default: fatal("MUL: %s", type2name(bt));
239 }
240 break;
241 }
242 case VECTOR_OP_DIV: {
243 switch (bt) {
244 case T_BYTE: // fall-through
245 case T_SHORT: // fall-through
246 case T_INT: return Op_DivI;
247 case T_LONG: return Op_DivL;
248 case T_FLOAT: return Op_DivF;
249 case T_DOUBLE: return Op_DivD;
250 default: fatal("DIV: %s", type2name(bt));
251 }
252 break;
253 }
254 case VECTOR_OP_MIN: {
255 switch (bt) {
256 case T_BYTE:
257 case T_SHORT:
258 case T_INT: return Op_MinI;
259 case T_LONG: return Op_MinL;
260 case T_FLOAT: return Op_MinF;
261 case T_DOUBLE: return Op_MinD;
262 default: fatal("MIN: %s", type2name(bt));
263 }
264 break;
265 }
266 case VECTOR_OP_MAX: {
267 switch (bt) {
268 case T_BYTE:
269 case T_SHORT:
270 case T_INT: return Op_MaxI;
271 case T_LONG: return Op_MaxL;
272 case T_FLOAT: return Op_MaxF;
273 case T_DOUBLE: return Op_MaxD;
274 default: fatal("MAX: %s", type2name(bt));
275 }
276 break;
277 }
278 case VECTOR_OP_ABS: {
279 switch (bt) {
280 case T_BYTE: // fall-through
281 case T_SHORT: // fall-through
282 case T_INT: return Op_AbsI;
283 case T_LONG: return Op_AbsL;
284 case T_FLOAT: return Op_AbsF;
285 case T_DOUBLE: return Op_AbsD;
286 default: fatal("ABS: %s", type2name(bt));
287 }
288 break;
289 }
290 case VECTOR_OP_NEG: {
291 switch (bt) {
292 case T_BYTE: // fall-through
293 case T_SHORT: // fall-through
294 case T_INT: return Op_NegI;
295 case T_LONG: return Op_NegL;
296 case T_FLOAT: return Op_NegF;
297 case T_DOUBLE: return Op_NegD;
298 default: fatal("NEG: %s", type2name(bt));
299 }
300 break;
301 }
302 case VECTOR_OP_AND: {
303 switch (bt) {
304 case T_BYTE: // fall-through
305 case T_SHORT: // fall-through
306 case T_INT: return Op_AndI;
307 case T_LONG: return Op_AndL;
308 default: fatal("AND: %s", type2name(bt));
309 }
310 break;
311 }
312 case VECTOR_OP_OR: {
313 switch (bt) {
314 case T_BYTE: // fall-through
315 case T_SHORT: // fall-through
316 case T_INT: return Op_OrI;
317 case T_LONG: return Op_OrL;
318 default: fatal("OR: %s", type2name(bt));
319 }
320 break;
321 }
322 case VECTOR_OP_XOR: {
323 switch (bt) {
324 case T_BYTE: // fall-through
325 case T_SHORT: // fall-through
326 case T_INT: return Op_XorI;
327 case T_LONG: return Op_XorL;
328 default: fatal("XOR: %s", type2name(bt));
329 }
330 break;
331 }
332 case VECTOR_OP_SQRT: {
333 switch (bt) {
334 case T_FLOAT: return Op_SqrtF;
335 case T_DOUBLE: return Op_SqrtD;
336 default: fatal("SQRT: %s", type2name(bt));
337 }
338 break;
339 }
340 case VECTOR_OP_FMA: {
341 switch (bt) {
342 case T_FLOAT: return Op_FmaF;
343 case T_DOUBLE: return Op_FmaD;
344 default: fatal("FMA: %s", type2name(bt));
345 }
346 break;
347 }
348 case VECTOR_OP_LSHIFT: {
349 switch (bt) {
350 case T_BYTE: // fall-through
351 case T_SHORT: // fall-through
352 case T_INT: return Op_LShiftI;
353 case T_LONG: return Op_LShiftL;
354 default: fatal("LSHIFT: %s", type2name(bt));
355 }
356 break;
357 }
358 case VECTOR_OP_RSHIFT: {
359 switch (bt) {
360 case T_BYTE: // fall-through
361 case T_SHORT: // fall-through
362 case T_INT: return Op_RShiftI;
363 case T_LONG: return Op_RShiftL;
364 default: fatal("RSHIFT: %s", type2name(bt));
365 }
366 break;
367 }
368 case VECTOR_OP_URSHIFT: {
369 switch (bt) {
370 case T_BYTE: return Op_URShiftB;
371 case T_SHORT: return Op_URShiftS;
372 case T_INT: return Op_URShiftI;
373 case T_LONG: return Op_URShiftL;
374 default: fatal("URSHIFT: %s", type2name(bt));
375 }
376 break;
377 }
378 case VECTOR_OP_LROTATE: {
379 switch (bt) {
380 case T_BYTE: // fall-through
381 case T_SHORT: // fall-through
382 case T_INT: // fall-through
383 case T_LONG: return Op_RotateLeft;
384 default: fatal("LROTATE: %s", type2name(bt));
385 }
386 break;
387 }
388 case VECTOR_OP_RROTATE: {
389 switch (bt) {
390 case T_BYTE: // fall-through
391 case T_SHORT: // fall-through
392 case T_INT: // fall-through
393 case T_LONG: return Op_RotateRight;
394 default: fatal("RROTATE: %s", type2name(bt));
395 }
396 break;
397 }
398 case VECTOR_OP_MASK_LASTTRUE: {
399 switch (bt) {
400 case T_BYTE: // fall-through
401 case T_SHORT: // fall-through
402 case T_INT: // fall-through
403 case T_LONG: // fall-through
404 case T_FLOAT: // fall-through
405 case T_DOUBLE: return Op_VectorMaskLastTrue;
406 default: fatal("MASK_LASTTRUE: %s", type2name(bt));
407 }
408 break;
409 }
410 case VECTOR_OP_MASK_FIRSTTRUE: {
411 switch (bt) {
412 case T_BYTE: // fall-through
413 case T_SHORT: // fall-through
414 case T_INT: // fall-through
415 case T_LONG: // fall-through
416 case T_FLOAT: // fall-through
417 case T_DOUBLE: return Op_VectorMaskFirstTrue;
418 default: fatal("MASK_FIRSTTRUE: %s", type2name(bt));
419 }
420 break;
421 }
422 case VECTOR_OP_MASK_TRUECOUNT: {
423 switch (bt) {
424 case T_BYTE: // fall-through
425 case T_SHORT: // fall-through
426 case T_INT: // fall-through
427 case T_LONG: // fall-through
428 case T_FLOAT: // fall-through
429 case T_DOUBLE: return Op_VectorMaskTrueCount;
430 default: fatal("MASK_TRUECOUNT: %s", type2name(bt));
431 }
432 break;
433 }
434 case VECTOR_OP_MASK_TOLONG: {
435 switch (bt) {
436 case T_BYTE: // fall-through
437 case T_SHORT: // fall-through
438 case T_INT: // fall-through
439 case T_LONG: // fall-through
440 case T_FLOAT: // fall-through
441 case T_DOUBLE: return Op_VectorMaskToLong;
442 default: fatal("MASK_TOLONG: %s", type2name(bt));
443 }
444 break;
445 }
446 case VECTOR_OP_EXPAND: {
447 switch (bt) {
448 case T_BYTE: // fall-through
449 case T_SHORT: // fall-through
450 case T_INT: // fall-through
451 case T_LONG: // fall-through
452 case T_FLOAT: // fall-through
453 case T_DOUBLE: return Op_ExpandV;
454 default: fatal("EXPAND: %s", type2name(bt));
455 }
456 break;
457 }
458 case VECTOR_OP_COMPRESS: {
459 switch (bt) {
460 case T_BYTE: // fall-through
461 case T_SHORT: // fall-through
462 case T_INT: // fall-through
463 case T_LONG: // fall-through
464 case T_FLOAT: // fall-through
465 case T_DOUBLE: return Op_CompressV;
466 default: fatal("COMPRESS: %s", type2name(bt));
467 }
468 break;
469 }
470 case VECTOR_OP_MASK_COMPRESS: {
471 switch (bt) {
472 case T_BYTE: // fall-through
473 case T_SHORT: // fall-through
474 case T_INT: // fall-through
475 case T_LONG: // fall-through
476 case T_FLOAT: // fall-through
477 case T_DOUBLE: return Op_CompressM;
478 default: fatal("MASK_COMPRESS: %s", type2name(bt));
479 }
480 break;
481 }
482 case VECTOR_OP_BIT_COUNT: {
483 switch (bt) {
484 case T_BYTE: // Returning Op_PopCountI
485 case T_SHORT: // for byte and short types temporarily
486 case T_INT: return Op_PopCountI;
487 case T_LONG: return Op_PopCountL;
488 default: fatal("BIT_COUNT: %s", type2name(bt));
489 }
490 break;
491 }
492 case VECTOR_OP_TZ_COUNT: {
493 switch (bt) {
494 case T_BYTE:
495 case T_SHORT:
496 case T_INT: return Op_CountTrailingZerosI;
497 case T_LONG: return Op_CountTrailingZerosL;
498 default: fatal("TZ_COUNT: %s", type2name(bt));
499 }
500 break;
501 }
502 case VECTOR_OP_LZ_COUNT: {
503 switch (bt) {
504 case T_BYTE:
505 case T_SHORT:
506 case T_INT: return Op_CountLeadingZerosI;
507 case T_LONG: return Op_CountLeadingZerosL;
508 default: fatal("LZ_COUNT: %s", type2name(bt));
509 }
510 break;
511 }
512 case VECTOR_OP_REVERSE: {
513 switch (bt) {
514 case T_BYTE: // Temporarily returning
515 case T_SHORT: // Op_ReverseI for byte and short
516 case T_INT: return Op_ReverseI;
517 case T_LONG: return Op_ReverseL;
518 default: fatal("REVERSE: %s", type2name(bt));
519 }
520 break;
521 }
522 case VECTOR_OP_REVERSE_BYTES: {
523 switch (bt) {
524 case T_BYTE:
525 case T_SHORT:
526 case T_INT: return Op_ReverseBytesI;
527 case T_LONG: return Op_ReverseBytesL;
528 default: fatal("REVERSE_BYTES: %s", type2name(bt));
529 }
530 break;
531 }
532 case VECTOR_OP_COMPRESS_BITS: {
533 switch (bt) {
534 case T_INT:
535 case T_LONG: return Op_CompressBits;
536 default: fatal("COMPRESS_BITS: %s", type2name(bt));
537 }
538 break;
539 }
540 case VECTOR_OP_EXPAND_BITS: {
541 switch (bt) {
542 case T_INT:
543 case T_LONG: return Op_ExpandBits;
544 default: fatal("EXPAND_BITS: %s", type2name(bt));
545 }
546 break;
547 }
548
549 case VECTOR_OP_TAN:
550 case VECTOR_OP_TANH:
551 case VECTOR_OP_SIN:
552 case VECTOR_OP_SINH:
553 case VECTOR_OP_COS:
554 case VECTOR_OP_COSH:
555 case VECTOR_OP_ASIN:
556 case VECTOR_OP_ACOS:
557 case VECTOR_OP_ATAN:
558 case VECTOR_OP_ATAN2:
559 case VECTOR_OP_CBRT:
560 case VECTOR_OP_LOG:
561 case VECTOR_OP_LOG10:
562 case VECTOR_OP_LOG1P:
563 case VECTOR_OP_POW:
564 case VECTOR_OP_EXP:
565 case VECTOR_OP_EXPM1:
566 case VECTOR_OP_HYPOT:
567 return Op_CallLeafVector;
568 default: fatal("unknown op: %d", vop);
569 }
570 return 0; // Unimplemented
571 }
572 #endif // COMPILER2
573
574 /**
575 * Implementation of the jdk.internal.vm.vector.VectorSupport class
576 */
577
578 JVM_ENTRY(jint, VectorSupport_GetMaxLaneCount(JNIEnv *env, jclass vsclazz, jobject clazz)) {
579 #ifdef COMPILER2
580 oop mirror = JNIHandles::resolve_non_null(clazz);
581 if (java_lang_Class::is_primitive(mirror)) {
582 BasicType bt = java_lang_Class::primitive_type(mirror);
583 return Matcher::max_vector_size(bt);
584 }
585 #endif // COMPILER2
586 return -1;
587 } JVM_END
588
589 // JVM_RegisterVectorSupportMethods
590
591 #define LANG "Ljava/lang/"
592 #define CLS LANG "Class;"
593
594 #define CC (char*) /*cast a literal from (const char*)*/
595 #define FN_PTR(f) CAST_FROM_FN_PTR(void*, &f)
596
597 static JNINativeMethod jdk_internal_vm_vector_VectorSupport_methods[] = {
598 {CC "getMaxLaneCount", CC "(" CLS ")I", FN_PTR(VectorSupport_GetMaxLaneCount)}
599 };
600
601 #undef CC
602 #undef FN_PTR
603
604 #undef LANG
605 #undef CLS
606
607 // This function is exported, used by NativeLookup.
608
609 JVM_ENTRY(void, JVM_RegisterVectorSupportMethods(JNIEnv* env, jclass vsclass)) {
610 ThreadToNativeFromVM ttnfv(thread);
611
612 int ok = env->RegisterNatives(vsclass, jdk_internal_vm_vector_VectorSupport_methods, sizeof(jdk_internal_vm_vector_VectorSupport_methods)/sizeof(JNINativeMethod));
613 guarantee(ok == 0, "register jdk.internal.vm.vector.VectorSupport natives");
614 } JVM_END
--- EOF ---