1 /*
2 * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "classfile/vmClasses.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "code/codeCache.hpp"
28 #include "code/compiledIC.hpp"
29 #include "code/nmethod.hpp"
30 #include "code/pcDesc.hpp"
31 #include "code/scopeDesc.hpp"
32 #include "code/vtableStubs.hpp"
33 #include "compiler/compilationMemoryStatistic.hpp"
34 #include "compiler/compileBroker.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/g1/g1HeapRegion.hpp"
37 #include "gc/shared/barrierSet.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gcLocker.hpp"
40 #include "interpreter/bytecode.hpp"
41 #include "interpreter/interpreter.hpp"
42 #include "interpreter/linkResolver.hpp"
43 #include "logging/log.hpp"
44 #include "logging/logStream.hpp"
45 #include "memory/oopFactory.hpp"
46 #include "memory/resourceArea.hpp"
47 #include "oops/flatArrayKlass.hpp"
48 #include "oops/flatArrayOop.inline.hpp"
49 #include "oops/objArrayKlass.hpp"
50 #include "oops/klass.inline.hpp"
51 #include "oops/oop.inline.hpp"
52 #include "oops/typeArrayOop.inline.hpp"
53 #include "opto/ad.hpp"
54 #include "opto/addnode.hpp"
55 #include "opto/callnode.hpp"
56 #include "opto/cfgnode.hpp"
57 #include "opto/graphKit.hpp"
58 #include "opto/machnode.hpp"
59 #include "opto/matcher.hpp"
60 #include "opto/memnode.hpp"
61 #include "opto/mulnode.hpp"
62 #include "opto/output.hpp"
63 #include "opto/runtime.hpp"
64 #include "opto/subnode.hpp"
65 #include "prims/jvmtiExport.hpp"
66 #include "runtime/atomic.hpp"
67 #include "runtime/frame.inline.hpp"
68 #include "runtime/handles.inline.hpp"
69 #include "runtime/interfaceSupport.inline.hpp"
70 #include "runtime/javaCalls.hpp"
71 #include "runtime/sharedRuntime.hpp"
72 #include "runtime/signature.hpp"
73 #include "runtime/stackWatermarkSet.hpp"
74 #include "runtime/synchronizer.hpp"
75 #include "runtime/threadCritical.hpp"
76 #include "runtime/threadWXSetters.inline.hpp"
77 #include "runtime/vframe.hpp"
78 #include "runtime/vframeArray.hpp"
79 #include "runtime/vframe_hp.hpp"
80 #include "utilities/copy.hpp"
81 #include "utilities/preserveException.hpp"
82
83
84 // For debugging purposes:
85 // To force FullGCALot inside a runtime function, add the following two lines
86 //
87 // Universe::release_fullgc_alot_dummy();
88 // Universe::heap()->collect();
89 //
90 // At command line specify the parameters: -XX:+FullGCALot -XX:FullGCALotStart=100000000
91
92
93 #define C2_BLOB_FIELD_DEFINE(name, type) \
94 type OptoRuntime:: BLOB_FIELD_NAME(name) = nullptr;
95 #define C2_STUB_FIELD_NAME(name) _ ## name ## _Java
96 #define C2_STUB_FIELD_DEFINE(name, f, t, r) \
97 address OptoRuntime:: C2_STUB_FIELD_NAME(name) = nullptr;
98 #define C2_JVMTI_STUB_FIELD_DEFINE(name) \
99 address OptoRuntime:: STUB_FIELD_NAME(name) = nullptr;
100 C2_STUBS_DO(C2_BLOB_FIELD_DEFINE, C2_STUB_FIELD_DEFINE, C2_JVMTI_STUB_FIELD_DEFINE)
101 #undef C2_BLOB_FIELD_DEFINE
102 #undef C2_STUB_FIELD_DEFINE
103 #undef C2_JVMTI_STUB_FIELD_DEFINE
104
105 #define C2_BLOB_NAME_DEFINE(name, type) "C2 Runtime " # name "_blob",
106 #define C2_STUB_NAME_DEFINE(name, f, t, r) "C2 Runtime " # name,
107 #define C2_JVMTI_STUB_NAME_DEFINE(name) "C2 Runtime " # name,
108 const char* OptoRuntime::_stub_names[] = {
109 C2_STUBS_DO(C2_BLOB_NAME_DEFINE, C2_STUB_NAME_DEFINE, C2_JVMTI_STUB_NAME_DEFINE)
110 };
111 #undef C2_BLOB_NAME_DEFINE
112 #undef C2_STUB_NAME_DEFINE
113 #undef C2_JVMTI_STUB_NAME_DEFINE
114
115 // This should be called in an assertion at the start of OptoRuntime routines
116 // which are entered from compiled code (all of them)
117 #ifdef ASSERT
118 static bool check_compiled_frame(JavaThread* thread) {
119 assert(thread->last_frame().is_runtime_frame(), "cannot call runtime directly from compiled code");
120 RegisterMap map(thread,
121 RegisterMap::UpdateMap::skip,
122 RegisterMap::ProcessFrames::include,
123 RegisterMap::WalkContinuation::skip);
124 frame caller = thread->last_frame().sender(&map);
125 assert(caller.is_compiled_frame(), "not being called from compiled like code");
126 return true;
127 }
128 #endif // ASSERT
129
130 /*
131 #define gen(env, var, type_func_gen, c_func, fancy_jump, pass_tls, return_pc) \
132 var = generate_stub(env, type_func_gen, CAST_FROM_FN_PTR(address, c_func), #var, fancy_jump, pass_tls, return_pc); \
133 if (var == nullptr) { return false; }
134 */
135
136 #define GEN_C2_BLOB(name, type) \
137 BLOB_FIELD_NAME(name) = \
138 generate_ ## name ## _blob(); \
139 if (BLOB_FIELD_NAME(name) == nullptr) { return false; }
140
141 // a few helper macros to conjure up generate_stub call arguments
142 #define C2_STUB_FIELD_NAME(name) _ ## name ## _Java
143 #define C2_STUB_TYPEFUNC(name) name ## _Type
144 #define C2_STUB_C_FUNC(name) CAST_FROM_FN_PTR(address, name ## _C)
145 #define C2_STUB_NAME(name) stub_name(OptoStubId::name ## _id)
146
147 // Almost all the C functions targeted from the generated stubs are
148 // implemented locally to OptoRuntime with names that can be generated
149 // from the stub name by appending suffix '_C'. However, in two cases
150 // a common target method also needs to be called from shared runtime
151 // stubs. In these two cases the opto stubs rely on method
152 // imlementations defined in class SharedRuntime. The following
153 // defines temporarily rebind the generated names to reference the
154 // relevant implementations.
155
156 #define GEN_C2_STUB(name, fancy_jump, pass_tls, pass_retpc ) \
157 C2_STUB_FIELD_NAME(name) = \
158 generate_stub(env, \
159 C2_STUB_TYPEFUNC(name), \
160 C2_STUB_C_FUNC(name), \
161 C2_STUB_NAME(name), \
162 fancy_jump, \
163 pass_tls, \
164 pass_retpc); \
165 if (C2_STUB_FIELD_NAME(name) == nullptr) { return false; } \
166
167 #define C2_JVMTI_STUB_C_FUNC(name) CAST_FROM_FN_PTR(address, SharedRuntime::name)
168
169 #define GEN_C2_JVMTI_STUB(name) \
170 STUB_FIELD_NAME(name) = \
171 generate_stub(env, \
172 notify_jvmti_vthread_Type, \
173 C2_JVMTI_STUB_C_FUNC(name), \
174 C2_STUB_NAME(name), \
175 0, \
176 true, \
177 false); \
178 if (STUB_FIELD_NAME(name) == nullptr) { return false; } \
179
180 bool OptoRuntime::generate(ciEnv* env) {
181
182 C2_STUBS_DO(GEN_C2_BLOB, GEN_C2_STUB, GEN_C2_JVMTI_STUB)
183
184 return true;
185 }
186
187 #undef GEN_C2_BLOB
188
189 #undef C2_STUB_FIELD_NAME
190 #undef C2_STUB_TYPEFUNC
191 #undef C2_STUB_C_FUNC
192 #undef C2_STUB_NAME
193 #undef GEN_C2_STUB
194
195 #undef C2_JVMTI_STUB_C_FUNC
196 #undef GEN_C2_JVMTI_STUB
197 // #undef gen
198
199 const TypeFunc* OptoRuntime::_new_instance_Type = nullptr;
200 const TypeFunc* OptoRuntime::_new_array_Type = nullptr;
201 const TypeFunc* OptoRuntime::_new_array_nozero_Type = nullptr;
202 const TypeFunc* OptoRuntime::_multianewarray2_Type = nullptr;
203 const TypeFunc* OptoRuntime::_multianewarray3_Type = nullptr;
204 const TypeFunc* OptoRuntime::_multianewarray4_Type = nullptr;
205 const TypeFunc* OptoRuntime::_multianewarray5_Type = nullptr;
206 const TypeFunc* OptoRuntime::_multianewarrayN_Type = nullptr;
207 const TypeFunc* OptoRuntime::_complete_monitor_enter_Type = nullptr;
208 const TypeFunc* OptoRuntime::_complete_monitor_exit_Type = nullptr;
209 const TypeFunc* OptoRuntime::_monitor_notify_Type = nullptr;
210 const TypeFunc* OptoRuntime::_uncommon_trap_Type = nullptr;
211 const TypeFunc* OptoRuntime::_athrow_Type = nullptr;
212 const TypeFunc* OptoRuntime::_rethrow_Type = nullptr;
213 const TypeFunc* OptoRuntime::_Math_D_D_Type = nullptr;
214 const TypeFunc* OptoRuntime::_Math_DD_D_Type = nullptr;
215 const TypeFunc* OptoRuntime::_modf_Type = nullptr;
216 const TypeFunc* OptoRuntime::_l2f_Type = nullptr;
217 const TypeFunc* OptoRuntime::_void_long_Type = nullptr;
218 const TypeFunc* OptoRuntime::_void_void_Type = nullptr;
219 const TypeFunc* OptoRuntime::_jfr_write_checkpoint_Type = nullptr;
220 const TypeFunc* OptoRuntime::_flush_windows_Type = nullptr;
221 const TypeFunc* OptoRuntime::_fast_arraycopy_Type = nullptr;
222 const TypeFunc* OptoRuntime::_checkcast_arraycopy_Type = nullptr;
223 const TypeFunc* OptoRuntime::_generic_arraycopy_Type = nullptr;
224 const TypeFunc* OptoRuntime::_slow_arraycopy_Type = nullptr;
225 const TypeFunc* OptoRuntime::_unsafe_setmemory_Type = nullptr;
226 const TypeFunc* OptoRuntime::_array_fill_Type = nullptr;
227 const TypeFunc* OptoRuntime::_array_sort_Type = nullptr;
228 const TypeFunc* OptoRuntime::_array_partition_Type = nullptr;
229 const TypeFunc* OptoRuntime::_aescrypt_block_Type = nullptr;
230 const TypeFunc* OptoRuntime::_cipherBlockChaining_aescrypt_Type = nullptr;
231 const TypeFunc* OptoRuntime::_electronicCodeBook_aescrypt_Type = nullptr;
232 const TypeFunc* OptoRuntime::_counterMode_aescrypt_Type = nullptr;
233 const TypeFunc* OptoRuntime::_galoisCounterMode_aescrypt_Type = nullptr;
234 const TypeFunc* OptoRuntime::_digestBase_implCompress_with_sha3_Type = nullptr;
235 const TypeFunc* OptoRuntime::_digestBase_implCompress_without_sha3_Type = nullptr;
236 const TypeFunc* OptoRuntime::_digestBase_implCompressMB_with_sha3_Type = nullptr;
237 const TypeFunc* OptoRuntime::_digestBase_implCompressMB_without_sha3_Type = nullptr;
238 const TypeFunc* OptoRuntime::_double_keccak_Type = nullptr;
239 const TypeFunc* OptoRuntime::_multiplyToLen_Type = nullptr;
240 const TypeFunc* OptoRuntime::_montgomeryMultiply_Type = nullptr;
241 const TypeFunc* OptoRuntime::_montgomerySquare_Type = nullptr;
242 const TypeFunc* OptoRuntime::_squareToLen_Type = nullptr;
243 const TypeFunc* OptoRuntime::_mulAdd_Type = nullptr;
244 const TypeFunc* OptoRuntime::_bigIntegerShift_Type = nullptr;
245 const TypeFunc* OptoRuntime::_vectorizedMismatch_Type = nullptr;
246 const TypeFunc* OptoRuntime::_ghash_processBlocks_Type = nullptr;
247 const TypeFunc* OptoRuntime::_chacha20Block_Type = nullptr;
248 const TypeFunc* OptoRuntime::_kyberNtt_Type = nullptr;
249 const TypeFunc* OptoRuntime::_kyberInverseNtt_Type = nullptr;
250 const TypeFunc* OptoRuntime::_kyberNttMult_Type = nullptr;
251 const TypeFunc* OptoRuntime::_kyberAddPoly_2_Type = nullptr;
252 const TypeFunc* OptoRuntime::_kyberAddPoly_3_Type = nullptr;
253 const TypeFunc* OptoRuntime::_kyber12To16_Type = nullptr;
254 const TypeFunc* OptoRuntime::_kyberBarrettReduce_Type = nullptr;
255 const TypeFunc* OptoRuntime::_dilithiumAlmostNtt_Type = nullptr;
256 const TypeFunc* OptoRuntime::_dilithiumAlmostInverseNtt_Type = nullptr;
257 const TypeFunc* OptoRuntime::_dilithiumNttMult_Type = nullptr;
258 const TypeFunc* OptoRuntime::_dilithiumMontMulByConstant_Type = nullptr;
259 const TypeFunc* OptoRuntime::_dilithiumDecomposePoly_Type = nullptr;
260 const TypeFunc* OptoRuntime::_base64_encodeBlock_Type = nullptr;
261 const TypeFunc* OptoRuntime::_base64_decodeBlock_Type = nullptr;
262 const TypeFunc* OptoRuntime::_string_IndexOf_Type = nullptr;
263 const TypeFunc* OptoRuntime::_poly1305_processBlocks_Type = nullptr;
264 const TypeFunc* OptoRuntime::_intpoly_montgomeryMult_P256_Type = nullptr;
265 const TypeFunc* OptoRuntime::_intpoly_assign_Type = nullptr;
266 const TypeFunc* OptoRuntime::_updateBytesCRC32_Type = nullptr;
267 const TypeFunc* OptoRuntime::_updateBytesCRC32C_Type = nullptr;
268 const TypeFunc* OptoRuntime::_updateBytesAdler32_Type = nullptr;
269 const TypeFunc* OptoRuntime::_osr_end_Type = nullptr;
270 const TypeFunc* OptoRuntime::_register_finalizer_Type = nullptr;
271 #if INCLUDE_JFR
272 const TypeFunc* OptoRuntime::_class_id_load_barrier_Type = nullptr;
273 #endif // INCLUDE_JFR
274 #if INCLUDE_JVMTI
275 const TypeFunc* OptoRuntime::_notify_jvmti_vthread_Type = nullptr;
276 #endif // INCLUDE_JVMTI
277 const TypeFunc* OptoRuntime::_dtrace_method_entry_exit_Type = nullptr;
278 const TypeFunc* OptoRuntime::_dtrace_object_alloc_Type = nullptr;
279
280 // Helper method to do generation of RunTimeStub's
281 address OptoRuntime::generate_stub(ciEnv* env,
282 TypeFunc_generator gen, address C_function,
283 const char *name, int is_fancy_jump,
284 bool pass_tls,
285 bool return_pc) {
286
287 // Matching the default directive, we currently have no method to match.
288 DirectiveSet* directive = DirectivesStack::getDefaultDirective(CompileBroker::compiler(CompLevel_full_optimization));
289 CompilationMemoryStatisticMark cmsm(directive);
290 ResourceMark rm;
291 Compile C(env, gen, C_function, name, is_fancy_jump, pass_tls, return_pc, directive);
292 DirectivesStack::release(directive);
293 return C.stub_entry_point();
294 }
295
296 const char* OptoRuntime::stub_name(address entry) {
297 #ifndef PRODUCT
298 CodeBlob* cb = CodeCache::find_blob(entry);
299 RuntimeStub* rs =(RuntimeStub *)cb;
300 assert(rs != nullptr && rs->is_runtime_stub(), "not a runtime stub");
301 return rs->name();
302 #else
303 // Fast implementation for product mode (maybe it should be inlined too)
304 return "runtime stub";
305 #endif
306 }
307
308 // local methods passed as arguments to stub generator that forward
309 // control to corresponding JRT methods of SharedRuntime
310
311 void OptoRuntime::slow_arraycopy_C(oopDesc* src, jint src_pos,
312 oopDesc* dest, jint dest_pos,
313 jint length, JavaThread* thread) {
314 SharedRuntime::slow_arraycopy_C(src, src_pos, dest, dest_pos, length, thread);
315 }
316
317 void OptoRuntime::complete_monitor_locking_C(oopDesc* obj, BasicLock* lock, JavaThread* current) {
318 SharedRuntime::complete_monitor_locking_C(obj, lock, current);
319 }
320
321
322 //=============================================================================
323 // Opto compiler runtime routines
324 //=============================================================================
325
326
327 //=============================allocation======================================
328 // We failed the fast-path allocation. Now we need to do a scavenge or GC
329 // and try allocation again.
330
331 // object allocation
332 JRT_BLOCK_ENTRY(void, OptoRuntime::new_instance_C(Klass* klass, bool is_larval, JavaThread* current))
333 JRT_BLOCK;
334 #ifndef PRODUCT
335 SharedRuntime::_new_instance_ctr++; // new instance requires GC
336 #endif
337 assert(check_compiled_frame(current), "incorrect caller");
338
339 // These checks are cheap to make and support reflective allocation.
340 int lh = klass->layout_helper();
341 if (Klass::layout_helper_needs_slow_path(lh) || !InstanceKlass::cast(klass)->is_initialized()) {
342 Handle holder(current, klass->klass_holder()); // keep the klass alive
343 klass->check_valid_for_instantiation(false, THREAD);
344 if (!HAS_PENDING_EXCEPTION) {
345 InstanceKlass::cast(klass)->initialize(THREAD);
346 }
347 }
348
349 if (!HAS_PENDING_EXCEPTION) {
350 // Scavenge and allocate an instance.
351 Handle holder(current, klass->klass_holder()); // keep the klass alive
352 instanceOop result = InstanceKlass::cast(klass)->allocate_instance(THREAD);
353 if (is_larval) {
354 // Check if this is a larval buffer allocation
355 result->set_mark(result->mark().enter_larval_state());
356 }
357 current->set_vm_result_oop(result);
358
359 // Pass oops back through thread local storage. Our apparent type to Java
360 // is that we return an oop, but we can block on exit from this routine and
361 // a GC can trash the oop in C's return register. The generated stub will
362 // fetch the oop from TLS after any possible GC.
363 }
364
365 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
366 JRT_BLOCK_END;
367
368 // inform GC that we won't do card marks for initializing writes.
369 SharedRuntime::on_slowpath_allocation_exit(current);
370 JRT_END
371
372
373 // array allocation
374 JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_C(Klass* array_type, int len, oopDesc* init_val, JavaThread* current))
375 JRT_BLOCK;
376 #ifndef PRODUCT
377 SharedRuntime::_new_array_ctr++; // new array requires GC
378 #endif
379 assert(check_compiled_frame(current), "incorrect caller");
380
381 // Scavenge and allocate an instance.
382 oop result;
383 Handle h_init_val(current, init_val); // keep the init_val object alive
384
385 if (array_type->is_flatArray_klass()) {
386 Handle holder(current, array_type->klass_holder()); // keep the array klass alive
387 FlatArrayKlass* fak = FlatArrayKlass::cast(array_type);
388 InlineKlass* vk = fak->element_klass();
389 result = oopFactory::new_flatArray(vk, len, fak->layout_kind(), THREAD);
390 if (array_type->is_null_free_array_klass() && !h_init_val.is_null()) {
391 // Null-free arrays need to be initialized
392 for (int i = 0; i < len; i++) {
393 vk->write_value_to_addr(h_init_val(), ((flatArrayOop)result)->value_at_addr(i, fak->layout_helper()), fak->layout_kind(), true, CHECK);
394 }
395 }
396 } else if (array_type->is_typeArray_klass()) {
397 // The oopFactory likes to work with the element type.
398 // (We could bypass the oopFactory, since it doesn't add much value.)
399 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
400 result = oopFactory::new_typeArray(elem_type, len, THREAD);
401 } else {
402 Handle holder(current, array_type->klass_holder()); // keep the array klass alive
403 ObjArrayKlass* array_klass = ObjArrayKlass::cast(array_type);
404 result = array_klass->allocate(len, THREAD);
405 if (array_type->is_null_free_array_klass() && !h_init_val.is_null()) {
406 // Null-free arrays need to be initialized
407 for (int i = 0; i < len; i++) {
408 ((objArrayOop)result)->obj_at_put(i, h_init_val());
409 }
410 }
411 }
412
413 // Pass oops back through thread local storage. Our apparent type to Java
414 // is that we return an oop, but we can block on exit from this routine and
415 // a GC can trash the oop in C's return register. The generated stub will
416 // fetch the oop from TLS after any possible GC.
417 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
418 current->set_vm_result_oop(result);
419 JRT_BLOCK_END;
420
421 // inform GC that we won't do card marks for initializing writes.
422 SharedRuntime::on_slowpath_allocation_exit(current);
423 JRT_END
424
425 // array allocation without zeroing
426 JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(Klass* array_type, int len, JavaThread* current))
427 JRT_BLOCK;
428 #ifndef PRODUCT
429 SharedRuntime::_new_array_ctr++; // new array requires GC
430 #endif
431 assert(check_compiled_frame(current), "incorrect caller");
432
433 // Scavenge and allocate an instance.
434 oop result;
435
436 assert(array_type->is_typeArray_klass(), "should be called only for type array");
437 // The oopFactory likes to work with the element type.
438 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
439 result = oopFactory::new_typeArray_nozero(elem_type, len, THREAD);
440
441 // Pass oops back through thread local storage. Our apparent type to Java
442 // is that we return an oop, but we can block on exit from this routine and
443 // a GC can trash the oop in C's return register. The generated stub will
444 // fetch the oop from TLS after any possible GC.
445 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
446 current->set_vm_result_oop(result);
447 JRT_BLOCK_END;
448
449
450 // inform GC that we won't do card marks for initializing writes.
451 SharedRuntime::on_slowpath_allocation_exit(current);
452
453 oop result = current->vm_result_oop();
454 if ((len > 0) && (result != nullptr) &&
455 is_deoptimized_caller_frame(current)) {
456 // Zero array here if the caller is deoptimized.
457 const size_t size = TypeArrayKlass::cast(array_type)->oop_size(result);
458 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
459 size_t hs_bytes = arrayOopDesc::base_offset_in_bytes(elem_type);
460 assert(is_aligned(hs_bytes, BytesPerInt), "must be 4 byte aligned");
461 HeapWord* obj = cast_from_oop<HeapWord*>(result);
462 if (!is_aligned(hs_bytes, BytesPerLong)) {
463 *reinterpret_cast<jint*>(reinterpret_cast<char*>(obj) + hs_bytes) = 0;
464 hs_bytes += BytesPerInt;
465 }
466
467 // Optimized zeroing.
468 assert(is_aligned(hs_bytes, BytesPerLong), "must be 8-byte aligned");
469 const size_t aligned_hs = hs_bytes / BytesPerLong;
470 Copy::fill_to_aligned_words(obj+aligned_hs, size-aligned_hs);
471 }
472
473 JRT_END
474
475 // Note: multianewarray for one dimension is handled inline by GraphKit::new_array.
476
477 // multianewarray for 2 dimensions
478 JRT_ENTRY(void, OptoRuntime::multianewarray2_C(Klass* elem_type, int len1, int len2, JavaThread* current))
479 #ifndef PRODUCT
480 SharedRuntime::_multi2_ctr++; // multianewarray for 1 dimension
481 #endif
482 assert(check_compiled_frame(current), "incorrect caller");
483 assert(elem_type->is_klass(), "not a class");
484 jint dims[2];
485 dims[0] = len1;
486 dims[1] = len2;
487 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
488 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(2, dims, THREAD);
489 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
490 current->set_vm_result_oop(obj);
491 JRT_END
492
493 // multianewarray for 3 dimensions
494 JRT_ENTRY(void, OptoRuntime::multianewarray3_C(Klass* elem_type, int len1, int len2, int len3, JavaThread* current))
495 #ifndef PRODUCT
496 SharedRuntime::_multi3_ctr++; // multianewarray for 1 dimension
497 #endif
498 assert(check_compiled_frame(current), "incorrect caller");
499 assert(elem_type->is_klass(), "not a class");
500 jint dims[3];
501 dims[0] = len1;
502 dims[1] = len2;
503 dims[2] = len3;
504 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
505 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(3, dims, THREAD);
506 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
507 current->set_vm_result_oop(obj);
508 JRT_END
509
510 // multianewarray for 4 dimensions
511 JRT_ENTRY(void, OptoRuntime::multianewarray4_C(Klass* elem_type, int len1, int len2, int len3, int len4, JavaThread* current))
512 #ifndef PRODUCT
513 SharedRuntime::_multi4_ctr++; // multianewarray for 1 dimension
514 #endif
515 assert(check_compiled_frame(current), "incorrect caller");
516 assert(elem_type->is_klass(), "not a class");
517 jint dims[4];
518 dims[0] = len1;
519 dims[1] = len2;
520 dims[2] = len3;
521 dims[3] = len4;
522 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
523 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(4, dims, THREAD);
524 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
525 current->set_vm_result_oop(obj);
526 JRT_END
527
528 // multianewarray for 5 dimensions
529 JRT_ENTRY(void, OptoRuntime::multianewarray5_C(Klass* elem_type, int len1, int len2, int len3, int len4, int len5, JavaThread* current))
530 #ifndef PRODUCT
531 SharedRuntime::_multi5_ctr++; // multianewarray for 1 dimension
532 #endif
533 assert(check_compiled_frame(current), "incorrect caller");
534 assert(elem_type->is_klass(), "not a class");
535 jint dims[5];
536 dims[0] = len1;
537 dims[1] = len2;
538 dims[2] = len3;
539 dims[3] = len4;
540 dims[4] = len5;
541 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
542 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(5, dims, THREAD);
543 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
544 current->set_vm_result_oop(obj);
545 JRT_END
546
547 JRT_ENTRY(void, OptoRuntime::multianewarrayN_C(Klass* elem_type, arrayOopDesc* dims, JavaThread* current))
548 assert(check_compiled_frame(current), "incorrect caller");
549 assert(elem_type->is_klass(), "not a class");
550 assert(oop(dims)->is_typeArray(), "not an array");
551
552 ResourceMark rm;
553 jint len = dims->length();
554 assert(len > 0, "Dimensions array should contain data");
555 jint *c_dims = NEW_RESOURCE_ARRAY(jint, len);
556 ArrayAccess<>::arraycopy_to_native<>(dims, typeArrayOopDesc::element_offset<jint>(0),
557 c_dims, len);
558
559 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
560 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(len, c_dims, THREAD);
561 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
562 current->set_vm_result_oop(obj);
563 JRT_END
564
565 JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notify_C(oopDesc* obj, JavaThread* current))
566
567 // Very few notify/notifyAll operations find any threads on the waitset, so
568 // the dominant fast-path is to simply return.
569 // Relatedly, it's critical that notify/notifyAll be fast in order to
570 // reduce lock hold times.
571 if (!SafepointSynchronize::is_synchronizing()) {
572 if (ObjectSynchronizer::quick_notify(obj, current, false)) {
573 return;
574 }
575 }
576
577 // This is the case the fast-path above isn't provisioned to handle.
578 // The fast-path is designed to handle frequently arising cases in an efficient manner.
579 // (The fast-path is just a degenerate variant of the slow-path).
580 // Perform the dreaded state transition and pass control into the slow-path.
581 JRT_BLOCK;
582 Handle h_obj(current, obj);
583 ObjectSynchronizer::notify(h_obj, CHECK);
584 JRT_BLOCK_END;
585 JRT_END
586
587 JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notifyAll_C(oopDesc* obj, JavaThread* current))
588
589 if (!SafepointSynchronize::is_synchronizing() ) {
590 if (ObjectSynchronizer::quick_notify(obj, current, true)) {
591 return;
592 }
593 }
594
595 // This is the case the fast-path above isn't provisioned to handle.
596 // The fast-path is designed to handle frequently arising cases in an efficient manner.
597 // (The fast-path is just a degenerate variant of the slow-path).
598 // Perform the dreaded state transition and pass control into the slow-path.
599 JRT_BLOCK;
600 Handle h_obj(current, obj);
601 ObjectSynchronizer::notifyall(h_obj, CHECK);
602 JRT_BLOCK_END;
603 JRT_END
604
605 static const TypeFunc* make_new_instance_Type() {
606 // create input type (domain)
607 const Type **fields = TypeTuple::fields(2);
608 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Klass to be allocated
609 fields[TypeFunc::Parms+1] = TypeInt::BOOL; // is_larval
610 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
611
612 // create result type (range)
613 fields = TypeTuple::fields(1);
614 fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Returned oop
615
616 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
617
618 return TypeFunc::make(domain, range);
619 }
620
621 #if INCLUDE_JVMTI
622 static const TypeFunc* make_notify_jvmti_vthread_Type() {
623 // create input type (domain)
624 const Type **fields = TypeTuple::fields(2);
625 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // VirtualThread oop
626 fields[TypeFunc::Parms+1] = TypeInt::BOOL; // jboolean
627 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
628
629 // no result type needed
630 fields = TypeTuple::fields(1);
631 fields[TypeFunc::Parms+0] = nullptr; // void
632 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
633
634 return TypeFunc::make(domain,range);
635 }
636 #endif
637
638 static const TypeFunc* make_athrow_Type() {
639 // create input type (domain)
640 const Type **fields = TypeTuple::fields(1);
641 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Klass to be allocated
642 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
643
644 // create result type (range)
645 fields = TypeTuple::fields(0);
646
647 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
648
649 return TypeFunc::make(domain, range);
650 }
651
652 static const TypeFunc* make_new_array_Type() {
653 // create input type (domain)
654 const Type **fields = TypeTuple::fields(3);
655 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // element klass
656 fields[TypeFunc::Parms+1] = TypeInt::INT; // array size
657 fields[TypeFunc::Parms+2] = TypeInstPtr::NOTNULL; // init value
658 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+3, fields);
659
660 // create result type (range)
661 fields = TypeTuple::fields(1);
662 fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Returned oop
663
664 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
665
666 return TypeFunc::make(domain, range);
667 }
668
669 static const TypeFunc* make_new_array_nozero_Type() {
670 // create input type (domain)
671 const Type **fields = TypeTuple::fields(2);
672 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // element klass
673 fields[TypeFunc::Parms+1] = TypeInt::INT; // array size
674 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
675
676 // create result type (range)
677 fields = TypeTuple::fields(1);
678 fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Returned oop
679
680 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
681
682 return TypeFunc::make(domain, range);
683 }
684
685 const TypeFunc* OptoRuntime::multianewarray_Type(int ndim) {
686 // create input type (domain)
687 const int nargs = ndim + 1;
688 const Type **fields = TypeTuple::fields(nargs);
689 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // element klass
690 for( int i = 1; i < nargs; i++ )
691 fields[TypeFunc::Parms + i] = TypeInt::INT; // array size
692 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+nargs, fields);
693
694 // create result type (range)
695 fields = TypeTuple::fields(1);
696 fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Returned oop
697 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
698
699 return TypeFunc::make(domain, range);
700 }
701
702 static const TypeFunc* make_multianewarrayN_Type() {
703 // create input type (domain)
704 const Type **fields = TypeTuple::fields(2);
705 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // element klass
706 fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL; // array of dim sizes
707 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
708
709 // create result type (range)
710 fields = TypeTuple::fields(1);
711 fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Returned oop
712 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
713
714 return TypeFunc::make(domain, range);
715 }
716
717 static const TypeFunc* make_uncommon_trap_Type() {
718 // create input type (domain)
719 const Type **fields = TypeTuple::fields(1);
720 fields[TypeFunc::Parms+0] = TypeInt::INT; // trap_reason (deopt reason and action)
721 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
722
723 // create result type (range)
724 fields = TypeTuple::fields(0);
725 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
726
727 return TypeFunc::make(domain, range);
728 }
729
730 //-----------------------------------------------------------------------------
731 // Monitor Handling
732
733 static const TypeFunc* make_complete_monitor_enter_Type() {
734 // create input type (domain)
735 const Type **fields = TypeTuple::fields(2);
736 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Object to be Locked
737 fields[TypeFunc::Parms+1] = TypeRawPtr::BOTTOM; // Address of stack location for lock
738 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
739
740 // create result type (range)
741 fields = TypeTuple::fields(0);
742
743 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
744
745 return TypeFunc::make(domain, range);
746 }
747
748 //-----------------------------------------------------------------------------
749
750 static const TypeFunc* make_complete_monitor_exit_Type() {
751 // create input type (domain)
752 const Type **fields = TypeTuple::fields(3);
753 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Object to be Locked
754 fields[TypeFunc::Parms+1] = TypeRawPtr::BOTTOM; // Address of stack location for lock - BasicLock
755 fields[TypeFunc::Parms+2] = TypeRawPtr::BOTTOM; // Thread pointer (Self)
756 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+3, fields);
757
758 // create result type (range)
759 fields = TypeTuple::fields(0);
760
761 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
762
763 return TypeFunc::make(domain, range);
764 }
765
766 static const TypeFunc* make_monitor_notify_Type() {
767 // create input type (domain)
768 const Type **fields = TypeTuple::fields(1);
769 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Object to be Locked
770 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
771
772 // create result type (range)
773 fields = TypeTuple::fields(0);
774 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
775 return TypeFunc::make(domain, range);
776 }
777
778 static const TypeFunc* make_flush_windows_Type() {
779 // create input type (domain)
780 const Type** fields = TypeTuple::fields(1);
781 fields[TypeFunc::Parms+0] = nullptr; // void
782 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields);
783
784 // create result type
785 fields = TypeTuple::fields(1);
786 fields[TypeFunc::Parms+0] = nullptr; // void
787 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields);
788
789 return TypeFunc::make(domain, range);
790 }
791
792 static const TypeFunc* make_l2f_Type() {
793 // create input type (domain)
794 const Type **fields = TypeTuple::fields(2);
795 fields[TypeFunc::Parms+0] = TypeLong::LONG;
796 fields[TypeFunc::Parms+1] = Type::HALF;
797 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
798
799 // create result type (range)
800 fields = TypeTuple::fields(1);
801 fields[TypeFunc::Parms+0] = Type::FLOAT;
802 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
803
804 return TypeFunc::make(domain, range);
805 }
806
807 static const TypeFunc* make_modf_Type() {
808 const Type **fields = TypeTuple::fields(2);
809 fields[TypeFunc::Parms+0] = Type::FLOAT;
810 fields[TypeFunc::Parms+1] = Type::FLOAT;
811 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
812
813 // create result type (range)
814 fields = TypeTuple::fields(1);
815 fields[TypeFunc::Parms+0] = Type::FLOAT;
816
817 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
818
819 return TypeFunc::make(domain, range);
820 }
821
822 static const TypeFunc* make_Math_D_D_Type() {
823 // create input type (domain)
824 const Type **fields = TypeTuple::fields(2);
825 // Symbol* name of class to be loaded
826 fields[TypeFunc::Parms+0] = Type::DOUBLE;
827 fields[TypeFunc::Parms+1] = Type::HALF;
828 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
829
830 // create result type (range)
831 fields = TypeTuple::fields(2);
832 fields[TypeFunc::Parms+0] = Type::DOUBLE;
833 fields[TypeFunc::Parms+1] = Type::HALF;
834 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+2, fields);
835
836 return TypeFunc::make(domain, range);
837 }
838
839 const TypeFunc* OptoRuntime::Math_Vector_Vector_Type(uint num_arg, const TypeVect* in_type, const TypeVect* out_type) {
840 // create input type (domain)
841 const Type **fields = TypeTuple::fields(num_arg);
842 // Symbol* name of class to be loaded
843 assert(num_arg > 0, "must have at least 1 input");
844 for (uint i = 0; i < num_arg; i++) {
845 fields[TypeFunc::Parms+i] = in_type;
846 }
847 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+num_arg, fields);
848
849 // create result type (range)
850 const uint num_ret = 1;
851 fields = TypeTuple::fields(num_ret);
852 fields[TypeFunc::Parms+0] = out_type;
853 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+num_ret, fields);
854
855 return TypeFunc::make(domain, range);
856 }
857
858 static const TypeFunc* make_Math_DD_D_Type() {
859 const Type **fields = TypeTuple::fields(4);
860 fields[TypeFunc::Parms+0] = Type::DOUBLE;
861 fields[TypeFunc::Parms+1] = Type::HALF;
862 fields[TypeFunc::Parms+2] = Type::DOUBLE;
863 fields[TypeFunc::Parms+3] = Type::HALF;
864 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+4, fields);
865
866 // create result type (range)
867 fields = TypeTuple::fields(2);
868 fields[TypeFunc::Parms+0] = Type::DOUBLE;
869 fields[TypeFunc::Parms+1] = Type::HALF;
870 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+2, fields);
871
872 return TypeFunc::make(domain, range);
873 }
874
875 //-------------- currentTimeMillis, currentTimeNanos, etc
876
877 static const TypeFunc* make_void_long_Type() {
878 // create input type (domain)
879 const Type **fields = TypeTuple::fields(0);
880 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+0, fields);
881
882 // create result type (range)
883 fields = TypeTuple::fields(2);
884 fields[TypeFunc::Parms+0] = TypeLong::LONG;
885 fields[TypeFunc::Parms+1] = Type::HALF;
886 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+2, fields);
887
888 return TypeFunc::make(domain, range);
889 }
890
891 static const TypeFunc* make_void_void_Type() {
892 // create input type (domain)
893 const Type **fields = TypeTuple::fields(0);
894 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+0, fields);
895
896 // create result type (range)
897 fields = TypeTuple::fields(0);
898 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
899 return TypeFunc::make(domain, range);
900 }
901
902 static const TypeFunc* make_jfr_write_checkpoint_Type() {
903 // create input type (domain)
904 const Type **fields = TypeTuple::fields(0);
905 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields);
906
907 // create result type (range)
908 fields = TypeTuple::fields(0);
909 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields);
910 return TypeFunc::make(domain, range);
911 }
912
913
914 // Takes as parameters:
915 // void *dest
916 // long size
917 // uchar byte
918
919 static const TypeFunc* make_setmemory_Type() {
920 // create input type (domain)
921 int argcnt = NOT_LP64(3) LP64_ONLY(4);
922 const Type** fields = TypeTuple::fields(argcnt);
923 int argp = TypeFunc::Parms;
924 fields[argp++] = TypePtr::NOTNULL; // dest
925 fields[argp++] = TypeX_X; // size
926 LP64_ONLY(fields[argp++] = Type::HALF); // size
927 fields[argp++] = TypeInt::UBYTE; // bytevalue
928 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
929 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
930
931 // no result type needed
932 fields = TypeTuple::fields(1);
933 fields[TypeFunc::Parms+0] = nullptr; // void
934 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
935 return TypeFunc::make(domain, range);
936 }
937
938 // arraycopy stub variations:
939 enum ArrayCopyType {
940 ac_fast, // void(ptr, ptr, size_t)
941 ac_checkcast, // int(ptr, ptr, size_t, size_t, ptr)
942 ac_slow, // void(ptr, int, ptr, int, int)
943 ac_generic // int(ptr, int, ptr, int, int)
944 };
945
946 static const TypeFunc* make_arraycopy_Type(ArrayCopyType act) {
947 // create input type (domain)
948 int num_args = (act == ac_fast ? 3 : 5);
949 int num_size_args = (act == ac_fast ? 1 : act == ac_checkcast ? 2 : 0);
950 int argcnt = num_args;
951 LP64_ONLY(argcnt += num_size_args); // halfwords for lengths
952 const Type** fields = TypeTuple::fields(argcnt);
953 int argp = TypeFunc::Parms;
954 fields[argp++] = TypePtr::NOTNULL; // src
955 if (num_size_args == 0) {
956 fields[argp++] = TypeInt::INT; // src_pos
957 }
958 fields[argp++] = TypePtr::NOTNULL; // dest
959 if (num_size_args == 0) {
960 fields[argp++] = TypeInt::INT; // dest_pos
961 fields[argp++] = TypeInt::INT; // length
962 }
963 while (num_size_args-- > 0) {
964 fields[argp++] = TypeX_X; // size in whatevers (size_t)
965 LP64_ONLY(fields[argp++] = Type::HALF); // other half of long length
966 }
967 if (act == ac_checkcast) {
968 fields[argp++] = TypePtr::NOTNULL; // super_klass
969 }
970 assert(argp == TypeFunc::Parms+argcnt, "correct decoding of act");
971 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
972
973 // create result type if needed
974 int retcnt = (act == ac_checkcast || act == ac_generic ? 1 : 0);
975 fields = TypeTuple::fields(1);
976 if (retcnt == 0)
977 fields[TypeFunc::Parms+0] = nullptr; // void
978 else
979 fields[TypeFunc::Parms+0] = TypeInt::INT; // status result, if needed
980 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+retcnt, fields);
981 return TypeFunc::make(domain, range);
982 }
983
984 static const TypeFunc* make_array_fill_Type() {
985 const Type** fields;
986 int argp = TypeFunc::Parms;
987 // create input type (domain): pointer, int, size_t
988 fields = TypeTuple::fields(3 LP64_ONLY( + 1));
989 fields[argp++] = TypePtr::NOTNULL;
990 fields[argp++] = TypeInt::INT;
991 fields[argp++] = TypeX_X; // size in whatevers (size_t)
992 LP64_ONLY(fields[argp++] = Type::HALF); // other half of long length
993 const TypeTuple *domain = TypeTuple::make(argp, fields);
994
995 // create result type
996 fields = TypeTuple::fields(1);
997 fields[TypeFunc::Parms+0] = nullptr; // void
998 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields);
999
1000 return TypeFunc::make(domain, range);
1001 }
1002
1003 static const TypeFunc* make_array_partition_Type() {
1004 // create input type (domain)
1005 int num_args = 7;
1006 int argcnt = num_args;
1007 const Type** fields = TypeTuple::fields(argcnt);
1008 int argp = TypeFunc::Parms;
1009 fields[argp++] = TypePtr::NOTNULL; // array
1010 fields[argp++] = TypeInt::INT; // element type
1011 fields[argp++] = TypeInt::INT; // low
1012 fields[argp++] = TypeInt::INT; // end
1013 fields[argp++] = TypePtr::NOTNULL; // pivot_indices (int array)
1014 fields[argp++] = TypeInt::INT; // indexPivot1
1015 fields[argp++] = TypeInt::INT; // indexPivot2
1016 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1017 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1018
1019 // no result type needed
1020 fields = TypeTuple::fields(1);
1021 fields[TypeFunc::Parms+0] = nullptr; // void
1022 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1023 return TypeFunc::make(domain, range);
1024 }
1025
1026 static const TypeFunc* make_array_sort_Type() {
1027 // create input type (domain)
1028 int num_args = 4;
1029 int argcnt = num_args;
1030 const Type** fields = TypeTuple::fields(argcnt);
1031 int argp = TypeFunc::Parms;
1032 fields[argp++] = TypePtr::NOTNULL; // array
1033 fields[argp++] = TypeInt::INT; // element type
1034 fields[argp++] = TypeInt::INT; // fromIndex
1035 fields[argp++] = TypeInt::INT; // toIndex
1036 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1037 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1038
1039 // no result type needed
1040 fields = TypeTuple::fields(1);
1041 fields[TypeFunc::Parms+0] = nullptr; // void
1042 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1043 return TypeFunc::make(domain, range);
1044 }
1045
1046 static const TypeFunc* make_aescrypt_block_Type() {
1047 // create input type (domain)
1048 int num_args = 3;
1049 int argcnt = num_args;
1050 const Type** fields = TypeTuple::fields(argcnt);
1051 int argp = TypeFunc::Parms;
1052 fields[argp++] = TypePtr::NOTNULL; // src
1053 fields[argp++] = TypePtr::NOTNULL; // dest
1054 fields[argp++] = TypePtr::NOTNULL; // k array
1055 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1056 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1057
1058 // no result type needed
1059 fields = TypeTuple::fields(1);
1060 fields[TypeFunc::Parms+0] = nullptr; // void
1061 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1062 return TypeFunc::make(domain, range);
1063 }
1064
1065 static const TypeFunc* make_updateBytesCRC32_Type() {
1066 // create input type (domain)
1067 int num_args = 3;
1068 int argcnt = num_args;
1069 const Type** fields = TypeTuple::fields(argcnt);
1070 int argp = TypeFunc::Parms;
1071 fields[argp++] = TypeInt::INT; // crc
1072 fields[argp++] = TypePtr::NOTNULL; // src
1073 fields[argp++] = TypeInt::INT; // len
1074 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1075 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1076
1077 // result type needed
1078 fields = TypeTuple::fields(1);
1079 fields[TypeFunc::Parms+0] = TypeInt::INT; // crc result
1080 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
1081 return TypeFunc::make(domain, range);
1082 }
1083
1084 static const TypeFunc* make_updateBytesCRC32C_Type() {
1085 // create input type (domain)
1086 int num_args = 4;
1087 int argcnt = num_args;
1088 const Type** fields = TypeTuple::fields(argcnt);
1089 int argp = TypeFunc::Parms;
1090 fields[argp++] = TypeInt::INT; // crc
1091 fields[argp++] = TypePtr::NOTNULL; // buf
1092 fields[argp++] = TypeInt::INT; // len
1093 fields[argp++] = TypePtr::NOTNULL; // table
1094 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1095 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1096
1097 // result type needed
1098 fields = TypeTuple::fields(1);
1099 fields[TypeFunc::Parms+0] = TypeInt::INT; // crc result
1100 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
1101 return TypeFunc::make(domain, range);
1102 }
1103
1104 static const TypeFunc* make_updateBytesAdler32_Type() {
1105 // create input type (domain)
1106 int num_args = 3;
1107 int argcnt = num_args;
1108 const Type** fields = TypeTuple::fields(argcnt);
1109 int argp = TypeFunc::Parms;
1110 fields[argp++] = TypeInt::INT; // crc
1111 fields[argp++] = TypePtr::NOTNULL; // src + offset
1112 fields[argp++] = TypeInt::INT; // len
1113 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1114 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1115
1116 // result type needed
1117 fields = TypeTuple::fields(1);
1118 fields[TypeFunc::Parms+0] = TypeInt::INT; // crc result
1119 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
1120 return TypeFunc::make(domain, range);
1121 }
1122
1123 static const TypeFunc* make_cipherBlockChaining_aescrypt_Type() {
1124 // create input type (domain)
1125 int num_args = 5;
1126 int argcnt = num_args;
1127 const Type** fields = TypeTuple::fields(argcnt);
1128 int argp = TypeFunc::Parms;
1129 fields[argp++] = TypePtr::NOTNULL; // src
1130 fields[argp++] = TypePtr::NOTNULL; // dest
1131 fields[argp++] = TypePtr::NOTNULL; // k array
1132 fields[argp++] = TypePtr::NOTNULL; // r array
1133 fields[argp++] = TypeInt::INT; // src len
1134 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1135 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1136
1137 // returning cipher len (int)
1138 fields = TypeTuple::fields(1);
1139 fields[TypeFunc::Parms+0] = TypeInt::INT;
1140 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
1141 return TypeFunc::make(domain, range);
1142 }
1143
1144 static const TypeFunc* make_electronicCodeBook_aescrypt_Type() {
1145 // create input type (domain)
1146 int num_args = 4;
1147 int argcnt = num_args;
1148 const Type** fields = TypeTuple::fields(argcnt);
1149 int argp = TypeFunc::Parms;
1150 fields[argp++] = TypePtr::NOTNULL; // src
1151 fields[argp++] = TypePtr::NOTNULL; // dest
1152 fields[argp++] = TypePtr::NOTNULL; // k array
1153 fields[argp++] = TypeInt::INT; // src len
1154 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1155 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1156
1157 // returning cipher len (int)
1158 fields = TypeTuple::fields(1);
1159 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1160 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1161 return TypeFunc::make(domain, range);
1162 }
1163
1164 static const TypeFunc* make_counterMode_aescrypt_Type() {
1165 // create input type (domain)
1166 int num_args = 7;
1167 int argcnt = num_args;
1168 const Type** fields = TypeTuple::fields(argcnt);
1169 int argp = TypeFunc::Parms;
1170 fields[argp++] = TypePtr::NOTNULL; // src
1171 fields[argp++] = TypePtr::NOTNULL; // dest
1172 fields[argp++] = TypePtr::NOTNULL; // k array
1173 fields[argp++] = TypePtr::NOTNULL; // counter array
1174 fields[argp++] = TypeInt::INT; // src len
1175 fields[argp++] = TypePtr::NOTNULL; // saved_encCounter
1176 fields[argp++] = TypePtr::NOTNULL; // saved used addr
1177 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1178 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1179 // returning cipher len (int)
1180 fields = TypeTuple::fields(1);
1181 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1182 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1183 return TypeFunc::make(domain, range);
1184 }
1185
1186 static const TypeFunc* make_galoisCounterMode_aescrypt_Type() {
1187 // create input type (domain)
1188 int num_args = 8;
1189 int argcnt = num_args;
1190 const Type** fields = TypeTuple::fields(argcnt);
1191 int argp = TypeFunc::Parms;
1192 fields[argp++] = TypePtr::NOTNULL; // byte[] in + inOfs
1193 fields[argp++] = TypeInt::INT; // int len
1194 fields[argp++] = TypePtr::NOTNULL; // byte[] ct + ctOfs
1195 fields[argp++] = TypePtr::NOTNULL; // byte[] out + outOfs
1196 fields[argp++] = TypePtr::NOTNULL; // byte[] key from AESCrypt obj
1197 fields[argp++] = TypePtr::NOTNULL; // long[] state from GHASH obj
1198 fields[argp++] = TypePtr::NOTNULL; // long[] subkeyHtbl from GHASH obj
1199 fields[argp++] = TypePtr::NOTNULL; // byte[] counter from GCTR obj
1200
1201 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1202 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1203 // returning cipher len (int)
1204 fields = TypeTuple::fields(1);
1205 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1206 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1207 return TypeFunc::make(domain, range);
1208 }
1209
1210 static const TypeFunc* make_digestBase_implCompress_Type(bool is_sha3) {
1211 // create input type (domain)
1212 int num_args = is_sha3 ? 3 : 2;
1213 int argcnt = num_args;
1214 const Type** fields = TypeTuple::fields(argcnt);
1215 int argp = TypeFunc::Parms;
1216 fields[argp++] = TypePtr::NOTNULL; // buf
1217 fields[argp++] = TypePtr::NOTNULL; // state
1218 if (is_sha3) fields[argp++] = TypeInt::INT; // block_size
1219 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1220 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1221
1222 // no result type needed
1223 fields = TypeTuple::fields(1);
1224 fields[TypeFunc::Parms+0] = nullptr; // void
1225 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1226 return TypeFunc::make(domain, range);
1227 }
1228
1229 /*
1230 * int implCompressMultiBlock(byte[] b, int ofs, int limit)
1231 */
1232 static const TypeFunc* make_digestBase_implCompressMB_Type(bool is_sha3) {
1233 // create input type (domain)
1234 int num_args = is_sha3 ? 5 : 4;
1235 int argcnt = num_args;
1236 const Type** fields = TypeTuple::fields(argcnt);
1237 int argp = TypeFunc::Parms;
1238 fields[argp++] = TypePtr::NOTNULL; // buf
1239 fields[argp++] = TypePtr::NOTNULL; // state
1240 if (is_sha3) fields[argp++] = TypeInt::INT; // block_size
1241 fields[argp++] = TypeInt::INT; // ofs
1242 fields[argp++] = TypeInt::INT; // limit
1243 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1244 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1245
1246 // returning ofs (int)
1247 fields = TypeTuple::fields(1);
1248 fields[TypeFunc::Parms+0] = TypeInt::INT; // ofs
1249 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
1250 return TypeFunc::make(domain, range);
1251 }
1252
1253 // SHAKE128Parallel doubleKeccak function
1254 static const TypeFunc* make_double_keccak_Type() {
1255 int argcnt = 2;
1256
1257 const Type** fields = TypeTuple::fields(argcnt);
1258 int argp = TypeFunc::Parms;
1259 fields[argp++] = TypePtr::NOTNULL; // status0
1260 fields[argp++] = TypePtr::NOTNULL; // status1
1261
1262 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1263 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1264
1265 // result type needed
1266 fields = TypeTuple::fields(1);
1267 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1268 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1269 return TypeFunc::make(domain, range);
1270 }
1271
1272 static const TypeFunc* make_multiplyToLen_Type() {
1273 // create input type (domain)
1274 int num_args = 5;
1275 int argcnt = num_args;
1276 const Type** fields = TypeTuple::fields(argcnt);
1277 int argp = TypeFunc::Parms;
1278 fields[argp++] = TypePtr::NOTNULL; // x
1279 fields[argp++] = TypeInt::INT; // xlen
1280 fields[argp++] = TypePtr::NOTNULL; // y
1281 fields[argp++] = TypeInt::INT; // ylen
1282 fields[argp++] = TypePtr::NOTNULL; // z
1283 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1284 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1285
1286 // no result type needed
1287 fields = TypeTuple::fields(1);
1288 fields[TypeFunc::Parms+0] = nullptr;
1289 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1290 return TypeFunc::make(domain, range);
1291 }
1292
1293 static const TypeFunc* make_squareToLen_Type() {
1294 // create input type (domain)
1295 int num_args = 4;
1296 int argcnt = num_args;
1297 const Type** fields = TypeTuple::fields(argcnt);
1298 int argp = TypeFunc::Parms;
1299 fields[argp++] = TypePtr::NOTNULL; // x
1300 fields[argp++] = TypeInt::INT; // len
1301 fields[argp++] = TypePtr::NOTNULL; // z
1302 fields[argp++] = TypeInt::INT; // zlen
1303 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1304 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1305
1306 // no result type needed
1307 fields = TypeTuple::fields(1);
1308 fields[TypeFunc::Parms+0] = nullptr;
1309 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1310 return TypeFunc::make(domain, range);
1311 }
1312
1313 static const TypeFunc* make_mulAdd_Type() {
1314 // create input type (domain)
1315 int num_args = 5;
1316 int argcnt = num_args;
1317 const Type** fields = TypeTuple::fields(argcnt);
1318 int argp = TypeFunc::Parms;
1319 fields[argp++] = TypePtr::NOTNULL; // out
1320 fields[argp++] = TypePtr::NOTNULL; // in
1321 fields[argp++] = TypeInt::INT; // offset
1322 fields[argp++] = TypeInt::INT; // len
1323 fields[argp++] = TypeInt::INT; // k
1324 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1325 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1326
1327 // returning carry (int)
1328 fields = TypeTuple::fields(1);
1329 fields[TypeFunc::Parms+0] = TypeInt::INT;
1330 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
1331 return TypeFunc::make(domain, range);
1332 }
1333
1334 static const TypeFunc* make_montgomeryMultiply_Type() {
1335 // create input type (domain)
1336 int num_args = 7;
1337 int argcnt = num_args;
1338 const Type** fields = TypeTuple::fields(argcnt);
1339 int argp = TypeFunc::Parms;
1340 fields[argp++] = TypePtr::NOTNULL; // a
1341 fields[argp++] = TypePtr::NOTNULL; // b
1342 fields[argp++] = TypePtr::NOTNULL; // n
1343 fields[argp++] = TypeInt::INT; // len
1344 fields[argp++] = TypeLong::LONG; // inv
1345 fields[argp++] = Type::HALF;
1346 fields[argp++] = TypePtr::NOTNULL; // result
1347 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1348 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1349
1350 // result type needed
1351 fields = TypeTuple::fields(1);
1352 fields[TypeFunc::Parms+0] = TypePtr::NOTNULL;
1353
1354 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1355 return TypeFunc::make(domain, range);
1356 }
1357
1358 static const TypeFunc* make_montgomerySquare_Type() {
1359 // create input type (domain)
1360 int num_args = 6;
1361 int argcnt = num_args;
1362 const Type** fields = TypeTuple::fields(argcnt);
1363 int argp = TypeFunc::Parms;
1364 fields[argp++] = TypePtr::NOTNULL; // a
1365 fields[argp++] = TypePtr::NOTNULL; // n
1366 fields[argp++] = TypeInt::INT; // len
1367 fields[argp++] = TypeLong::LONG; // inv
1368 fields[argp++] = Type::HALF;
1369 fields[argp++] = TypePtr::NOTNULL; // result
1370 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1371 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1372
1373 // result type needed
1374 fields = TypeTuple::fields(1);
1375 fields[TypeFunc::Parms+0] = TypePtr::NOTNULL;
1376
1377 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1378 return TypeFunc::make(domain, range);
1379 }
1380
1381 static const TypeFunc* make_bigIntegerShift_Type() {
1382 int argcnt = 5;
1383 const Type** fields = TypeTuple::fields(argcnt);
1384 int argp = TypeFunc::Parms;
1385 fields[argp++] = TypePtr::NOTNULL; // newArr
1386 fields[argp++] = TypePtr::NOTNULL; // oldArr
1387 fields[argp++] = TypeInt::INT; // newIdx
1388 fields[argp++] = TypeInt::INT; // shiftCount
1389 fields[argp++] = TypeInt::INT; // numIter
1390 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1391 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1392
1393 // no result type needed
1394 fields = TypeTuple::fields(1);
1395 fields[TypeFunc::Parms + 0] = nullptr;
1396 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1397 return TypeFunc::make(domain, range);
1398 }
1399
1400 static const TypeFunc* make_vectorizedMismatch_Type() {
1401 // create input type (domain)
1402 int num_args = 4;
1403 int argcnt = num_args;
1404 const Type** fields = TypeTuple::fields(argcnt);
1405 int argp = TypeFunc::Parms;
1406 fields[argp++] = TypePtr::NOTNULL; // obja
1407 fields[argp++] = TypePtr::NOTNULL; // objb
1408 fields[argp++] = TypeInt::INT; // length, number of elements
1409 fields[argp++] = TypeInt::INT; // log2scale, element size
1410 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1411 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1412
1413 //return mismatch index (int)
1414 fields = TypeTuple::fields(1);
1415 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1416 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1417 return TypeFunc::make(domain, range);
1418 }
1419
1420 static const TypeFunc* make_ghash_processBlocks_Type() {
1421 int argcnt = 4;
1422
1423 const Type** fields = TypeTuple::fields(argcnt);
1424 int argp = TypeFunc::Parms;
1425 fields[argp++] = TypePtr::NOTNULL; // state
1426 fields[argp++] = TypePtr::NOTNULL; // subkeyH
1427 fields[argp++] = TypePtr::NOTNULL; // data
1428 fields[argp++] = TypeInt::INT; // blocks
1429 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1430 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1431
1432 // result type needed
1433 fields = TypeTuple::fields(1);
1434 fields[TypeFunc::Parms+0] = nullptr; // void
1435 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1436 return TypeFunc::make(domain, range);
1437 }
1438
1439 static const TypeFunc* make_chacha20Block_Type() {
1440 int argcnt = 2;
1441
1442 const Type** fields = TypeTuple::fields(argcnt);
1443 int argp = TypeFunc::Parms;
1444 fields[argp++] = TypePtr::NOTNULL; // state
1445 fields[argp++] = TypePtr::NOTNULL; // result
1446
1447 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1448 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1449
1450 // result type needed
1451 fields = TypeTuple::fields(1);
1452 fields[TypeFunc::Parms + 0] = TypeInt::INT; // key stream outlen as int
1453 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1454 return TypeFunc::make(domain, range);
1455 }
1456
1457 // Kyber NTT function
1458 static const TypeFunc* make_kyberNtt_Type() {
1459 int argcnt = 2;
1460
1461 const Type** fields = TypeTuple::fields(argcnt);
1462 int argp = TypeFunc::Parms;
1463 fields[argp++] = TypePtr::NOTNULL; // coeffs
1464 fields[argp++] = TypePtr::NOTNULL; // NTT zetas
1465
1466 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1467 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1468
1469 // result type needed
1470 fields = TypeTuple::fields(1);
1471 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1472 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1473 return TypeFunc::make(domain, range);
1474 }
1475
1476 // Kyber inverse NTT function
1477 static const TypeFunc* make_kyberInverseNtt_Type() {
1478 int argcnt = 2;
1479
1480 const Type** fields = TypeTuple::fields(argcnt);
1481 int argp = TypeFunc::Parms;
1482 fields[argp++] = TypePtr::NOTNULL; // coeffs
1483 fields[argp++] = TypePtr::NOTNULL; // inverse NTT zetas
1484
1485 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1486 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1487
1488 // result type needed
1489 fields = TypeTuple::fields(1);
1490 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1491 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1492 return TypeFunc::make(domain, range);
1493 }
1494
1495 // Kyber NTT multiply function
1496 static const TypeFunc* make_kyberNttMult_Type() {
1497 int argcnt = 4;
1498
1499 const Type** fields = TypeTuple::fields(argcnt);
1500 int argp = TypeFunc::Parms;
1501 fields[argp++] = TypePtr::NOTNULL; // result
1502 fields[argp++] = TypePtr::NOTNULL; // ntta
1503 fields[argp++] = TypePtr::NOTNULL; // nttb
1504 fields[argp++] = TypePtr::NOTNULL; // NTT multiply zetas
1505
1506 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1507 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1508
1509 // result type needed
1510 fields = TypeTuple::fields(1);
1511 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1512 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1513 return TypeFunc::make(domain, range);
1514 }
1515 // Kyber add 2 polynomials function
1516 static const TypeFunc* make_kyberAddPoly_2_Type() {
1517 int argcnt = 3;
1518
1519 const Type** fields = TypeTuple::fields(argcnt);
1520 int argp = TypeFunc::Parms;
1521 fields[argp++] = TypePtr::NOTNULL; // result
1522 fields[argp++] = TypePtr::NOTNULL; // a
1523 fields[argp++] = TypePtr::NOTNULL; // b
1524
1525 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1526 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1527
1528 // result type needed
1529 fields = TypeTuple::fields(1);
1530 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1531 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1532 return TypeFunc::make(domain, range);
1533 }
1534
1535
1536 // Kyber add 3 polynomials function
1537 static const TypeFunc* make_kyberAddPoly_3_Type() {
1538 int argcnt = 4;
1539
1540 const Type** fields = TypeTuple::fields(argcnt);
1541 int argp = TypeFunc::Parms;
1542 fields[argp++] = TypePtr::NOTNULL; // result
1543 fields[argp++] = TypePtr::NOTNULL; // a
1544 fields[argp++] = TypePtr::NOTNULL; // b
1545 fields[argp++] = TypePtr::NOTNULL; // c
1546
1547 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1548 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1549
1550 // result type needed
1551 fields = TypeTuple::fields(1);
1552 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1553 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1554 return TypeFunc::make(domain, range);
1555 }
1556
1557
1558 // Kyber XOF output parsing into polynomial coefficients candidates
1559 // or decompress(12,...) function
1560 static const TypeFunc* make_kyber12To16_Type() {
1561 int argcnt = 4;
1562
1563 const Type** fields = TypeTuple::fields(argcnt);
1564 int argp = TypeFunc::Parms;
1565 fields[argp++] = TypePtr::NOTNULL; // condensed
1566 fields[argp++] = TypeInt::INT; // condensedOffs
1567 fields[argp++] = TypePtr::NOTNULL; // parsed
1568 fields[argp++] = TypeInt::INT; // parsedLength
1569
1570 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1571 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1572
1573 // result type needed
1574 fields = TypeTuple::fields(1);
1575 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1576 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1577 return TypeFunc::make(domain, range);
1578 }
1579
1580 // Kyber Barrett reduce function
1581 static const TypeFunc* make_kyberBarrettReduce_Type() {
1582 int argcnt = 1;
1583
1584 const Type** fields = TypeTuple::fields(argcnt);
1585 int argp = TypeFunc::Parms;
1586 fields[argp++] = TypePtr::NOTNULL; // coeffs
1587 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1588 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1589
1590 // result type needed
1591 fields = TypeTuple::fields(1);
1592 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1593 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1594 return TypeFunc::make(domain, range);
1595 }
1596
1597 // Dilithium NTT function except for the final "normalization" to |coeff| < Q
1598 static const TypeFunc* make_dilithiumAlmostNtt_Type() {
1599 int argcnt = 2;
1600
1601 const Type** fields = TypeTuple::fields(argcnt);
1602 int argp = TypeFunc::Parms;
1603 fields[argp++] = TypePtr::NOTNULL; // coeffs
1604 fields[argp++] = TypePtr::NOTNULL; // NTT zetas
1605
1606 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1607 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1608
1609 // result type needed
1610 fields = TypeTuple::fields(1);
1611 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1612 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1613 return TypeFunc::make(domain, range);
1614 }
1615
1616 // Dilithium inverse NTT function except the final mod Q division by 2^256
1617 static const TypeFunc* make_dilithiumAlmostInverseNtt_Type() {
1618 int argcnt = 2;
1619
1620 const Type** fields = TypeTuple::fields(argcnt);
1621 int argp = TypeFunc::Parms;
1622 fields[argp++] = TypePtr::NOTNULL; // coeffs
1623 fields[argp++] = TypePtr::NOTNULL; // inverse NTT zetas
1624
1625 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1626 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1627
1628 // result type needed
1629 fields = TypeTuple::fields(1);
1630 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1631 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1632 return TypeFunc::make(domain, range);
1633 }
1634
1635 // Dilithium NTT multiply function
1636 static const TypeFunc* make_dilithiumNttMult_Type() {
1637 int argcnt = 3;
1638
1639 const Type** fields = TypeTuple::fields(argcnt);
1640 int argp = TypeFunc::Parms;
1641 fields[argp++] = TypePtr::NOTNULL; // result
1642 fields[argp++] = TypePtr::NOTNULL; // ntta
1643 fields[argp++] = TypePtr::NOTNULL; // nttb
1644
1645 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1646 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1647
1648 // result type needed
1649 fields = TypeTuple::fields(1);
1650 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1651 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1652 return TypeFunc::make(domain, range);
1653 }
1654
1655 // Dilithium Montgomery multiply a polynome coefficient array by a constant
1656 static const TypeFunc* make_dilithiumMontMulByConstant_Type() {
1657 int argcnt = 2;
1658
1659 const Type** fields = TypeTuple::fields(argcnt);
1660 int argp = TypeFunc::Parms;
1661 fields[argp++] = TypePtr::NOTNULL; // coeffs
1662 fields[argp++] = TypeInt::INT; // constant multiplier
1663
1664 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1665 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1666
1667 // result type needed
1668 fields = TypeTuple::fields(1);
1669 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1670 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1671 return TypeFunc::make(domain, range);
1672 }
1673
1674 // Dilithium decompose polynomial
1675 static const TypeFunc* make_dilithiumDecomposePoly_Type() {
1676 int argcnt = 5;
1677
1678 const Type** fields = TypeTuple::fields(argcnt);
1679 int argp = TypeFunc::Parms;
1680 fields[argp++] = TypePtr::NOTNULL; // input
1681 fields[argp++] = TypePtr::NOTNULL; // lowPart
1682 fields[argp++] = TypePtr::NOTNULL; // highPart
1683 fields[argp++] = TypeInt::INT; // 2 * gamma2
1684 fields[argp++] = TypeInt::INT; // multiplier
1685
1686 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1687 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1688
1689 // result type needed
1690 fields = TypeTuple::fields(1);
1691 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1692 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1693 return TypeFunc::make(domain, range);
1694 }
1695
1696 static const TypeFunc* make_base64_encodeBlock_Type() {
1697 int argcnt = 6;
1698
1699 const Type** fields = TypeTuple::fields(argcnt);
1700 int argp = TypeFunc::Parms;
1701 fields[argp++] = TypePtr::NOTNULL; // src array
1702 fields[argp++] = TypeInt::INT; // offset
1703 fields[argp++] = TypeInt::INT; // length
1704 fields[argp++] = TypePtr::NOTNULL; // dest array
1705 fields[argp++] = TypeInt::INT; // dp
1706 fields[argp++] = TypeInt::BOOL; // isURL
1707 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1708 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1709
1710 // result type needed
1711 fields = TypeTuple::fields(1);
1712 fields[TypeFunc::Parms + 0] = nullptr; // void
1713 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1714 return TypeFunc::make(domain, range);
1715 }
1716
1717 static const TypeFunc* make_string_IndexOf_Type() {
1718 int argcnt = 4;
1719
1720 const Type** fields = TypeTuple::fields(argcnt);
1721 int argp = TypeFunc::Parms;
1722 fields[argp++] = TypePtr::NOTNULL; // haystack array
1723 fields[argp++] = TypeInt::INT; // haystack length
1724 fields[argp++] = TypePtr::NOTNULL; // needle array
1725 fields[argp++] = TypeInt::INT; // needle length
1726 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1727 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1728
1729 // result type needed
1730 fields = TypeTuple::fields(1);
1731 fields[TypeFunc::Parms + 0] = TypeInt::INT; // Index of needle in haystack
1732 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1733 return TypeFunc::make(domain, range);
1734 }
1735
1736 static const TypeFunc* make_base64_decodeBlock_Type() {
1737 int argcnt = 7;
1738
1739 const Type** fields = TypeTuple::fields(argcnt);
1740 int argp = TypeFunc::Parms;
1741 fields[argp++] = TypePtr::NOTNULL; // src array
1742 fields[argp++] = TypeInt::INT; // src offset
1743 fields[argp++] = TypeInt::INT; // src length
1744 fields[argp++] = TypePtr::NOTNULL; // dest array
1745 fields[argp++] = TypeInt::INT; // dest offset
1746 fields[argp++] = TypeInt::BOOL; // isURL
1747 fields[argp++] = TypeInt::BOOL; // isMIME
1748 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1749 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1750
1751 // result type needed
1752 fields = TypeTuple::fields(1);
1753 fields[TypeFunc::Parms + 0] = TypeInt::INT; // count of bytes written to dst
1754 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1755 return TypeFunc::make(domain, range);
1756 }
1757
1758 static const TypeFunc* make_poly1305_processBlocks_Type() {
1759 int argcnt = 4;
1760
1761 const Type** fields = TypeTuple::fields(argcnt);
1762 int argp = TypeFunc::Parms;
1763 fields[argp++] = TypePtr::NOTNULL; // input array
1764 fields[argp++] = TypeInt::INT; // input length
1765 fields[argp++] = TypePtr::NOTNULL; // accumulator array
1766 fields[argp++] = TypePtr::NOTNULL; // r array
1767 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1768 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1769
1770 // result type needed
1771 fields = TypeTuple::fields(1);
1772 fields[TypeFunc::Parms + 0] = nullptr; // void
1773 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1774 return TypeFunc::make(domain, range);
1775 }
1776
1777 static const TypeFunc* make_intpoly_montgomeryMult_P256_Type() {
1778 int argcnt = 3;
1779
1780 const Type** fields = TypeTuple::fields(argcnt);
1781 int argp = TypeFunc::Parms;
1782 fields[argp++] = TypePtr::NOTNULL; // a array
1783 fields[argp++] = TypePtr::NOTNULL; // b array
1784 fields[argp++] = TypePtr::NOTNULL; // r(esult) array
1785 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1786 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1787
1788 // result type needed
1789 fields = TypeTuple::fields(1);
1790 fields[TypeFunc::Parms + 0] = nullptr; // void
1791 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1792 return TypeFunc::make(domain, range);
1793 }
1794
1795 static const TypeFunc* make_intpoly_assign_Type() {
1796 int argcnt = 4;
1797
1798 const Type** fields = TypeTuple::fields(argcnt);
1799 int argp = TypeFunc::Parms;
1800 fields[argp++] = TypeInt::INT; // set flag
1801 fields[argp++] = TypePtr::NOTNULL; // a array (result)
1802 fields[argp++] = TypePtr::NOTNULL; // b array (if set is set)
1803 fields[argp++] = TypeInt::INT; // array length
1804 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1805 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1806
1807 // result type needed
1808 fields = TypeTuple::fields(1);
1809 fields[TypeFunc::Parms + 0] = nullptr; // void
1810 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1811 return TypeFunc::make(domain, range);
1812 }
1813
1814 //------------- Interpreter state for on stack replacement
1815 static const TypeFunc* make_osr_end_Type() {
1816 // create input type (domain)
1817 const Type **fields = TypeTuple::fields(1);
1818 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // OSR temp buf
1819 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
1820
1821 // create result type
1822 fields = TypeTuple::fields(1);
1823 // fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // locked oop
1824 fields[TypeFunc::Parms+0] = nullptr; // void
1825 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields);
1826 return TypeFunc::make(domain, range);
1827 }
1828
1829 //-------------------------------------------------------------------------------------
1830 // register policy
1831
1832 bool OptoRuntime::is_callee_saved_register(MachRegisterNumbers reg) {
1833 assert(reg >= 0 && reg < _last_Mach_Reg, "must be a machine register");
1834 switch (register_save_policy[reg]) {
1835 case 'C': return false; //SOC
1836 case 'E': return true ; //SOE
1837 case 'N': return false; //NS
1838 case 'A': return false; //AS
1839 }
1840 ShouldNotReachHere();
1841 return false;
1842 }
1843
1844 //-----------------------------------------------------------------------
1845 // Exceptions
1846 //
1847
1848 static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg);
1849
1850 // The method is an entry that is always called by a C++ method not
1851 // directly from compiled code. Compiled code will call the C++ method following.
1852 // We can't allow async exception to be installed during exception processing.
1853 JRT_ENTRY_NO_ASYNC(address, OptoRuntime::handle_exception_C_helper(JavaThread* current, nmethod* &nm))
1854 // The frame we rethrow the exception to might not have been processed by the GC yet.
1855 // The stack watermark barrier takes care of detecting that and ensuring the frame
1856 // has updated oops.
1857 StackWatermarkSet::after_unwind(current);
1858
1859 // Do not confuse exception_oop with pending_exception. The exception_oop
1860 // is only used to pass arguments into the method. Not for general
1861 // exception handling. DO NOT CHANGE IT to use pending_exception, since
1862 // the runtime stubs checks this on exit.
1863 assert(current->exception_oop() != nullptr, "exception oop is found");
1864 address handler_address = nullptr;
1865
1866 Handle exception(current, current->exception_oop());
1867 address pc = current->exception_pc();
1868
1869 // Clear out the exception oop and pc since looking up an
1870 // exception handler can cause class loading, which might throw an
1871 // exception and those fields are expected to be clear during
1872 // normal bytecode execution.
1873 current->clear_exception_oop_and_pc();
1874
1875 LogTarget(Info, exceptions) lt;
1876 if (lt.is_enabled()) {
1877 LogStream ls(lt);
1878 trace_exception(&ls, exception(), pc, "");
1879 }
1880
1881 // for AbortVMOnException flag
1882 Exceptions::debug_check_abort(exception);
1883
1884 #ifdef ASSERT
1885 if (!(exception->is_a(vmClasses::Throwable_klass()))) {
1886 // should throw an exception here
1887 ShouldNotReachHere();
1888 }
1889 #endif
1890
1891 // new exception handling: this method is entered only from adapters
1892 // exceptions from compiled java methods are handled in compiled code
1893 // using rethrow node
1894
1895 nm = CodeCache::find_nmethod(pc);
1896 assert(nm != nullptr, "No NMethod found");
1897 if (nm->is_native_method()) {
1898 fatal("Native method should not have path to exception handling");
1899 } else {
1900 // we are switching to old paradigm: search for exception handler in caller_frame
1901 // instead in exception handler of caller_frame.sender()
1902
1903 if (JvmtiExport::can_post_on_exceptions()) {
1904 // "Full-speed catching" is not necessary here,
1905 // since we're notifying the VM on every catch.
1906 // Force deoptimization and the rest of the lookup
1907 // will be fine.
1908 deoptimize_caller_frame(current);
1909 }
1910
1911 // Check the stack guard pages. If enabled, look for handler in this frame;
1912 // otherwise, forcibly unwind the frame.
1913 //
1914 // 4826555: use default current sp for reguard_stack instead of &nm: it's more accurate.
1915 bool force_unwind = !current->stack_overflow_state()->reguard_stack();
1916 bool deopting = false;
1917 if (nm->is_deopt_pc(pc)) {
1918 deopting = true;
1919 RegisterMap map(current,
1920 RegisterMap::UpdateMap::skip,
1921 RegisterMap::ProcessFrames::include,
1922 RegisterMap::WalkContinuation::skip);
1923 frame deoptee = current->last_frame().sender(&map);
1924 assert(deoptee.is_deoptimized_frame(), "must be deopted");
1925 // Adjust the pc back to the original throwing pc
1926 pc = deoptee.pc();
1927 }
1928
1929 // If we are forcing an unwind because of stack overflow then deopt is
1930 // irrelevant since we are throwing the frame away anyway.
1931
1932 if (deopting && !force_unwind) {
1933 handler_address = SharedRuntime::deopt_blob()->unpack_with_exception();
1934 } else {
1935
1936 handler_address =
1937 force_unwind ? nullptr : nm->handler_for_exception_and_pc(exception, pc);
1938
1939 if (handler_address == nullptr) {
1940 bool recursive_exception = false;
1941 handler_address = SharedRuntime::compute_compiled_exc_handler(nm, pc, exception, force_unwind, true, recursive_exception);
1942 assert (handler_address != nullptr, "must have compiled handler");
1943 // Update the exception cache only when the unwind was not forced
1944 // and there didn't happen another exception during the computation of the
1945 // compiled exception handler. Checking for exception oop equality is not
1946 // sufficient because some exceptions are pre-allocated and reused.
1947 if (!force_unwind && !recursive_exception) {
1948 nm->add_handler_for_exception_and_pc(exception,pc,handler_address);
1949 }
1950 } else {
1951 #ifdef ASSERT
1952 bool recursive_exception = false;
1953 address computed_address = SharedRuntime::compute_compiled_exc_handler(nm, pc, exception, force_unwind, true, recursive_exception);
1954 vmassert(recursive_exception || (handler_address == computed_address), "Handler address inconsistency: " PTR_FORMAT " != " PTR_FORMAT,
1955 p2i(handler_address), p2i(computed_address));
1956 #endif
1957 }
1958 }
1959
1960 current->set_exception_pc(pc);
1961 current->set_exception_handler_pc(handler_address);
1962
1963 // Check if the exception PC is a MethodHandle call site.
1964 current->set_is_method_handle_return(nm->is_method_handle_return(pc));
1965 }
1966
1967 // Restore correct return pc. Was saved above.
1968 current->set_exception_oop(exception());
1969 return handler_address;
1970
1971 JRT_END
1972
1973 // We are entering here from exception_blob
1974 // If there is a compiled exception handler in this method, we will continue there;
1975 // otherwise we will unwind the stack and continue at the caller of top frame method
1976 // Note we enter without the usual JRT wrapper. We will call a helper routine that
1977 // will do the normal VM entry. We do it this way so that we can see if the nmethod
1978 // we looked up the handler for has been deoptimized in the meantime. If it has been
1979 // we must not use the handler and instead return the deopt blob.
1980 address OptoRuntime::handle_exception_C(JavaThread* current) {
1981 //
1982 // We are in Java not VM and in debug mode we have a NoHandleMark
1983 //
1984 #ifndef PRODUCT
1985 SharedRuntime::_find_handler_ctr++; // find exception handler
1986 #endif
1987 debug_only(NoHandleMark __hm;)
1988 nmethod* nm = nullptr;
1989 address handler_address = nullptr;
1990 {
1991 // Enter the VM
1992
1993 ResetNoHandleMark rnhm;
1994 handler_address = handle_exception_C_helper(current, nm);
1995 }
1996
1997 // Back in java: Use no oops, DON'T safepoint
1998
1999 // Now check to see if the handler we are returning is in a now
2000 // deoptimized frame
2001
2002 if (nm != nullptr) {
2003 RegisterMap map(current,
2004 RegisterMap::UpdateMap::skip,
2005 RegisterMap::ProcessFrames::skip,
2006 RegisterMap::WalkContinuation::skip);
2007 frame caller = current->last_frame().sender(&map);
2008 #ifdef ASSERT
2009 assert(caller.is_compiled_frame(), "must be");
2010 #endif // ASSERT
2011 if (caller.is_deoptimized_frame()) {
2012 handler_address = SharedRuntime::deopt_blob()->unpack_with_exception();
2013 }
2014 }
2015 return handler_address;
2016 }
2017
2018 //------------------------------rethrow----------------------------------------
2019 // We get here after compiled code has executed a 'RethrowNode'. The callee
2020 // is either throwing or rethrowing an exception. The callee-save registers
2021 // have been restored, synchronized objects have been unlocked and the callee
2022 // stack frame has been removed. The return address was passed in.
2023 // Exception oop is passed as the 1st argument. This routine is then called
2024 // from the stub. On exit, we know where to jump in the caller's code.
2025 // After this C code exits, the stub will pop his frame and end in a jump
2026 // (instead of a return). We enter the caller's default handler.
2027 //
2028 // This must be JRT_LEAF:
2029 // - caller will not change its state as we cannot block on exit,
2030 // therefore raw_exception_handler_for_return_address is all it takes
2031 // to handle deoptimized blobs
2032 //
2033 // However, there needs to be a safepoint check in the middle! So compiled
2034 // safepoints are completely watertight.
2035 //
2036 // Thus, it cannot be a leaf since it contains the NoSafepointVerifier.
2037 //
2038 // *THIS IS NOT RECOMMENDED PROGRAMMING STYLE*
2039 //
2040 address OptoRuntime::rethrow_C(oopDesc* exception, JavaThread* thread, address ret_pc) {
2041 // ret_pc will have been loaded from the stack, so for AArch64 will be signed.
2042 AARCH64_PORT_ONLY(ret_pc = pauth_strip_verifiable(ret_pc));
2043
2044 #ifndef PRODUCT
2045 SharedRuntime::_rethrow_ctr++; // count rethrows
2046 #endif
2047 assert (exception != nullptr, "should have thrown a NullPointerException");
2048 #ifdef ASSERT
2049 if (!(exception->is_a(vmClasses::Throwable_klass()))) {
2050 // should throw an exception here
2051 ShouldNotReachHere();
2052 }
2053 #endif
2054
2055 thread->set_vm_result_oop(exception);
2056 // Frame not compiled (handles deoptimization blob)
2057 return SharedRuntime::raw_exception_handler_for_return_address(thread, ret_pc);
2058 }
2059
2060 static const TypeFunc* make_rethrow_Type() {
2061 // create input type (domain)
2062 const Type **fields = TypeTuple::fields(1);
2063 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Exception oop
2064 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1,fields);
2065
2066 // create result type (range)
2067 fields = TypeTuple::fields(1);
2068 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Exception oop
2069 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
2070
2071 return TypeFunc::make(domain, range);
2072 }
2073
2074
2075 void OptoRuntime::deoptimize_caller_frame(JavaThread *thread, bool doit) {
2076 // Deoptimize the caller before continuing, as the compiled
2077 // exception handler table may not be valid.
2078 if (!StressCompiledExceptionHandlers && doit) {
2079 deoptimize_caller_frame(thread);
2080 }
2081 }
2082
2083 void OptoRuntime::deoptimize_caller_frame(JavaThread *thread) {
2084 // Called from within the owner thread, so no need for safepoint
2085 RegisterMap reg_map(thread,
2086 RegisterMap::UpdateMap::include,
2087 RegisterMap::ProcessFrames::include,
2088 RegisterMap::WalkContinuation::skip);
2089 frame stub_frame = thread->last_frame();
2090 assert(stub_frame.is_runtime_frame() || exception_blob()->contains(stub_frame.pc()), "sanity check");
2091 frame caller_frame = stub_frame.sender(®_map);
2092
2093 // Deoptimize the caller frame.
2094 Deoptimization::deoptimize_frame(thread, caller_frame.id());
2095 }
2096
2097
2098 bool OptoRuntime::is_deoptimized_caller_frame(JavaThread *thread) {
2099 // Called from within the owner thread, so no need for safepoint
2100 RegisterMap reg_map(thread,
2101 RegisterMap::UpdateMap::include,
2102 RegisterMap::ProcessFrames::include,
2103 RegisterMap::WalkContinuation::skip);
2104 frame stub_frame = thread->last_frame();
2105 assert(stub_frame.is_runtime_frame() || exception_blob()->contains(stub_frame.pc()), "sanity check");
2106 frame caller_frame = stub_frame.sender(®_map);
2107 return caller_frame.is_deoptimized_frame();
2108 }
2109
2110 static const TypeFunc* make_register_finalizer_Type() {
2111 // create input type (domain)
2112 const Type **fields = TypeTuple::fields(1);
2113 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // oop; Receiver
2114 // // The JavaThread* is passed to each routine as the last argument
2115 // fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // JavaThread *; Executing thread
2116 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1,fields);
2117
2118 // create result type (range)
2119 fields = TypeTuple::fields(0);
2120
2121 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2122
2123 return TypeFunc::make(domain, range);
2124 }
2125
2126 #if INCLUDE_JFR
2127 static const TypeFunc* make_class_id_load_barrier_Type() {
2128 // create input type (domain)
2129 const Type **fields = TypeTuple::fields(1);
2130 fields[TypeFunc::Parms+0] = TypeInstPtr::KLASS;
2131 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms + 1, fields);
2132
2133 // create result type (range)
2134 fields = TypeTuple::fields(0);
2135
2136 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms + 0, fields);
2137
2138 return TypeFunc::make(domain,range);
2139 }
2140 #endif // INCLUDE_JFR
2141
2142 //-----------------------------------------------------------------------------
2143 static const TypeFunc* make_dtrace_method_entry_exit_Type() {
2144 // create input type (domain)
2145 const Type **fields = TypeTuple::fields(2);
2146 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2147 fields[TypeFunc::Parms+1] = TypeMetadataPtr::BOTTOM; // Method*; Method we are entering
2148 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
2149
2150 // create result type (range)
2151 fields = TypeTuple::fields(0);
2152
2153 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2154
2155 return TypeFunc::make(domain, range);
2156 }
2157
2158 static const TypeFunc* make_dtrace_object_alloc_Type() {
2159 // create input type (domain)
2160 const Type **fields = TypeTuple::fields(2);
2161 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2162 fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL; // oop; newly allocated object
2163
2164 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
2165
2166 // create result type (range)
2167 fields = TypeTuple::fields(0);
2168
2169 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2170
2171 return TypeFunc::make(domain, range);
2172 }
2173
2174 JRT_ENTRY_NO_ASYNC(void, OptoRuntime::register_finalizer_C(oopDesc* obj, JavaThread* current))
2175 assert(oopDesc::is_oop(obj), "must be a valid oop");
2176 assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
2177 InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
2178 JRT_END
2179
2180 //-----------------------------------------------------------------------------
2181
2182 NamedCounter * volatile OptoRuntime::_named_counters = nullptr;
2183
2184 //
2185 // dump the collected NamedCounters.
2186 //
2187 void OptoRuntime::print_named_counters() {
2188 int total_lock_count = 0;
2189 int eliminated_lock_count = 0;
2190
2191 NamedCounter* c = _named_counters;
2192 while (c) {
2193 if (c->tag() == NamedCounter::LockCounter || c->tag() == NamedCounter::EliminatedLockCounter) {
2194 int count = c->count();
2195 if (count > 0) {
2196 bool eliminated = c->tag() == NamedCounter::EliminatedLockCounter;
2197 if (Verbose) {
2198 tty->print_cr("%d %s%s", count, c->name(), eliminated ? " (eliminated)" : "");
2199 }
2200 total_lock_count += count;
2201 if (eliminated) {
2202 eliminated_lock_count += count;
2203 }
2204 }
2205 }
2206 c = c->next();
2207 }
2208 if (total_lock_count > 0) {
2209 tty->print_cr("dynamic locks: %d", total_lock_count);
2210 if (eliminated_lock_count) {
2211 tty->print_cr("eliminated locks: %d (%d%%)", eliminated_lock_count,
2212 (int)(eliminated_lock_count * 100.0 / total_lock_count));
2213 }
2214 }
2215 }
2216
2217 //
2218 // Allocate a new NamedCounter. The JVMState is used to generate the
2219 // name which consists of method@line for the inlining tree.
2220 //
2221
2222 NamedCounter* OptoRuntime::new_named_counter(JVMState* youngest_jvms, NamedCounter::CounterTag tag) {
2223 int max_depth = youngest_jvms->depth();
2224
2225 // Visit scopes from youngest to oldest.
2226 bool first = true;
2227 stringStream st;
2228 for (int depth = max_depth; depth >= 1; depth--) {
2229 JVMState* jvms = youngest_jvms->of_depth(depth);
2230 ciMethod* m = jvms->has_method() ? jvms->method() : nullptr;
2231 if (!first) {
2232 st.print(" ");
2233 } else {
2234 first = false;
2235 }
2236 int bci = jvms->bci();
2237 if (bci < 0) bci = 0;
2238 if (m != nullptr) {
2239 st.print("%s.%s", m->holder()->name()->as_utf8(), m->name()->as_utf8());
2240 } else {
2241 st.print("no method");
2242 }
2243 st.print("@%d", bci);
2244 // To print linenumbers instead of bci use: m->line_number_from_bci(bci)
2245 }
2246 NamedCounter* c = new NamedCounter(st.freeze(), tag);
2247
2248 // atomically add the new counter to the head of the list. We only
2249 // add counters so this is safe.
2250 NamedCounter* head;
2251 do {
2252 c->set_next(nullptr);
2253 head = _named_counters;
2254 c->set_next(head);
2255 } while (Atomic::cmpxchg(&_named_counters, head, c) != head);
2256 return c;
2257 }
2258
2259 void OptoRuntime::initialize_types() {
2260 _new_instance_Type = make_new_instance_Type();
2261 _new_array_Type = make_new_array_Type();
2262 _new_array_nozero_Type = make_new_array_nozero_Type();
2263 _multianewarray2_Type = multianewarray_Type(2);
2264 _multianewarray3_Type = multianewarray_Type(3);
2265 _multianewarray4_Type = multianewarray_Type(4);
2266 _multianewarray5_Type = multianewarray_Type(5);
2267 _multianewarrayN_Type = make_multianewarrayN_Type();
2268 _complete_monitor_enter_Type = make_complete_monitor_enter_Type();
2269 _complete_monitor_exit_Type = make_complete_monitor_exit_Type();
2270 _monitor_notify_Type = make_monitor_notify_Type();
2271 _uncommon_trap_Type = make_uncommon_trap_Type();
2272 _athrow_Type = make_athrow_Type();
2273 _rethrow_Type = make_rethrow_Type();
2274 _Math_D_D_Type = make_Math_D_D_Type();
2275 _Math_DD_D_Type = make_Math_DD_D_Type();
2276 _modf_Type = make_modf_Type();
2277 _l2f_Type = make_l2f_Type();
2278 _void_long_Type = make_void_long_Type();
2279 _void_void_Type = make_void_void_Type();
2280 _jfr_write_checkpoint_Type = make_jfr_write_checkpoint_Type();
2281 _flush_windows_Type = make_flush_windows_Type();
2282 _fast_arraycopy_Type = make_arraycopy_Type(ac_fast);
2283 _checkcast_arraycopy_Type = make_arraycopy_Type(ac_checkcast);
2284 _generic_arraycopy_Type = make_arraycopy_Type(ac_generic);
2285 _slow_arraycopy_Type = make_arraycopy_Type(ac_slow);
2286 _unsafe_setmemory_Type = make_setmemory_Type();
2287 _array_fill_Type = make_array_fill_Type();
2288 _array_sort_Type = make_array_sort_Type();
2289 _array_partition_Type = make_array_partition_Type();
2290 _aescrypt_block_Type = make_aescrypt_block_Type();
2291 _cipherBlockChaining_aescrypt_Type = make_cipherBlockChaining_aescrypt_Type();
2292 _electronicCodeBook_aescrypt_Type = make_electronicCodeBook_aescrypt_Type();
2293 _counterMode_aescrypt_Type = make_counterMode_aescrypt_Type();
2294 _galoisCounterMode_aescrypt_Type = make_galoisCounterMode_aescrypt_Type();
2295 _digestBase_implCompress_with_sha3_Type = make_digestBase_implCompress_Type( /* is_sha3= */ true);
2296 _digestBase_implCompress_without_sha3_Type = make_digestBase_implCompress_Type( /* is_sha3= */ false);;
2297 _digestBase_implCompressMB_with_sha3_Type = make_digestBase_implCompressMB_Type(/* is_sha3= */ true);
2298 _digestBase_implCompressMB_without_sha3_Type = make_digestBase_implCompressMB_Type(/* is_sha3= */ false);
2299 _double_keccak_Type = make_double_keccak_Type();
2300 _multiplyToLen_Type = make_multiplyToLen_Type();
2301 _montgomeryMultiply_Type = make_montgomeryMultiply_Type();
2302 _montgomerySquare_Type = make_montgomerySquare_Type();
2303 _squareToLen_Type = make_squareToLen_Type();
2304 _mulAdd_Type = make_mulAdd_Type();
2305 _bigIntegerShift_Type = make_bigIntegerShift_Type();
2306 _vectorizedMismatch_Type = make_vectorizedMismatch_Type();
2307 _ghash_processBlocks_Type = make_ghash_processBlocks_Type();
2308 _chacha20Block_Type = make_chacha20Block_Type();
2309 _kyberNtt_Type = make_kyberNtt_Type();
2310 _kyberInverseNtt_Type = make_kyberInverseNtt_Type();
2311 _kyberNttMult_Type = make_kyberNttMult_Type();
2312 _kyberAddPoly_2_Type = make_kyberAddPoly_2_Type();
2313 _kyberAddPoly_3_Type = make_kyberAddPoly_3_Type();
2314 _kyber12To16_Type = make_kyber12To16_Type();
2315 _kyberBarrettReduce_Type = make_kyberBarrettReduce_Type();
2316 _dilithiumAlmostNtt_Type = make_dilithiumAlmostNtt_Type();
2317 _dilithiumAlmostInverseNtt_Type = make_dilithiumAlmostInverseNtt_Type();
2318 _dilithiumNttMult_Type = make_dilithiumNttMult_Type();
2319 _dilithiumMontMulByConstant_Type = make_dilithiumMontMulByConstant_Type();
2320 _dilithiumDecomposePoly_Type = make_dilithiumDecomposePoly_Type();
2321 _base64_encodeBlock_Type = make_base64_encodeBlock_Type();
2322 _base64_decodeBlock_Type = make_base64_decodeBlock_Type();
2323 _string_IndexOf_Type = make_string_IndexOf_Type();
2324 _poly1305_processBlocks_Type = make_poly1305_processBlocks_Type();
2325 _intpoly_montgomeryMult_P256_Type = make_intpoly_montgomeryMult_P256_Type();
2326 _intpoly_assign_Type = make_intpoly_assign_Type();
2327 _updateBytesCRC32_Type = make_updateBytesCRC32_Type();
2328 _updateBytesCRC32C_Type = make_updateBytesCRC32C_Type();
2329 _updateBytesAdler32_Type = make_updateBytesAdler32_Type();
2330 _osr_end_Type = make_osr_end_Type();
2331 _register_finalizer_Type = make_register_finalizer_Type();
2332 JFR_ONLY(
2333 _class_id_load_barrier_Type = make_class_id_load_barrier_Type();
2334 )
2335 #if INCLUDE_JVMTI
2336 _notify_jvmti_vthread_Type = make_notify_jvmti_vthread_Type();
2337 #endif // INCLUDE_JVMTI
2338 _dtrace_method_entry_exit_Type = make_dtrace_method_entry_exit_Type();
2339 _dtrace_object_alloc_Type = make_dtrace_object_alloc_Type();
2340 }
2341
2342 int trace_exception_counter = 0;
2343 static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg) {
2344 trace_exception_counter++;
2345 stringStream tempst;
2346
2347 tempst.print("%d [Exception (%s): ", trace_exception_counter, msg);
2348 exception_oop->print_value_on(&tempst);
2349 tempst.print(" in ");
2350 CodeBlob* blob = CodeCache::find_blob(exception_pc);
2351 if (blob->is_nmethod()) {
2352 blob->as_nmethod()->method()->print_value_on(&tempst);
2353 } else if (blob->is_runtime_stub()) {
2354 tempst.print("<runtime-stub>");
2355 } else {
2356 tempst.print("<unknown>");
2357 }
2358 tempst.print(" at " INTPTR_FORMAT, p2i(exception_pc));
2359 tempst.print("]");
2360
2361 st->print_raw_cr(tempst.freeze());
2362 }
2363
2364 const TypeFunc *OptoRuntime::store_inline_type_fields_Type() {
2365 // create input type (domain)
2366 uint total = SharedRuntime::java_return_convention_max_int + SharedRuntime::java_return_convention_max_float*2;
2367 const Type **fields = TypeTuple::fields(total);
2368 // We don't know the number of returned values and their
2369 // types. Assume all registers available to the return convention
2370 // are used.
2371 fields[TypeFunc::Parms] = TypePtr::BOTTOM;
2372 uint i = 1;
2373 for (; i < SharedRuntime::java_return_convention_max_int; i++) {
2374 fields[TypeFunc::Parms+i] = TypeInt::INT;
2375 }
2376 for (; i < total; i+=2) {
2377 fields[TypeFunc::Parms+i] = Type::DOUBLE;
2378 fields[TypeFunc::Parms+i+1] = Type::HALF;
2379 }
2380 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + total, fields);
2381
2382 // create result type (range)
2383 fields = TypeTuple::fields(1);
2384 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
2385
2386 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1,fields);
2387
2388 return TypeFunc::make(domain, range);
2389 }
2390
2391 const TypeFunc *OptoRuntime::pack_inline_type_Type() {
2392 // create input type (domain)
2393 uint total = 1 + SharedRuntime::java_return_convention_max_int + SharedRuntime::java_return_convention_max_float*2;
2394 const Type **fields = TypeTuple::fields(total);
2395 // We don't know the number of returned values and their
2396 // types. Assume all registers available to the return convention
2397 // are used.
2398 fields[TypeFunc::Parms] = TypeRawPtr::BOTTOM;
2399 fields[TypeFunc::Parms+1] = TypeRawPtr::BOTTOM;
2400 uint i = 2;
2401 for (; i < SharedRuntime::java_return_convention_max_int+1; i++) {
2402 fields[TypeFunc::Parms+i] = TypeInt::INT;
2403 }
2404 for (; i < total; i+=2) {
2405 fields[TypeFunc::Parms+i] = Type::DOUBLE;
2406 fields[TypeFunc::Parms+i+1] = Type::HALF;
2407 }
2408 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + total, fields);
2409
2410 // create result type (range)
2411 fields = TypeTuple::fields(1);
2412 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
2413
2414 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1,fields);
2415
2416 return TypeFunc::make(domain, range);
2417 }
2418
2419 JRT_BLOCK_ENTRY(void, OptoRuntime::load_unknown_inline_C(flatArrayOopDesc* array, int index, JavaThread* current))
2420 JRT_BLOCK;
2421 oop buffer = array->read_value_from_flat_array(index, THREAD);
2422 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
2423 current->set_vm_result_oop(buffer);
2424 JRT_BLOCK_END;
2425 JRT_END
2426
2427 const TypeFunc* OptoRuntime::load_unknown_inline_Type() {
2428 // create input type (domain)
2429 const Type** fields = TypeTuple::fields(2);
2430 fields[TypeFunc::Parms] = TypeOopPtr::NOTNULL;
2431 fields[TypeFunc::Parms+1] = TypeInt::POS;
2432
2433 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+2, fields);
2434
2435 // create result type (range)
2436 fields = TypeTuple::fields(1);
2437 fields[TypeFunc::Parms] = TypeInstPtr::BOTTOM;
2438
2439 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
2440
2441 return TypeFunc::make(domain, range);
2442 }
2443
2444 JRT_BLOCK_ENTRY(void, OptoRuntime::store_unknown_inline_C(instanceOopDesc* buffer, flatArrayOopDesc* array, int index, JavaThread* current))
2445 JRT_BLOCK;
2446 array->write_value_to_flat_array(buffer, index, THREAD);
2447 if (HAS_PENDING_EXCEPTION) {
2448 fatal("This entry must be changed to be a non-leaf entry because writing to a flat array can now throw an exception");
2449 }
2450 JRT_BLOCK_END;
2451 JRT_END
2452
2453 const TypeFunc* OptoRuntime::store_unknown_inline_Type() {
2454 // create input type (domain)
2455 const Type** fields = TypeTuple::fields(3);
2456 fields[TypeFunc::Parms] = TypeInstPtr::NOTNULL;
2457 fields[TypeFunc::Parms+1] = TypeOopPtr::NOTNULL;
2458 fields[TypeFunc::Parms+2] = TypeInt::POS;
2459
2460 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+3, fields);
2461
2462 // create result type (range)
2463 fields = TypeTuple::fields(0);
2464 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
2465
2466 return TypeFunc::make(domain, range);
2467 }