1 /*
2 * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "classfile/vmClasses.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "code/codeCache.hpp"
28 #include "code/compiledIC.hpp"
29 #include "code/nmethod.hpp"
30 #include "code/pcDesc.hpp"
31 #include "code/scopeDesc.hpp"
32 #include "code/vtableStubs.hpp"
33 #include "compiler/compilationMemoryStatistic.hpp"
34 #include "compiler/compileBroker.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/g1/g1HeapRegion.hpp"
37 #include "gc/shared/barrierSet.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gcLocker.hpp"
40 #include "interpreter/bytecode.hpp"
41 #include "interpreter/interpreter.hpp"
42 #include "interpreter/linkResolver.hpp"
43 #include "logging/log.hpp"
44 #include "logging/logStream.hpp"
45 #include "memory/oopFactory.hpp"
46 #include "memory/resourceArea.hpp"
47 #include "oops/flatArrayKlass.hpp"
48 #include "oops/flatArrayOop.inline.hpp"
49 #include "oops/objArrayKlass.hpp"
50 #include "oops/klass.inline.hpp"
51 #include "oops/oop.inline.hpp"
52 #include "oops/typeArrayOop.inline.hpp"
53 #include "opto/ad.hpp"
54 #include "opto/addnode.hpp"
55 #include "opto/callnode.hpp"
56 #include "opto/cfgnode.hpp"
57 #include "opto/graphKit.hpp"
58 #include "opto/machnode.hpp"
59 #include "opto/matcher.hpp"
60 #include "opto/memnode.hpp"
61 #include "opto/mulnode.hpp"
62 #include "opto/output.hpp"
63 #include "opto/runtime.hpp"
64 #include "opto/subnode.hpp"
65 #include "prims/jvmtiExport.hpp"
66 #include "runtime/atomic.hpp"
67 #include "runtime/frame.inline.hpp"
68 #include "runtime/handles.inline.hpp"
69 #include "runtime/interfaceSupport.inline.hpp"
70 #include "runtime/javaCalls.hpp"
71 #include "runtime/sharedRuntime.hpp"
72 #include "runtime/signature.hpp"
73 #include "runtime/stackWatermarkSet.hpp"
74 #include "runtime/synchronizer.hpp"
75 #include "runtime/threadWXSetters.inline.hpp"
76 #include "runtime/vframe.hpp"
77 #include "runtime/vframeArray.hpp"
78 #include "runtime/vframe_hp.hpp"
79 #include "utilities/copy.hpp"
80 #include "utilities/preserveException.hpp"
81
82
83 // For debugging purposes:
84 // To force FullGCALot inside a runtime function, add the following two lines
85 //
86 // Universe::release_fullgc_alot_dummy();
87 // Universe::heap()->collect();
88 //
89 // At command line specify the parameters: -XX:+FullGCALot -XX:FullGCALotStart=100000000
90
91
92 #define C2_BLOB_FIELD_DEFINE(name, type) \
93 type OptoRuntime:: BLOB_FIELD_NAME(name) = nullptr;
94 #define C2_STUB_FIELD_NAME(name) _ ## name ## _Java
95 #define C2_STUB_FIELD_DEFINE(name, f, t, r) \
96 address OptoRuntime:: C2_STUB_FIELD_NAME(name) = nullptr;
97 #define C2_JVMTI_STUB_FIELD_DEFINE(name) \
98 address OptoRuntime:: STUB_FIELD_NAME(name) = nullptr;
99 C2_STUBS_DO(C2_BLOB_FIELD_DEFINE, C2_STUB_FIELD_DEFINE, C2_JVMTI_STUB_FIELD_DEFINE)
100 #undef C2_BLOB_FIELD_DEFINE
101 #undef C2_STUB_FIELD_DEFINE
102 #undef C2_JVMTI_STUB_FIELD_DEFINE
103
104 #define C2_BLOB_NAME_DEFINE(name, type) "C2 Runtime " # name "_blob",
105 #define C2_STUB_NAME_DEFINE(name, f, t, r) "C2 Runtime " # name,
106 #define C2_JVMTI_STUB_NAME_DEFINE(name) "C2 Runtime " # name,
107 const char* OptoRuntime::_stub_names[] = {
108 C2_STUBS_DO(C2_BLOB_NAME_DEFINE, C2_STUB_NAME_DEFINE, C2_JVMTI_STUB_NAME_DEFINE)
109 };
110 #undef C2_BLOB_NAME_DEFINE
111 #undef C2_STUB_NAME_DEFINE
112 #undef C2_JVMTI_STUB_NAME_DEFINE
113
114 // This should be called in an assertion at the start of OptoRuntime routines
115 // which are entered from compiled code (all of them)
116 #ifdef ASSERT
117 static bool check_compiled_frame(JavaThread* thread) {
118 assert(thread->last_frame().is_runtime_frame(), "cannot call runtime directly from compiled code");
119 RegisterMap map(thread,
120 RegisterMap::UpdateMap::skip,
121 RegisterMap::ProcessFrames::include,
122 RegisterMap::WalkContinuation::skip);
123 frame caller = thread->last_frame().sender(&map);
124 assert(caller.is_compiled_frame(), "not being called from compiled like code");
125 return true;
126 }
127 #endif // ASSERT
128
129 /*
130 #define gen(env, var, type_func_gen, c_func, fancy_jump, pass_tls, return_pc) \
131 var = generate_stub(env, type_func_gen, CAST_FROM_FN_PTR(address, c_func), #var, fancy_jump, pass_tls, return_pc); \
132 if (var == nullptr) { return false; }
133 */
134
135 #define GEN_C2_BLOB(name, type) \
136 BLOB_FIELD_NAME(name) = \
137 generate_ ## name ## _blob(); \
138 if (BLOB_FIELD_NAME(name) == nullptr) { return false; }
139
140 // a few helper macros to conjure up generate_stub call arguments
141 #define C2_STUB_FIELD_NAME(name) _ ## name ## _Java
142 #define C2_STUB_TYPEFUNC(name) name ## _Type
143 #define C2_STUB_C_FUNC(name) CAST_FROM_FN_PTR(address, name ## _C)
144 #define C2_STUB_NAME(name) stub_name(OptoStubId::name ## _id)
145 #define C2_STUB_ID(name) OptoStubId::name ## _id
146
147 // Almost all the C functions targeted from the generated stubs are
148 // implemented locally to OptoRuntime with names that can be generated
149 // from the stub name by appending suffix '_C'. However, in two cases
150 // a common target method also needs to be called from shared runtime
151 // stubs. In these two cases the opto stubs rely on method
152 // imlementations defined in class SharedRuntime. The following
153 // defines temporarily rebind the generated names to reference the
154 // relevant implementations.
155
156 #define GEN_C2_STUB(name, fancy_jump, pass_tls, pass_retpc ) \
157 C2_STUB_FIELD_NAME(name) = \
158 generate_stub(env, \
159 C2_STUB_TYPEFUNC(name), \
160 C2_STUB_C_FUNC(name), \
161 C2_STUB_NAME(name), \
162 (int)C2_STUB_ID(name), \
163 fancy_jump, \
164 pass_tls, \
165 pass_retpc); \
166 if (C2_STUB_FIELD_NAME(name) == nullptr) { return false; } \
167
168 #define C2_JVMTI_STUB_C_FUNC(name) CAST_FROM_FN_PTR(address, SharedRuntime::name)
169
170 #define GEN_C2_JVMTI_STUB(name) \
171 STUB_FIELD_NAME(name) = \
172 generate_stub(env, \
173 notify_jvmti_vthread_Type, \
174 C2_JVMTI_STUB_C_FUNC(name), \
175 C2_STUB_NAME(name), \
176 (int)C2_STUB_ID(name), \
177 0, \
178 true, \
179 false); \
180 if (STUB_FIELD_NAME(name) == nullptr) { return false; } \
181
182 bool OptoRuntime::generate(ciEnv* env) {
183
184 C2_STUBS_DO(GEN_C2_BLOB, GEN_C2_STUB, GEN_C2_JVMTI_STUB)
185
186 return true;
187 }
188
189 #undef GEN_C2_BLOB
190
191 #undef C2_STUB_FIELD_NAME
192 #undef C2_STUB_TYPEFUNC
193 #undef C2_STUB_C_FUNC
194 #undef C2_STUB_NAME
195 #undef GEN_C2_STUB
196
197 #undef C2_JVMTI_STUB_C_FUNC
198 #undef GEN_C2_JVMTI_STUB
199 // #undef gen
200
201 const TypeFunc* OptoRuntime::_new_instance_Type = nullptr;
202 const TypeFunc* OptoRuntime::_new_array_Type = nullptr;
203 const TypeFunc* OptoRuntime::_new_array_nozero_Type = nullptr;
204 const TypeFunc* OptoRuntime::_multianewarray2_Type = nullptr;
205 const TypeFunc* OptoRuntime::_multianewarray3_Type = nullptr;
206 const TypeFunc* OptoRuntime::_multianewarray4_Type = nullptr;
207 const TypeFunc* OptoRuntime::_multianewarray5_Type = nullptr;
208 const TypeFunc* OptoRuntime::_multianewarrayN_Type = nullptr;
209 const TypeFunc* OptoRuntime::_complete_monitor_enter_Type = nullptr;
210 const TypeFunc* OptoRuntime::_complete_monitor_exit_Type = nullptr;
211 const TypeFunc* OptoRuntime::_monitor_notify_Type = nullptr;
212 const TypeFunc* OptoRuntime::_uncommon_trap_Type = nullptr;
213 const TypeFunc* OptoRuntime::_athrow_Type = nullptr;
214 const TypeFunc* OptoRuntime::_rethrow_Type = nullptr;
215 const TypeFunc* OptoRuntime::_Math_D_D_Type = nullptr;
216 const TypeFunc* OptoRuntime::_Math_DD_D_Type = nullptr;
217 const TypeFunc* OptoRuntime::_modf_Type = nullptr;
218 const TypeFunc* OptoRuntime::_l2f_Type = nullptr;
219 const TypeFunc* OptoRuntime::_void_long_Type = nullptr;
220 const TypeFunc* OptoRuntime::_void_void_Type = nullptr;
221 const TypeFunc* OptoRuntime::_jfr_write_checkpoint_Type = nullptr;
222 const TypeFunc* OptoRuntime::_flush_windows_Type = nullptr;
223 const TypeFunc* OptoRuntime::_fast_arraycopy_Type = nullptr;
224 const TypeFunc* OptoRuntime::_checkcast_arraycopy_Type = nullptr;
225 const TypeFunc* OptoRuntime::_generic_arraycopy_Type = nullptr;
226 const TypeFunc* OptoRuntime::_slow_arraycopy_Type = nullptr;
227 const TypeFunc* OptoRuntime::_unsafe_setmemory_Type = nullptr;
228 const TypeFunc* OptoRuntime::_array_fill_Type = nullptr;
229 const TypeFunc* OptoRuntime::_array_sort_Type = nullptr;
230 const TypeFunc* OptoRuntime::_array_partition_Type = nullptr;
231 const TypeFunc* OptoRuntime::_aescrypt_block_Type = nullptr;
232 const TypeFunc* OptoRuntime::_cipherBlockChaining_aescrypt_Type = nullptr;
233 const TypeFunc* OptoRuntime::_electronicCodeBook_aescrypt_Type = nullptr;
234 const TypeFunc* OptoRuntime::_counterMode_aescrypt_Type = nullptr;
235 const TypeFunc* OptoRuntime::_galoisCounterMode_aescrypt_Type = nullptr;
236 const TypeFunc* OptoRuntime::_digestBase_implCompress_with_sha3_Type = nullptr;
237 const TypeFunc* OptoRuntime::_digestBase_implCompress_without_sha3_Type = nullptr;
238 const TypeFunc* OptoRuntime::_digestBase_implCompressMB_with_sha3_Type = nullptr;
239 const TypeFunc* OptoRuntime::_digestBase_implCompressMB_without_sha3_Type = nullptr;
240 const TypeFunc* OptoRuntime::_double_keccak_Type = nullptr;
241 const TypeFunc* OptoRuntime::_multiplyToLen_Type = nullptr;
242 const TypeFunc* OptoRuntime::_montgomeryMultiply_Type = nullptr;
243 const TypeFunc* OptoRuntime::_montgomerySquare_Type = nullptr;
244 const TypeFunc* OptoRuntime::_squareToLen_Type = nullptr;
245 const TypeFunc* OptoRuntime::_mulAdd_Type = nullptr;
246 const TypeFunc* OptoRuntime::_bigIntegerShift_Type = nullptr;
247 const TypeFunc* OptoRuntime::_vectorizedMismatch_Type = nullptr;
248 const TypeFunc* OptoRuntime::_ghash_processBlocks_Type = nullptr;
249 const TypeFunc* OptoRuntime::_chacha20Block_Type = nullptr;
250 const TypeFunc* OptoRuntime::_kyberNtt_Type = nullptr;
251 const TypeFunc* OptoRuntime::_kyberInverseNtt_Type = nullptr;
252 const TypeFunc* OptoRuntime::_kyberNttMult_Type = nullptr;
253 const TypeFunc* OptoRuntime::_kyberAddPoly_2_Type = nullptr;
254 const TypeFunc* OptoRuntime::_kyberAddPoly_3_Type = nullptr;
255 const TypeFunc* OptoRuntime::_kyber12To16_Type = nullptr;
256 const TypeFunc* OptoRuntime::_kyberBarrettReduce_Type = nullptr;
257 const TypeFunc* OptoRuntime::_dilithiumAlmostNtt_Type = nullptr;
258 const TypeFunc* OptoRuntime::_dilithiumAlmostInverseNtt_Type = nullptr;
259 const TypeFunc* OptoRuntime::_dilithiumNttMult_Type = nullptr;
260 const TypeFunc* OptoRuntime::_dilithiumMontMulByConstant_Type = nullptr;
261 const TypeFunc* OptoRuntime::_dilithiumDecomposePoly_Type = nullptr;
262 const TypeFunc* OptoRuntime::_base64_encodeBlock_Type = nullptr;
263 const TypeFunc* OptoRuntime::_base64_decodeBlock_Type = nullptr;
264 const TypeFunc* OptoRuntime::_string_IndexOf_Type = nullptr;
265 const TypeFunc* OptoRuntime::_poly1305_processBlocks_Type = nullptr;
266 const TypeFunc* OptoRuntime::_intpoly_montgomeryMult_P256_Type = nullptr;
267 const TypeFunc* OptoRuntime::_intpoly_assign_Type = nullptr;
268 const TypeFunc* OptoRuntime::_updateBytesCRC32_Type = nullptr;
269 const TypeFunc* OptoRuntime::_updateBytesCRC32C_Type = nullptr;
270 const TypeFunc* OptoRuntime::_updateBytesAdler32_Type = nullptr;
271 const TypeFunc* OptoRuntime::_osr_end_Type = nullptr;
272 const TypeFunc* OptoRuntime::_register_finalizer_Type = nullptr;
273 #if INCLUDE_JFR
274 const TypeFunc* OptoRuntime::_class_id_load_barrier_Type = nullptr;
275 #endif // INCLUDE_JFR
276 #if INCLUDE_JVMTI
277 const TypeFunc* OptoRuntime::_notify_jvmti_vthread_Type = nullptr;
278 #endif // INCLUDE_JVMTI
279 const TypeFunc* OptoRuntime::_dtrace_method_entry_exit_Type = nullptr;
280 const TypeFunc* OptoRuntime::_dtrace_object_alloc_Type = nullptr;
281
282 // Helper method to do generation of RunTimeStub's
283 address OptoRuntime::generate_stub(ciEnv* env,
284 TypeFunc_generator gen, address C_function,
285 const char *name, int stub_id,
286 int is_fancy_jump, bool pass_tls,
287 bool return_pc) {
288
289 // Matching the default directive, we currently have no method to match.
290 DirectiveSet* directive = DirectivesStack::getDefaultDirective(CompileBroker::compiler(CompLevel_full_optimization));
291 CompilationMemoryStatisticMark cmsm(directive);
292 ResourceMark rm;
293 Compile C(env, gen, C_function, name, stub_id, is_fancy_jump, pass_tls, return_pc, directive);
294 DirectivesStack::release(directive);
295 return C.stub_entry_point();
296 }
297
298 const char* OptoRuntime::stub_name(address entry) {
299 #ifndef PRODUCT
300 CodeBlob* cb = CodeCache::find_blob(entry);
301 RuntimeStub* rs =(RuntimeStub *)cb;
302 assert(rs != nullptr && rs->is_runtime_stub(), "not a runtime stub");
303 return rs->name();
304 #else
305 // Fast implementation for product mode (maybe it should be inlined too)
306 return "runtime stub";
307 #endif
308 }
309
310 // local methods passed as arguments to stub generator that forward
311 // control to corresponding JRT methods of SharedRuntime
312
313 void OptoRuntime::slow_arraycopy_C(oopDesc* src, jint src_pos,
314 oopDesc* dest, jint dest_pos,
315 jint length, JavaThread* thread) {
316 SharedRuntime::slow_arraycopy_C(src, src_pos, dest, dest_pos, length, thread);
317 }
318
319 void OptoRuntime::complete_monitor_locking_C(oopDesc* obj, BasicLock* lock, JavaThread* current) {
320 SharedRuntime::complete_monitor_locking_C(obj, lock, current);
321 }
322
323
324 //=============================================================================
325 // Opto compiler runtime routines
326 //=============================================================================
327
328
329 //=============================allocation======================================
330 // We failed the fast-path allocation. Now we need to do a scavenge or GC
331 // and try allocation again.
332
333 // object allocation
334 JRT_BLOCK_ENTRY(void, OptoRuntime::new_instance_C(Klass* klass, bool is_larval, JavaThread* current))
335 JRT_BLOCK;
336 #ifndef PRODUCT
337 SharedRuntime::_new_instance_ctr++; // new instance requires GC
338 #endif
339 assert(check_compiled_frame(current), "incorrect caller");
340
341 // These checks are cheap to make and support reflective allocation.
342 int lh = klass->layout_helper();
343 if (Klass::layout_helper_needs_slow_path(lh) || !InstanceKlass::cast(klass)->is_initialized()) {
344 Handle holder(current, klass->klass_holder()); // keep the klass alive
345 klass->check_valid_for_instantiation(false, THREAD);
346 if (!HAS_PENDING_EXCEPTION) {
347 InstanceKlass::cast(klass)->initialize(THREAD);
348 }
349 }
350
351 if (!HAS_PENDING_EXCEPTION) {
352 // Scavenge and allocate an instance.
353 Handle holder(current, klass->klass_holder()); // keep the klass alive
354 instanceOop result = InstanceKlass::cast(klass)->allocate_instance(THREAD);
355 if (is_larval) {
356 // Check if this is a larval buffer allocation
357 result->set_mark(result->mark().enter_larval_state());
358 }
359 current->set_vm_result_oop(result);
360
361 // Pass oops back through thread local storage. Our apparent type to Java
362 // is that we return an oop, but we can block on exit from this routine and
363 // a GC can trash the oop in C's return register. The generated stub will
364 // fetch the oop from TLS after any possible GC.
365 }
366
367 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
368 JRT_BLOCK_END;
369
370 // inform GC that we won't do card marks for initializing writes.
371 SharedRuntime::on_slowpath_allocation_exit(current);
372 JRT_END
373
374
375 // array allocation
376 JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_C(Klass* array_type, int len, oopDesc* init_val, JavaThread* current))
377 JRT_BLOCK;
378 #ifndef PRODUCT
379 SharedRuntime::_new_array_ctr++; // new array requires GC
380 #endif
381 assert(check_compiled_frame(current), "incorrect caller");
382
383 // Scavenge and allocate an instance.
384 oop result;
385 Handle h_init_val(current, init_val); // keep the init_val object alive
386
387 if (array_type->is_flatArray_klass()) {
388 Handle holder(current, array_type->klass_holder()); // keep the array klass alive
389 FlatArrayKlass* fak = FlatArrayKlass::cast(array_type);
390 InlineKlass* vk = fak->element_klass();
391 result = oopFactory::new_flatArray(vk, len, fak->layout_kind(), THREAD);
392 if (array_type->is_null_free_array_klass() && !h_init_val.is_null()) {
393 // Null-free arrays need to be initialized
394 for (int i = 0; i < len; i++) {
395 vk->write_value_to_addr(h_init_val(), ((flatArrayOop)result)->value_at_addr(i, fak->layout_helper()), fak->layout_kind(), true, CHECK);
396 }
397 }
398 } else if (array_type->is_typeArray_klass()) {
399 // The oopFactory likes to work with the element type.
400 // (We could bypass the oopFactory, since it doesn't add much value.)
401 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
402 result = oopFactory::new_typeArray(elem_type, len, THREAD);
403 } else {
404 Handle holder(current, array_type->klass_holder()); // keep the array klass alive
405 ObjArrayKlass* array_klass = ObjArrayKlass::cast(array_type);
406 result = array_klass->allocate(len, THREAD);
407 if (array_type->is_null_free_array_klass() && !h_init_val.is_null()) {
408 // Null-free arrays need to be initialized
409 for (int i = 0; i < len; i++) {
410 ((objArrayOop)result)->obj_at_put(i, h_init_val());
411 }
412 }
413 }
414
415 // Pass oops back through thread local storage. Our apparent type to Java
416 // is that we return an oop, but we can block on exit from this routine and
417 // a GC can trash the oop in C's return register. The generated stub will
418 // fetch the oop from TLS after any possible GC.
419 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
420 current->set_vm_result_oop(result);
421 JRT_BLOCK_END;
422
423 // inform GC that we won't do card marks for initializing writes.
424 SharedRuntime::on_slowpath_allocation_exit(current);
425 JRT_END
426
427 // array allocation without zeroing
428 JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(Klass* array_type, int len, JavaThread* current))
429 JRT_BLOCK;
430 #ifndef PRODUCT
431 SharedRuntime::_new_array_ctr++; // new array requires GC
432 #endif
433 assert(check_compiled_frame(current), "incorrect caller");
434
435 // Scavenge and allocate an instance.
436 oop result;
437
438 assert(array_type->is_typeArray_klass(), "should be called only for type array");
439 // The oopFactory likes to work with the element type.
440 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
441 result = oopFactory::new_typeArray_nozero(elem_type, len, THREAD);
442
443 // Pass oops back through thread local storage. Our apparent type to Java
444 // is that we return an oop, but we can block on exit from this routine and
445 // a GC can trash the oop in C's return register. The generated stub will
446 // fetch the oop from TLS after any possible GC.
447 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
448 current->set_vm_result_oop(result);
449 JRT_BLOCK_END;
450
451
452 // inform GC that we won't do card marks for initializing writes.
453 SharedRuntime::on_slowpath_allocation_exit(current);
454
455 oop result = current->vm_result_oop();
456 if ((len > 0) && (result != nullptr) &&
457 is_deoptimized_caller_frame(current)) {
458 // Zero array here if the caller is deoptimized.
459 const size_t size = TypeArrayKlass::cast(array_type)->oop_size(result);
460 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
461 size_t hs_bytes = arrayOopDesc::base_offset_in_bytes(elem_type);
462 assert(is_aligned(hs_bytes, BytesPerInt), "must be 4 byte aligned");
463 HeapWord* obj = cast_from_oop<HeapWord*>(result);
464 if (!is_aligned(hs_bytes, BytesPerLong)) {
465 *reinterpret_cast<jint*>(reinterpret_cast<char*>(obj) + hs_bytes) = 0;
466 hs_bytes += BytesPerInt;
467 }
468
469 // Optimized zeroing.
470 assert(is_aligned(hs_bytes, BytesPerLong), "must be 8-byte aligned");
471 const size_t aligned_hs = hs_bytes / BytesPerLong;
472 Copy::fill_to_aligned_words(obj+aligned_hs, size-aligned_hs);
473 }
474
475 JRT_END
476
477 // Note: multianewarray for one dimension is handled inline by GraphKit::new_array.
478
479 // multianewarray for 2 dimensions
480 JRT_ENTRY(void, OptoRuntime::multianewarray2_C(Klass* elem_type, int len1, int len2, JavaThread* current))
481 #ifndef PRODUCT
482 SharedRuntime::_multi2_ctr++; // multianewarray for 1 dimension
483 #endif
484 assert(check_compiled_frame(current), "incorrect caller");
485 assert(elem_type->is_klass(), "not a class");
486 jint dims[2];
487 dims[0] = len1;
488 dims[1] = len2;
489 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
490 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(2, dims, THREAD);
491 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
492 current->set_vm_result_oop(obj);
493 JRT_END
494
495 // multianewarray for 3 dimensions
496 JRT_ENTRY(void, OptoRuntime::multianewarray3_C(Klass* elem_type, int len1, int len2, int len3, JavaThread* current))
497 #ifndef PRODUCT
498 SharedRuntime::_multi3_ctr++; // multianewarray for 1 dimension
499 #endif
500 assert(check_compiled_frame(current), "incorrect caller");
501 assert(elem_type->is_klass(), "not a class");
502 jint dims[3];
503 dims[0] = len1;
504 dims[1] = len2;
505 dims[2] = len3;
506 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
507 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(3, dims, THREAD);
508 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
509 current->set_vm_result_oop(obj);
510 JRT_END
511
512 // multianewarray for 4 dimensions
513 JRT_ENTRY(void, OptoRuntime::multianewarray4_C(Klass* elem_type, int len1, int len2, int len3, int len4, JavaThread* current))
514 #ifndef PRODUCT
515 SharedRuntime::_multi4_ctr++; // multianewarray for 1 dimension
516 #endif
517 assert(check_compiled_frame(current), "incorrect caller");
518 assert(elem_type->is_klass(), "not a class");
519 jint dims[4];
520 dims[0] = len1;
521 dims[1] = len2;
522 dims[2] = len3;
523 dims[3] = len4;
524 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
525 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(4, dims, THREAD);
526 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
527 current->set_vm_result_oop(obj);
528 JRT_END
529
530 // multianewarray for 5 dimensions
531 JRT_ENTRY(void, OptoRuntime::multianewarray5_C(Klass* elem_type, int len1, int len2, int len3, int len4, int len5, JavaThread* current))
532 #ifndef PRODUCT
533 SharedRuntime::_multi5_ctr++; // multianewarray for 1 dimension
534 #endif
535 assert(check_compiled_frame(current), "incorrect caller");
536 assert(elem_type->is_klass(), "not a class");
537 jint dims[5];
538 dims[0] = len1;
539 dims[1] = len2;
540 dims[2] = len3;
541 dims[3] = len4;
542 dims[4] = len5;
543 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
544 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(5, dims, THREAD);
545 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
546 current->set_vm_result_oop(obj);
547 JRT_END
548
549 JRT_ENTRY(void, OptoRuntime::multianewarrayN_C(Klass* elem_type, arrayOopDesc* dims, JavaThread* current))
550 assert(check_compiled_frame(current), "incorrect caller");
551 assert(elem_type->is_klass(), "not a class");
552 assert(oop(dims)->is_typeArray(), "not an array");
553
554 ResourceMark rm;
555 jint len = dims->length();
556 assert(len > 0, "Dimensions array should contain data");
557 jint *c_dims = NEW_RESOURCE_ARRAY(jint, len);
558 ArrayAccess<>::arraycopy_to_native<>(dims, typeArrayOopDesc::element_offset<jint>(0),
559 c_dims, len);
560
561 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
562 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(len, c_dims, THREAD);
563 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
564 current->set_vm_result_oop(obj);
565 JRT_END
566
567 JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notify_C(oopDesc* obj, JavaThread* current))
568
569 // Very few notify/notifyAll operations find any threads on the waitset, so
570 // the dominant fast-path is to simply return.
571 // Relatedly, it's critical that notify/notifyAll be fast in order to
572 // reduce lock hold times.
573 if (!SafepointSynchronize::is_synchronizing()) {
574 if (ObjectSynchronizer::quick_notify(obj, current, false)) {
575 return;
576 }
577 }
578
579 // This is the case the fast-path above isn't provisioned to handle.
580 // The fast-path is designed to handle frequently arising cases in an efficient manner.
581 // (The fast-path is just a degenerate variant of the slow-path).
582 // Perform the dreaded state transition and pass control into the slow-path.
583 JRT_BLOCK;
584 Handle h_obj(current, obj);
585 ObjectSynchronizer::notify(h_obj, CHECK);
586 JRT_BLOCK_END;
587 JRT_END
588
589 JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notifyAll_C(oopDesc* obj, JavaThread* current))
590
591 if (!SafepointSynchronize::is_synchronizing() ) {
592 if (ObjectSynchronizer::quick_notify(obj, current, true)) {
593 return;
594 }
595 }
596
597 // This is the case the fast-path above isn't provisioned to handle.
598 // The fast-path is designed to handle frequently arising cases in an efficient manner.
599 // (The fast-path is just a degenerate variant of the slow-path).
600 // Perform the dreaded state transition and pass control into the slow-path.
601 JRT_BLOCK;
602 Handle h_obj(current, obj);
603 ObjectSynchronizer::notifyall(h_obj, CHECK);
604 JRT_BLOCK_END;
605 JRT_END
606
607 static const TypeFunc* make_new_instance_Type() {
608 // create input type (domain)
609 const Type **fields = TypeTuple::fields(2);
610 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Klass to be allocated
611 fields[TypeFunc::Parms+1] = TypeInt::BOOL; // is_larval
612 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
613
614 // create result type (range)
615 fields = TypeTuple::fields(1);
616 fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Returned oop
617
618 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
619
620 return TypeFunc::make(domain, range);
621 }
622
623 #if INCLUDE_JVMTI
624 static const TypeFunc* make_notify_jvmti_vthread_Type() {
625 // create input type (domain)
626 const Type **fields = TypeTuple::fields(2);
627 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // VirtualThread oop
628 fields[TypeFunc::Parms+1] = TypeInt::BOOL; // jboolean
629 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
630
631 // no result type needed
632 fields = TypeTuple::fields(1);
633 fields[TypeFunc::Parms+0] = nullptr; // void
634 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
635
636 return TypeFunc::make(domain,range);
637 }
638 #endif
639
640 static const TypeFunc* make_athrow_Type() {
641 // create input type (domain)
642 const Type **fields = TypeTuple::fields(1);
643 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Klass to be allocated
644 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
645
646 // create result type (range)
647 fields = TypeTuple::fields(0);
648
649 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
650
651 return TypeFunc::make(domain, range);
652 }
653
654 static const TypeFunc* make_new_array_Type() {
655 // create input type (domain)
656 const Type **fields = TypeTuple::fields(3);
657 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // element klass
658 fields[TypeFunc::Parms+1] = TypeInt::INT; // array size
659 fields[TypeFunc::Parms+2] = TypeInstPtr::NOTNULL; // init value
660 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+3, fields);
661
662 // create result type (range)
663 fields = TypeTuple::fields(1);
664 fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Returned oop
665
666 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
667
668 return TypeFunc::make(domain, range);
669 }
670
671 static const TypeFunc* make_new_array_nozero_Type() {
672 // create input type (domain)
673 const Type **fields = TypeTuple::fields(2);
674 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // element klass
675 fields[TypeFunc::Parms+1] = TypeInt::INT; // array size
676 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
677
678 // create result type (range)
679 fields = TypeTuple::fields(1);
680 fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Returned oop
681
682 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
683
684 return TypeFunc::make(domain, range);
685 }
686
687 const TypeFunc* OptoRuntime::multianewarray_Type(int ndim) {
688 // create input type (domain)
689 const int nargs = ndim + 1;
690 const Type **fields = TypeTuple::fields(nargs);
691 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // element klass
692 for( int i = 1; i < nargs; i++ )
693 fields[TypeFunc::Parms + i] = TypeInt::INT; // array size
694 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+nargs, fields);
695
696 // create result type (range)
697 fields = TypeTuple::fields(1);
698 fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Returned oop
699 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
700
701 return TypeFunc::make(domain, range);
702 }
703
704 static const TypeFunc* make_multianewarrayN_Type() {
705 // create input type (domain)
706 const Type **fields = TypeTuple::fields(2);
707 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // element klass
708 fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL; // array of dim sizes
709 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
710
711 // create result type (range)
712 fields = TypeTuple::fields(1);
713 fields[TypeFunc::Parms+0] = TypeRawPtr::NOTNULL; // Returned oop
714 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
715
716 return TypeFunc::make(domain, range);
717 }
718
719 static const TypeFunc* make_uncommon_trap_Type() {
720 // create input type (domain)
721 const Type **fields = TypeTuple::fields(1);
722 fields[TypeFunc::Parms+0] = TypeInt::INT; // trap_reason (deopt reason and action)
723 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
724
725 // create result type (range)
726 fields = TypeTuple::fields(0);
727 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
728
729 return TypeFunc::make(domain, range);
730 }
731
732 //-----------------------------------------------------------------------------
733 // Monitor Handling
734
735 static const TypeFunc* make_complete_monitor_enter_Type() {
736 // create input type (domain)
737 const Type **fields = TypeTuple::fields(2);
738 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Object to be Locked
739 fields[TypeFunc::Parms+1] = TypeRawPtr::BOTTOM; // Address of stack location for lock
740 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
741
742 // create result type (range)
743 fields = TypeTuple::fields(0);
744
745 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
746
747 return TypeFunc::make(domain, range);
748 }
749
750 //-----------------------------------------------------------------------------
751
752 static const TypeFunc* make_complete_monitor_exit_Type() {
753 // create input type (domain)
754 const Type **fields = TypeTuple::fields(3);
755 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Object to be Locked
756 fields[TypeFunc::Parms+1] = TypeRawPtr::BOTTOM; // Address of stack location for lock - BasicLock
757 fields[TypeFunc::Parms+2] = TypeRawPtr::BOTTOM; // Thread pointer (Self)
758 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+3, fields);
759
760 // create result type (range)
761 fields = TypeTuple::fields(0);
762
763 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
764
765 return TypeFunc::make(domain, range);
766 }
767
768 static const TypeFunc* make_monitor_notify_Type() {
769 // create input type (domain)
770 const Type **fields = TypeTuple::fields(1);
771 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Object to be Locked
772 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
773
774 // create result type (range)
775 fields = TypeTuple::fields(0);
776 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
777 return TypeFunc::make(domain, range);
778 }
779
780 static const TypeFunc* make_flush_windows_Type() {
781 // create input type (domain)
782 const Type** fields = TypeTuple::fields(1);
783 fields[TypeFunc::Parms+0] = nullptr; // void
784 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields);
785
786 // create result type
787 fields = TypeTuple::fields(1);
788 fields[TypeFunc::Parms+0] = nullptr; // void
789 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields);
790
791 return TypeFunc::make(domain, range);
792 }
793
794 static const TypeFunc* make_l2f_Type() {
795 // create input type (domain)
796 const Type **fields = TypeTuple::fields(2);
797 fields[TypeFunc::Parms+0] = TypeLong::LONG;
798 fields[TypeFunc::Parms+1] = Type::HALF;
799 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
800
801 // create result type (range)
802 fields = TypeTuple::fields(1);
803 fields[TypeFunc::Parms+0] = Type::FLOAT;
804 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
805
806 return TypeFunc::make(domain, range);
807 }
808
809 static const TypeFunc* make_modf_Type() {
810 const Type **fields = TypeTuple::fields(2);
811 fields[TypeFunc::Parms+0] = Type::FLOAT;
812 fields[TypeFunc::Parms+1] = Type::FLOAT;
813 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
814
815 // create result type (range)
816 fields = TypeTuple::fields(1);
817 fields[TypeFunc::Parms+0] = Type::FLOAT;
818
819 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
820
821 return TypeFunc::make(domain, range);
822 }
823
824 static const TypeFunc* make_Math_D_D_Type() {
825 // create input type (domain)
826 const Type **fields = TypeTuple::fields(2);
827 // Symbol* name of class to be loaded
828 fields[TypeFunc::Parms+0] = Type::DOUBLE;
829 fields[TypeFunc::Parms+1] = Type::HALF;
830 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2, fields);
831
832 // create result type (range)
833 fields = TypeTuple::fields(2);
834 fields[TypeFunc::Parms+0] = Type::DOUBLE;
835 fields[TypeFunc::Parms+1] = Type::HALF;
836 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+2, fields);
837
838 return TypeFunc::make(domain, range);
839 }
840
841 const TypeFunc* OptoRuntime::Math_Vector_Vector_Type(uint num_arg, const TypeVect* in_type, const TypeVect* out_type) {
842 // create input type (domain)
843 const Type **fields = TypeTuple::fields(num_arg);
844 // Symbol* name of class to be loaded
845 assert(num_arg > 0, "must have at least 1 input");
846 for (uint i = 0; i < num_arg; i++) {
847 fields[TypeFunc::Parms+i] = in_type;
848 }
849 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+num_arg, fields);
850
851 // create result type (range)
852 const uint num_ret = 1;
853 fields = TypeTuple::fields(num_ret);
854 fields[TypeFunc::Parms+0] = out_type;
855 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+num_ret, fields);
856
857 return TypeFunc::make(domain, range);
858 }
859
860 static const TypeFunc* make_Math_DD_D_Type() {
861 const Type **fields = TypeTuple::fields(4);
862 fields[TypeFunc::Parms+0] = Type::DOUBLE;
863 fields[TypeFunc::Parms+1] = Type::HALF;
864 fields[TypeFunc::Parms+2] = Type::DOUBLE;
865 fields[TypeFunc::Parms+3] = Type::HALF;
866 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+4, fields);
867
868 // create result type (range)
869 fields = TypeTuple::fields(2);
870 fields[TypeFunc::Parms+0] = Type::DOUBLE;
871 fields[TypeFunc::Parms+1] = Type::HALF;
872 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+2, fields);
873
874 return TypeFunc::make(domain, range);
875 }
876
877 //-------------- currentTimeMillis, currentTimeNanos, etc
878
879 static const TypeFunc* make_void_long_Type() {
880 // create input type (domain)
881 const Type **fields = TypeTuple::fields(0);
882 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+0, fields);
883
884 // create result type (range)
885 fields = TypeTuple::fields(2);
886 fields[TypeFunc::Parms+0] = TypeLong::LONG;
887 fields[TypeFunc::Parms+1] = Type::HALF;
888 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+2, fields);
889
890 return TypeFunc::make(domain, range);
891 }
892
893 static const TypeFunc* make_void_void_Type() {
894 // create input type (domain)
895 const Type **fields = TypeTuple::fields(0);
896 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+0, fields);
897
898 // create result type (range)
899 fields = TypeTuple::fields(0);
900 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
901 return TypeFunc::make(domain, range);
902 }
903
904 static const TypeFunc* make_jfr_write_checkpoint_Type() {
905 // create input type (domain)
906 const Type **fields = TypeTuple::fields(0);
907 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms, fields);
908
909 // create result type (range)
910 fields = TypeTuple::fields(0);
911 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields);
912 return TypeFunc::make(domain, range);
913 }
914
915
916 // Takes as parameters:
917 // void *dest
918 // long size
919 // uchar byte
920
921 static const TypeFunc* make_setmemory_Type() {
922 // create input type (domain)
923 int argcnt = NOT_LP64(3) LP64_ONLY(4);
924 const Type** fields = TypeTuple::fields(argcnt);
925 int argp = TypeFunc::Parms;
926 fields[argp++] = TypePtr::NOTNULL; // dest
927 fields[argp++] = TypeX_X; // size
928 LP64_ONLY(fields[argp++] = Type::HALF); // size
929 fields[argp++] = TypeInt::UBYTE; // bytevalue
930 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
931 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
932
933 // no result type needed
934 fields = TypeTuple::fields(1);
935 fields[TypeFunc::Parms+0] = nullptr; // void
936 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
937 return TypeFunc::make(domain, range);
938 }
939
940 // arraycopy stub variations:
941 enum ArrayCopyType {
942 ac_fast, // void(ptr, ptr, size_t)
943 ac_checkcast, // int(ptr, ptr, size_t, size_t, ptr)
944 ac_slow, // void(ptr, int, ptr, int, int)
945 ac_generic // int(ptr, int, ptr, int, int)
946 };
947
948 static const TypeFunc* make_arraycopy_Type(ArrayCopyType act) {
949 // create input type (domain)
950 int num_args = (act == ac_fast ? 3 : 5);
951 int num_size_args = (act == ac_fast ? 1 : act == ac_checkcast ? 2 : 0);
952 int argcnt = num_args;
953 LP64_ONLY(argcnt += num_size_args); // halfwords for lengths
954 const Type** fields = TypeTuple::fields(argcnt);
955 int argp = TypeFunc::Parms;
956 fields[argp++] = TypePtr::NOTNULL; // src
957 if (num_size_args == 0) {
958 fields[argp++] = TypeInt::INT; // src_pos
959 }
960 fields[argp++] = TypePtr::NOTNULL; // dest
961 if (num_size_args == 0) {
962 fields[argp++] = TypeInt::INT; // dest_pos
963 fields[argp++] = TypeInt::INT; // length
964 }
965 while (num_size_args-- > 0) {
966 fields[argp++] = TypeX_X; // size in whatevers (size_t)
967 LP64_ONLY(fields[argp++] = Type::HALF); // other half of long length
968 }
969 if (act == ac_checkcast) {
970 fields[argp++] = TypePtr::NOTNULL; // super_klass
971 }
972 assert(argp == TypeFunc::Parms+argcnt, "correct decoding of act");
973 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
974
975 // create result type if needed
976 int retcnt = (act == ac_checkcast || act == ac_generic ? 1 : 0);
977 fields = TypeTuple::fields(1);
978 if (retcnt == 0)
979 fields[TypeFunc::Parms+0] = nullptr; // void
980 else
981 fields[TypeFunc::Parms+0] = TypeInt::INT; // status result, if needed
982 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+retcnt, fields);
983 return TypeFunc::make(domain, range);
984 }
985
986 static const TypeFunc* make_array_fill_Type() {
987 const Type** fields;
988 int argp = TypeFunc::Parms;
989 // create input type (domain): pointer, int, size_t
990 fields = TypeTuple::fields(3 LP64_ONLY( + 1));
991 fields[argp++] = TypePtr::NOTNULL;
992 fields[argp++] = TypeInt::INT;
993 fields[argp++] = TypeX_X; // size in whatevers (size_t)
994 LP64_ONLY(fields[argp++] = Type::HALF); // other half of long length
995 const TypeTuple *domain = TypeTuple::make(argp, fields);
996
997 // create result type
998 fields = TypeTuple::fields(1);
999 fields[TypeFunc::Parms+0] = nullptr; // void
1000 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields);
1001
1002 return TypeFunc::make(domain, range);
1003 }
1004
1005 static const TypeFunc* make_array_partition_Type() {
1006 // create input type (domain)
1007 int num_args = 7;
1008 int argcnt = num_args;
1009 const Type** fields = TypeTuple::fields(argcnt);
1010 int argp = TypeFunc::Parms;
1011 fields[argp++] = TypePtr::NOTNULL; // array
1012 fields[argp++] = TypeInt::INT; // element type
1013 fields[argp++] = TypeInt::INT; // low
1014 fields[argp++] = TypeInt::INT; // end
1015 fields[argp++] = TypePtr::NOTNULL; // pivot_indices (int array)
1016 fields[argp++] = TypeInt::INT; // indexPivot1
1017 fields[argp++] = TypeInt::INT; // indexPivot2
1018 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1019 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1020
1021 // no result type needed
1022 fields = TypeTuple::fields(1);
1023 fields[TypeFunc::Parms+0] = nullptr; // void
1024 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1025 return TypeFunc::make(domain, range);
1026 }
1027
1028 static const TypeFunc* make_array_sort_Type() {
1029 // create input type (domain)
1030 int num_args = 4;
1031 int argcnt = num_args;
1032 const Type** fields = TypeTuple::fields(argcnt);
1033 int argp = TypeFunc::Parms;
1034 fields[argp++] = TypePtr::NOTNULL; // array
1035 fields[argp++] = TypeInt::INT; // element type
1036 fields[argp++] = TypeInt::INT; // fromIndex
1037 fields[argp++] = TypeInt::INT; // toIndex
1038 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1039 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1040
1041 // no result type needed
1042 fields = TypeTuple::fields(1);
1043 fields[TypeFunc::Parms+0] = nullptr; // void
1044 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1045 return TypeFunc::make(domain, range);
1046 }
1047
1048 static const TypeFunc* make_aescrypt_block_Type() {
1049 // create input type (domain)
1050 int num_args = 3;
1051 int argcnt = num_args;
1052 const Type** fields = TypeTuple::fields(argcnt);
1053 int argp = TypeFunc::Parms;
1054 fields[argp++] = TypePtr::NOTNULL; // src
1055 fields[argp++] = TypePtr::NOTNULL; // dest
1056 fields[argp++] = TypePtr::NOTNULL; // k array
1057 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1058 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1059
1060 // no result type needed
1061 fields = TypeTuple::fields(1);
1062 fields[TypeFunc::Parms+0] = nullptr; // void
1063 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1064 return TypeFunc::make(domain, range);
1065 }
1066
1067 static const TypeFunc* make_updateBytesCRC32_Type() {
1068 // create input type (domain)
1069 int num_args = 3;
1070 int argcnt = num_args;
1071 const Type** fields = TypeTuple::fields(argcnt);
1072 int argp = TypeFunc::Parms;
1073 fields[argp++] = TypeInt::INT; // crc
1074 fields[argp++] = TypePtr::NOTNULL; // src
1075 fields[argp++] = TypeInt::INT; // len
1076 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1077 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1078
1079 // result type needed
1080 fields = TypeTuple::fields(1);
1081 fields[TypeFunc::Parms+0] = TypeInt::INT; // crc result
1082 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
1083 return TypeFunc::make(domain, range);
1084 }
1085
1086 static const TypeFunc* make_updateBytesCRC32C_Type() {
1087 // create input type (domain)
1088 int num_args = 4;
1089 int argcnt = num_args;
1090 const Type** fields = TypeTuple::fields(argcnt);
1091 int argp = TypeFunc::Parms;
1092 fields[argp++] = TypeInt::INT; // crc
1093 fields[argp++] = TypePtr::NOTNULL; // buf
1094 fields[argp++] = TypeInt::INT; // len
1095 fields[argp++] = TypePtr::NOTNULL; // table
1096 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1097 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1098
1099 // result type needed
1100 fields = TypeTuple::fields(1);
1101 fields[TypeFunc::Parms+0] = TypeInt::INT; // crc result
1102 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
1103 return TypeFunc::make(domain, range);
1104 }
1105
1106 static const TypeFunc* make_updateBytesAdler32_Type() {
1107 // create input type (domain)
1108 int num_args = 3;
1109 int argcnt = num_args;
1110 const Type** fields = TypeTuple::fields(argcnt);
1111 int argp = TypeFunc::Parms;
1112 fields[argp++] = TypeInt::INT; // crc
1113 fields[argp++] = TypePtr::NOTNULL; // src + offset
1114 fields[argp++] = TypeInt::INT; // len
1115 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1116 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1117
1118 // result type needed
1119 fields = TypeTuple::fields(1);
1120 fields[TypeFunc::Parms+0] = TypeInt::INT; // crc result
1121 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
1122 return TypeFunc::make(domain, range);
1123 }
1124
1125 static const TypeFunc* make_cipherBlockChaining_aescrypt_Type() {
1126 // create input type (domain)
1127 int num_args = 5;
1128 int argcnt = num_args;
1129 const Type** fields = TypeTuple::fields(argcnt);
1130 int argp = TypeFunc::Parms;
1131 fields[argp++] = TypePtr::NOTNULL; // src
1132 fields[argp++] = TypePtr::NOTNULL; // dest
1133 fields[argp++] = TypePtr::NOTNULL; // k array
1134 fields[argp++] = TypePtr::NOTNULL; // r array
1135 fields[argp++] = TypeInt::INT; // src len
1136 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1137 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1138
1139 // returning cipher len (int)
1140 fields = TypeTuple::fields(1);
1141 fields[TypeFunc::Parms+0] = TypeInt::INT;
1142 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
1143 return TypeFunc::make(domain, range);
1144 }
1145
1146 static const TypeFunc* make_electronicCodeBook_aescrypt_Type() {
1147 // create input type (domain)
1148 int num_args = 4;
1149 int argcnt = num_args;
1150 const Type** fields = TypeTuple::fields(argcnt);
1151 int argp = TypeFunc::Parms;
1152 fields[argp++] = TypePtr::NOTNULL; // src
1153 fields[argp++] = TypePtr::NOTNULL; // dest
1154 fields[argp++] = TypePtr::NOTNULL; // k array
1155 fields[argp++] = TypeInt::INT; // src len
1156 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1157 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1158
1159 // returning cipher len (int)
1160 fields = TypeTuple::fields(1);
1161 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1162 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1163 return TypeFunc::make(domain, range);
1164 }
1165
1166 static const TypeFunc* make_counterMode_aescrypt_Type() {
1167 // create input type (domain)
1168 int num_args = 7;
1169 int argcnt = num_args;
1170 const Type** fields = TypeTuple::fields(argcnt);
1171 int argp = TypeFunc::Parms;
1172 fields[argp++] = TypePtr::NOTNULL; // src
1173 fields[argp++] = TypePtr::NOTNULL; // dest
1174 fields[argp++] = TypePtr::NOTNULL; // k array
1175 fields[argp++] = TypePtr::NOTNULL; // counter array
1176 fields[argp++] = TypeInt::INT; // src len
1177 fields[argp++] = TypePtr::NOTNULL; // saved_encCounter
1178 fields[argp++] = TypePtr::NOTNULL; // saved used addr
1179 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1180 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1181 // returning cipher len (int)
1182 fields = TypeTuple::fields(1);
1183 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1184 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1185 return TypeFunc::make(domain, range);
1186 }
1187
1188 static const TypeFunc* make_galoisCounterMode_aescrypt_Type() {
1189 // create input type (domain)
1190 int num_args = 8;
1191 int argcnt = num_args;
1192 const Type** fields = TypeTuple::fields(argcnt);
1193 int argp = TypeFunc::Parms;
1194 fields[argp++] = TypePtr::NOTNULL; // byte[] in + inOfs
1195 fields[argp++] = TypeInt::INT; // int len
1196 fields[argp++] = TypePtr::NOTNULL; // byte[] ct + ctOfs
1197 fields[argp++] = TypePtr::NOTNULL; // byte[] out + outOfs
1198 fields[argp++] = TypePtr::NOTNULL; // byte[] key from AESCrypt obj
1199 fields[argp++] = TypePtr::NOTNULL; // long[] state from GHASH obj
1200 fields[argp++] = TypePtr::NOTNULL; // long[] subkeyHtbl from GHASH obj
1201 fields[argp++] = TypePtr::NOTNULL; // byte[] counter from GCTR obj
1202
1203 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1204 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1205 // returning cipher len (int)
1206 fields = TypeTuple::fields(1);
1207 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1208 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1209 return TypeFunc::make(domain, range);
1210 }
1211
1212 static const TypeFunc* make_digestBase_implCompress_Type(bool is_sha3) {
1213 // create input type (domain)
1214 int num_args = is_sha3 ? 3 : 2;
1215 int argcnt = num_args;
1216 const Type** fields = TypeTuple::fields(argcnt);
1217 int argp = TypeFunc::Parms;
1218 fields[argp++] = TypePtr::NOTNULL; // buf
1219 fields[argp++] = TypePtr::NOTNULL; // state
1220 if (is_sha3) fields[argp++] = TypeInt::INT; // block_size
1221 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1222 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1223
1224 // no result type needed
1225 fields = TypeTuple::fields(1);
1226 fields[TypeFunc::Parms+0] = nullptr; // void
1227 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1228 return TypeFunc::make(domain, range);
1229 }
1230
1231 /*
1232 * int implCompressMultiBlock(byte[] b, int ofs, int limit)
1233 */
1234 static const TypeFunc* make_digestBase_implCompressMB_Type(bool is_sha3) {
1235 // create input type (domain)
1236 int num_args = is_sha3 ? 5 : 4;
1237 int argcnt = num_args;
1238 const Type** fields = TypeTuple::fields(argcnt);
1239 int argp = TypeFunc::Parms;
1240 fields[argp++] = TypePtr::NOTNULL; // buf
1241 fields[argp++] = TypePtr::NOTNULL; // state
1242 if (is_sha3) fields[argp++] = TypeInt::INT; // block_size
1243 fields[argp++] = TypeInt::INT; // ofs
1244 fields[argp++] = TypeInt::INT; // limit
1245 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1246 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1247
1248 // returning ofs (int)
1249 fields = TypeTuple::fields(1);
1250 fields[TypeFunc::Parms+0] = TypeInt::INT; // ofs
1251 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
1252 return TypeFunc::make(domain, range);
1253 }
1254
1255 // SHAKE128Parallel doubleKeccak function
1256 static const TypeFunc* make_double_keccak_Type() {
1257 int argcnt = 2;
1258
1259 const Type** fields = TypeTuple::fields(argcnt);
1260 int argp = TypeFunc::Parms;
1261 fields[argp++] = TypePtr::NOTNULL; // status0
1262 fields[argp++] = TypePtr::NOTNULL; // status1
1263
1264 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1265 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1266
1267 // result type needed
1268 fields = TypeTuple::fields(1);
1269 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1270 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1271 return TypeFunc::make(domain, range);
1272 }
1273
1274 static const TypeFunc* make_multiplyToLen_Type() {
1275 // create input type (domain)
1276 int num_args = 5;
1277 int argcnt = num_args;
1278 const Type** fields = TypeTuple::fields(argcnt);
1279 int argp = TypeFunc::Parms;
1280 fields[argp++] = TypePtr::NOTNULL; // x
1281 fields[argp++] = TypeInt::INT; // xlen
1282 fields[argp++] = TypePtr::NOTNULL; // y
1283 fields[argp++] = TypeInt::INT; // ylen
1284 fields[argp++] = TypePtr::NOTNULL; // z
1285 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1286 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1287
1288 // no result type needed
1289 fields = TypeTuple::fields(1);
1290 fields[TypeFunc::Parms+0] = nullptr;
1291 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1292 return TypeFunc::make(domain, range);
1293 }
1294
1295 static const TypeFunc* make_squareToLen_Type() {
1296 // create input type (domain)
1297 int num_args = 4;
1298 int argcnt = num_args;
1299 const Type** fields = TypeTuple::fields(argcnt);
1300 int argp = TypeFunc::Parms;
1301 fields[argp++] = TypePtr::NOTNULL; // x
1302 fields[argp++] = TypeInt::INT; // len
1303 fields[argp++] = TypePtr::NOTNULL; // z
1304 fields[argp++] = TypeInt::INT; // zlen
1305 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1306 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1307
1308 // no result type needed
1309 fields = TypeTuple::fields(1);
1310 fields[TypeFunc::Parms+0] = nullptr;
1311 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1312 return TypeFunc::make(domain, range);
1313 }
1314
1315 static const TypeFunc* make_mulAdd_Type() {
1316 // create input type (domain)
1317 int num_args = 5;
1318 int argcnt = num_args;
1319 const Type** fields = TypeTuple::fields(argcnt);
1320 int argp = TypeFunc::Parms;
1321 fields[argp++] = TypePtr::NOTNULL; // out
1322 fields[argp++] = TypePtr::NOTNULL; // in
1323 fields[argp++] = TypeInt::INT; // offset
1324 fields[argp++] = TypeInt::INT; // len
1325 fields[argp++] = TypeInt::INT; // k
1326 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1327 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1328
1329 // returning carry (int)
1330 fields = TypeTuple::fields(1);
1331 fields[TypeFunc::Parms+0] = TypeInt::INT;
1332 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
1333 return TypeFunc::make(domain, range);
1334 }
1335
1336 static const TypeFunc* make_montgomeryMultiply_Type() {
1337 // create input type (domain)
1338 int num_args = 7;
1339 int argcnt = num_args;
1340 const Type** fields = TypeTuple::fields(argcnt);
1341 int argp = TypeFunc::Parms;
1342 fields[argp++] = TypePtr::NOTNULL; // a
1343 fields[argp++] = TypePtr::NOTNULL; // b
1344 fields[argp++] = TypePtr::NOTNULL; // n
1345 fields[argp++] = TypeInt::INT; // len
1346 fields[argp++] = TypeLong::LONG; // inv
1347 fields[argp++] = Type::HALF;
1348 fields[argp++] = TypePtr::NOTNULL; // result
1349 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1350 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1351
1352 // result type needed
1353 fields = TypeTuple::fields(1);
1354 fields[TypeFunc::Parms+0] = TypePtr::NOTNULL;
1355
1356 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1357 return TypeFunc::make(domain, range);
1358 }
1359
1360 static const TypeFunc* make_montgomerySquare_Type() {
1361 // create input type (domain)
1362 int num_args = 6;
1363 int argcnt = num_args;
1364 const Type** fields = TypeTuple::fields(argcnt);
1365 int argp = TypeFunc::Parms;
1366 fields[argp++] = TypePtr::NOTNULL; // a
1367 fields[argp++] = TypePtr::NOTNULL; // n
1368 fields[argp++] = TypeInt::INT; // len
1369 fields[argp++] = TypeLong::LONG; // inv
1370 fields[argp++] = Type::HALF;
1371 fields[argp++] = TypePtr::NOTNULL; // result
1372 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1373 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1374
1375 // result type needed
1376 fields = TypeTuple::fields(1);
1377 fields[TypeFunc::Parms+0] = TypePtr::NOTNULL;
1378
1379 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1380 return TypeFunc::make(domain, range);
1381 }
1382
1383 static const TypeFunc* make_bigIntegerShift_Type() {
1384 int argcnt = 5;
1385 const Type** fields = TypeTuple::fields(argcnt);
1386 int argp = TypeFunc::Parms;
1387 fields[argp++] = TypePtr::NOTNULL; // newArr
1388 fields[argp++] = TypePtr::NOTNULL; // oldArr
1389 fields[argp++] = TypeInt::INT; // newIdx
1390 fields[argp++] = TypeInt::INT; // shiftCount
1391 fields[argp++] = TypeInt::INT; // numIter
1392 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1393 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1394
1395 // no result type needed
1396 fields = TypeTuple::fields(1);
1397 fields[TypeFunc::Parms + 0] = nullptr;
1398 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1399 return TypeFunc::make(domain, range);
1400 }
1401
1402 static const TypeFunc* make_vectorizedMismatch_Type() {
1403 // create input type (domain)
1404 int num_args = 4;
1405 int argcnt = num_args;
1406 const Type** fields = TypeTuple::fields(argcnt);
1407 int argp = TypeFunc::Parms;
1408 fields[argp++] = TypePtr::NOTNULL; // obja
1409 fields[argp++] = TypePtr::NOTNULL; // objb
1410 fields[argp++] = TypeInt::INT; // length, number of elements
1411 fields[argp++] = TypeInt::INT; // log2scale, element size
1412 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1413 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1414
1415 //return mismatch index (int)
1416 fields = TypeTuple::fields(1);
1417 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1418 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1419 return TypeFunc::make(domain, range);
1420 }
1421
1422 static const TypeFunc* make_ghash_processBlocks_Type() {
1423 int argcnt = 4;
1424
1425 const Type** fields = TypeTuple::fields(argcnt);
1426 int argp = TypeFunc::Parms;
1427 fields[argp++] = TypePtr::NOTNULL; // state
1428 fields[argp++] = TypePtr::NOTNULL; // subkeyH
1429 fields[argp++] = TypePtr::NOTNULL; // data
1430 fields[argp++] = TypeInt::INT; // blocks
1431 assert(argp == TypeFunc::Parms+argcnt, "correct decoding");
1432 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1433
1434 // result type needed
1435 fields = TypeTuple::fields(1);
1436 fields[TypeFunc::Parms+0] = nullptr; // void
1437 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1438 return TypeFunc::make(domain, range);
1439 }
1440
1441 static const TypeFunc* make_chacha20Block_Type() {
1442 int argcnt = 2;
1443
1444 const Type** fields = TypeTuple::fields(argcnt);
1445 int argp = TypeFunc::Parms;
1446 fields[argp++] = TypePtr::NOTNULL; // state
1447 fields[argp++] = TypePtr::NOTNULL; // result
1448
1449 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1450 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1451
1452 // result type needed
1453 fields = TypeTuple::fields(1);
1454 fields[TypeFunc::Parms + 0] = TypeInt::INT; // key stream outlen as int
1455 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1456 return TypeFunc::make(domain, range);
1457 }
1458
1459 // Kyber NTT function
1460 static const TypeFunc* make_kyberNtt_Type() {
1461 int argcnt = 2;
1462
1463 const Type** fields = TypeTuple::fields(argcnt);
1464 int argp = TypeFunc::Parms;
1465 fields[argp++] = TypePtr::NOTNULL; // coeffs
1466 fields[argp++] = TypePtr::NOTNULL; // NTT zetas
1467
1468 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1469 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1470
1471 // result type needed
1472 fields = TypeTuple::fields(1);
1473 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1474 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1475 return TypeFunc::make(domain, range);
1476 }
1477
1478 // Kyber inverse NTT function
1479 static const TypeFunc* make_kyberInverseNtt_Type() {
1480 int argcnt = 2;
1481
1482 const Type** fields = TypeTuple::fields(argcnt);
1483 int argp = TypeFunc::Parms;
1484 fields[argp++] = TypePtr::NOTNULL; // coeffs
1485 fields[argp++] = TypePtr::NOTNULL; // inverse NTT zetas
1486
1487 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1488 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1489
1490 // result type needed
1491 fields = TypeTuple::fields(1);
1492 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1493 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1494 return TypeFunc::make(domain, range);
1495 }
1496
1497 // Kyber NTT multiply function
1498 static const TypeFunc* make_kyberNttMult_Type() {
1499 int argcnt = 4;
1500
1501 const Type** fields = TypeTuple::fields(argcnt);
1502 int argp = TypeFunc::Parms;
1503 fields[argp++] = TypePtr::NOTNULL; // result
1504 fields[argp++] = TypePtr::NOTNULL; // ntta
1505 fields[argp++] = TypePtr::NOTNULL; // nttb
1506 fields[argp++] = TypePtr::NOTNULL; // NTT multiply zetas
1507
1508 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1509 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1510
1511 // result type needed
1512 fields = TypeTuple::fields(1);
1513 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1514 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1515 return TypeFunc::make(domain, range);
1516 }
1517
1518 // Kyber add 2 polynomials function
1519 static const TypeFunc* make_kyberAddPoly_2_Type() {
1520 int argcnt = 3;
1521
1522 const Type** fields = TypeTuple::fields(argcnt);
1523 int argp = TypeFunc::Parms;
1524 fields[argp++] = TypePtr::NOTNULL; // result
1525 fields[argp++] = TypePtr::NOTNULL; // a
1526 fields[argp++] = TypePtr::NOTNULL; // b
1527
1528 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1529 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1530
1531 // result type needed
1532 fields = TypeTuple::fields(1);
1533 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1534 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1535 return TypeFunc::make(domain, range);
1536 }
1537
1538
1539 // Kyber add 3 polynomials function
1540 static const TypeFunc* make_kyberAddPoly_3_Type() {
1541 int argcnt = 4;
1542
1543 const Type** fields = TypeTuple::fields(argcnt);
1544 int argp = TypeFunc::Parms;
1545 fields[argp++] = TypePtr::NOTNULL; // result
1546 fields[argp++] = TypePtr::NOTNULL; // a
1547 fields[argp++] = TypePtr::NOTNULL; // b
1548 fields[argp++] = TypePtr::NOTNULL; // c
1549
1550 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1551 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1552
1553 // result type needed
1554 fields = TypeTuple::fields(1);
1555 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1556 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1557 return TypeFunc::make(domain, range);
1558 }
1559
1560
1561 // Kyber XOF output parsing into polynomial coefficients candidates
1562 // or decompress(12,...) function
1563 static const TypeFunc* make_kyber12To16_Type() {
1564 int argcnt = 4;
1565
1566 const Type** fields = TypeTuple::fields(argcnt);
1567 int argp = TypeFunc::Parms;
1568 fields[argp++] = TypePtr::NOTNULL; // condensed
1569 fields[argp++] = TypeInt::INT; // condensedOffs
1570 fields[argp++] = TypePtr::NOTNULL; // parsed
1571 fields[argp++] = TypeInt::INT; // parsedLength
1572
1573 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1574 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1575
1576 // result type needed
1577 fields = TypeTuple::fields(1);
1578 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1579 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1580 return TypeFunc::make(domain, range);
1581 }
1582
1583 // Kyber Barrett reduce function
1584 static const TypeFunc* make_kyberBarrettReduce_Type() {
1585 int argcnt = 1;
1586
1587 const Type** fields = TypeTuple::fields(argcnt);
1588 int argp = TypeFunc::Parms;
1589 fields[argp++] = TypePtr::NOTNULL; // coeffs
1590
1591 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1592 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1593
1594 // result type needed
1595 fields = TypeTuple::fields(1);
1596 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1597 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1598 return TypeFunc::make(domain, range);
1599 }
1600
1601 // Dilithium NTT function except for the final "normalization" to |coeff| < Q
1602 static const TypeFunc* make_dilithiumAlmostNtt_Type() {
1603 int argcnt = 2;
1604
1605 const Type** fields = TypeTuple::fields(argcnt);
1606 int argp = TypeFunc::Parms;
1607 fields[argp++] = TypePtr::NOTNULL; // coeffs
1608 fields[argp++] = TypePtr::NOTNULL; // NTT zetas
1609
1610 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1611 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1612
1613 // result type needed
1614 fields = TypeTuple::fields(1);
1615 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1616 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1617 return TypeFunc::make(domain, range);
1618 }
1619
1620 // Dilithium inverse NTT function except the final mod Q division by 2^256
1621 static const TypeFunc* make_dilithiumAlmostInverseNtt_Type() {
1622 int argcnt = 2;
1623
1624 const Type** fields = TypeTuple::fields(argcnt);
1625 int argp = TypeFunc::Parms;
1626 fields[argp++] = TypePtr::NOTNULL; // coeffs
1627 fields[argp++] = TypePtr::NOTNULL; // inverse NTT zetas
1628
1629 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1630 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1631
1632 // result type needed
1633 fields = TypeTuple::fields(1);
1634 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1635 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1636 return TypeFunc::make(domain, range);
1637 }
1638
1639 // Dilithium NTT multiply function
1640 static const TypeFunc* make_dilithiumNttMult_Type() {
1641 int argcnt = 3;
1642
1643 const Type** fields = TypeTuple::fields(argcnt);
1644 int argp = TypeFunc::Parms;
1645 fields[argp++] = TypePtr::NOTNULL; // result
1646 fields[argp++] = TypePtr::NOTNULL; // ntta
1647 fields[argp++] = TypePtr::NOTNULL; // nttb
1648
1649 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1650 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1651
1652 // result type needed
1653 fields = TypeTuple::fields(1);
1654 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1655 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1656 return TypeFunc::make(domain, range);
1657 }
1658
1659 // Dilithium Montgomery multiply a polynome coefficient array by a constant
1660 static const TypeFunc* make_dilithiumMontMulByConstant_Type() {
1661 int argcnt = 2;
1662
1663 const Type** fields = TypeTuple::fields(argcnt);
1664 int argp = TypeFunc::Parms;
1665 fields[argp++] = TypePtr::NOTNULL; // coeffs
1666 fields[argp++] = TypeInt::INT; // constant multiplier
1667
1668 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1669 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1670
1671 // result type needed
1672 fields = TypeTuple::fields(1);
1673 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1674 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1675 return TypeFunc::make(domain, range);
1676 }
1677
1678 // Dilithium decompose polynomial
1679 static const TypeFunc* make_dilithiumDecomposePoly_Type() {
1680 int argcnt = 5;
1681
1682 const Type** fields = TypeTuple::fields(argcnt);
1683 int argp = TypeFunc::Parms;
1684 fields[argp++] = TypePtr::NOTNULL; // input
1685 fields[argp++] = TypePtr::NOTNULL; // lowPart
1686 fields[argp++] = TypePtr::NOTNULL; // highPart
1687 fields[argp++] = TypeInt::INT; // 2 * gamma2
1688 fields[argp++] = TypeInt::INT; // multiplier
1689
1690 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1691 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + argcnt, fields);
1692
1693 // result type needed
1694 fields = TypeTuple::fields(1);
1695 fields[TypeFunc::Parms + 0] = TypeInt::INT;
1696 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1697 return TypeFunc::make(domain, range);
1698 }
1699
1700 static const TypeFunc* make_base64_encodeBlock_Type() {
1701 int argcnt = 6;
1702
1703 const Type** fields = TypeTuple::fields(argcnt);
1704 int argp = TypeFunc::Parms;
1705 fields[argp++] = TypePtr::NOTNULL; // src array
1706 fields[argp++] = TypeInt::INT; // offset
1707 fields[argp++] = TypeInt::INT; // length
1708 fields[argp++] = TypePtr::NOTNULL; // dest array
1709 fields[argp++] = TypeInt::INT; // dp
1710 fields[argp++] = TypeInt::BOOL; // isURL
1711 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1712 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1713
1714 // result type needed
1715 fields = TypeTuple::fields(1);
1716 fields[TypeFunc::Parms + 0] = nullptr; // void
1717 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1718 return TypeFunc::make(domain, range);
1719 }
1720
1721 static const TypeFunc* make_string_IndexOf_Type() {
1722 int argcnt = 4;
1723
1724 const Type** fields = TypeTuple::fields(argcnt);
1725 int argp = TypeFunc::Parms;
1726 fields[argp++] = TypePtr::NOTNULL; // haystack array
1727 fields[argp++] = TypeInt::INT; // haystack length
1728 fields[argp++] = TypePtr::NOTNULL; // needle array
1729 fields[argp++] = TypeInt::INT; // needle length
1730 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1731 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1732
1733 // result type needed
1734 fields = TypeTuple::fields(1);
1735 fields[TypeFunc::Parms + 0] = TypeInt::INT; // Index of needle in haystack
1736 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1737 return TypeFunc::make(domain, range);
1738 }
1739
1740 static const TypeFunc* make_base64_decodeBlock_Type() {
1741 int argcnt = 7;
1742
1743 const Type** fields = TypeTuple::fields(argcnt);
1744 int argp = TypeFunc::Parms;
1745 fields[argp++] = TypePtr::NOTNULL; // src array
1746 fields[argp++] = TypeInt::INT; // src offset
1747 fields[argp++] = TypeInt::INT; // src length
1748 fields[argp++] = TypePtr::NOTNULL; // dest array
1749 fields[argp++] = TypeInt::INT; // dest offset
1750 fields[argp++] = TypeInt::BOOL; // isURL
1751 fields[argp++] = TypeInt::BOOL; // isMIME
1752 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1753 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1754
1755 // result type needed
1756 fields = TypeTuple::fields(1);
1757 fields[TypeFunc::Parms + 0] = TypeInt::INT; // count of bytes written to dst
1758 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 1, fields);
1759 return TypeFunc::make(domain, range);
1760 }
1761
1762 static const TypeFunc* make_poly1305_processBlocks_Type() {
1763 int argcnt = 4;
1764
1765 const Type** fields = TypeTuple::fields(argcnt);
1766 int argp = TypeFunc::Parms;
1767 fields[argp++] = TypePtr::NOTNULL; // input array
1768 fields[argp++] = TypeInt::INT; // input length
1769 fields[argp++] = TypePtr::NOTNULL; // accumulator array
1770 fields[argp++] = TypePtr::NOTNULL; // r array
1771 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1772 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1773
1774 // result type needed
1775 fields = TypeTuple::fields(1);
1776 fields[TypeFunc::Parms + 0] = nullptr; // void
1777 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1778 return TypeFunc::make(domain, range);
1779 }
1780
1781 static const TypeFunc* make_intpoly_montgomeryMult_P256_Type() {
1782 int argcnt = 3;
1783
1784 const Type** fields = TypeTuple::fields(argcnt);
1785 int argp = TypeFunc::Parms;
1786 fields[argp++] = TypePtr::NOTNULL; // a array
1787 fields[argp++] = TypePtr::NOTNULL; // b array
1788 fields[argp++] = TypePtr::NOTNULL; // r(esult) array
1789 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1790 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1791
1792 // result type needed
1793 fields = TypeTuple::fields(1);
1794 fields[TypeFunc::Parms + 0] = nullptr; // void
1795 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1796 return TypeFunc::make(domain, range);
1797 }
1798
1799 static const TypeFunc* make_intpoly_assign_Type() {
1800 int argcnt = 4;
1801
1802 const Type** fields = TypeTuple::fields(argcnt);
1803 int argp = TypeFunc::Parms;
1804 fields[argp++] = TypeInt::INT; // set flag
1805 fields[argp++] = TypePtr::NOTNULL; // a array (result)
1806 fields[argp++] = TypePtr::NOTNULL; // b array (if set is set)
1807 fields[argp++] = TypeInt::INT; // array length
1808 assert(argp == TypeFunc::Parms + argcnt, "correct decoding");
1809 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+argcnt, fields);
1810
1811 // result type needed
1812 fields = TypeTuple::fields(1);
1813 fields[TypeFunc::Parms + 0] = nullptr; // void
1814 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
1815 return TypeFunc::make(domain, range);
1816 }
1817
1818 //------------- Interpreter state for on stack replacement
1819 static const TypeFunc* make_osr_end_Type() {
1820 // create input type (domain)
1821 const Type **fields = TypeTuple::fields(1);
1822 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // OSR temp buf
1823 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1, fields);
1824
1825 // create result type
1826 fields = TypeTuple::fields(1);
1827 // fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // locked oop
1828 fields[TypeFunc::Parms+0] = nullptr; // void
1829 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms, fields);
1830 return TypeFunc::make(domain, range);
1831 }
1832
1833 //-------------------------------------------------------------------------------------
1834 // register policy
1835
1836 bool OptoRuntime::is_callee_saved_register(MachRegisterNumbers reg) {
1837 assert(reg >= 0 && reg < _last_Mach_Reg, "must be a machine register");
1838 switch (register_save_policy[reg]) {
1839 case 'C': return false; //SOC
1840 case 'E': return true ; //SOE
1841 case 'N': return false; //NS
1842 case 'A': return false; //AS
1843 }
1844 ShouldNotReachHere();
1845 return false;
1846 }
1847
1848 //-----------------------------------------------------------------------
1849 // Exceptions
1850 //
1851
1852 static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg);
1853
1854 // The method is an entry that is always called by a C++ method not
1855 // directly from compiled code. Compiled code will call the C++ method following.
1856 // We can't allow async exception to be installed during exception processing.
1857 JRT_ENTRY_NO_ASYNC(address, OptoRuntime::handle_exception_C_helper(JavaThread* current, nmethod* &nm))
1858 // The frame we rethrow the exception to might not have been processed by the GC yet.
1859 // The stack watermark barrier takes care of detecting that and ensuring the frame
1860 // has updated oops.
1861 StackWatermarkSet::after_unwind(current);
1862
1863 // Do not confuse exception_oop with pending_exception. The exception_oop
1864 // is only used to pass arguments into the method. Not for general
1865 // exception handling. DO NOT CHANGE IT to use pending_exception, since
1866 // the runtime stubs checks this on exit.
1867 assert(current->exception_oop() != nullptr, "exception oop is found");
1868 address handler_address = nullptr;
1869
1870 Handle exception(current, current->exception_oop());
1871 address pc = current->exception_pc();
1872
1873 // Clear out the exception oop and pc since looking up an
1874 // exception handler can cause class loading, which might throw an
1875 // exception and those fields are expected to be clear during
1876 // normal bytecode execution.
1877 current->clear_exception_oop_and_pc();
1878
1879 LogTarget(Info, exceptions) lt;
1880 if (lt.is_enabled()) {
1881 LogStream ls(lt);
1882 trace_exception(&ls, exception(), pc, "");
1883 }
1884
1885 // for AbortVMOnException flag
1886 Exceptions::debug_check_abort(exception);
1887
1888 #ifdef ASSERT
1889 if (!(exception->is_a(vmClasses::Throwable_klass()))) {
1890 // should throw an exception here
1891 ShouldNotReachHere();
1892 }
1893 #endif
1894
1895 // new exception handling: this method is entered only from adapters
1896 // exceptions from compiled java methods are handled in compiled code
1897 // using rethrow node
1898
1899 nm = CodeCache::find_nmethod(pc);
1900 assert(nm != nullptr, "No NMethod found");
1901 if (nm->is_native_method()) {
1902 fatal("Native method should not have path to exception handling");
1903 } else {
1904 // we are switching to old paradigm: search for exception handler in caller_frame
1905 // instead in exception handler of caller_frame.sender()
1906
1907 if (JvmtiExport::can_post_on_exceptions()) {
1908 // "Full-speed catching" is not necessary here,
1909 // since we're notifying the VM on every catch.
1910 // Force deoptimization and the rest of the lookup
1911 // will be fine.
1912 deoptimize_caller_frame(current);
1913 }
1914
1915 // Check the stack guard pages. If enabled, look for handler in this frame;
1916 // otherwise, forcibly unwind the frame.
1917 //
1918 // 4826555: use default current sp for reguard_stack instead of &nm: it's more accurate.
1919 bool force_unwind = !current->stack_overflow_state()->reguard_stack();
1920 bool deopting = false;
1921 if (nm->is_deopt_pc(pc)) {
1922 deopting = true;
1923 RegisterMap map(current,
1924 RegisterMap::UpdateMap::skip,
1925 RegisterMap::ProcessFrames::include,
1926 RegisterMap::WalkContinuation::skip);
1927 frame deoptee = current->last_frame().sender(&map);
1928 assert(deoptee.is_deoptimized_frame(), "must be deopted");
1929 // Adjust the pc back to the original throwing pc
1930 pc = deoptee.pc();
1931 }
1932
1933 // If we are forcing an unwind because of stack overflow then deopt is
1934 // irrelevant since we are throwing the frame away anyway.
1935
1936 if (deopting && !force_unwind) {
1937 handler_address = SharedRuntime::deopt_blob()->unpack_with_exception();
1938 } else {
1939
1940 handler_address =
1941 force_unwind ? nullptr : nm->handler_for_exception_and_pc(exception, pc);
1942
1943 if (handler_address == nullptr) {
1944 bool recursive_exception = false;
1945 handler_address = SharedRuntime::compute_compiled_exc_handler(nm, pc, exception, force_unwind, true, recursive_exception);
1946 assert (handler_address != nullptr, "must have compiled handler");
1947 // Update the exception cache only when the unwind was not forced
1948 // and there didn't happen another exception during the computation of the
1949 // compiled exception handler. Checking for exception oop equality is not
1950 // sufficient because some exceptions are pre-allocated and reused.
1951 if (!force_unwind && !recursive_exception) {
1952 nm->add_handler_for_exception_and_pc(exception,pc,handler_address);
1953 }
1954 } else {
1955 #ifdef ASSERT
1956 bool recursive_exception = false;
1957 address computed_address = SharedRuntime::compute_compiled_exc_handler(nm, pc, exception, force_unwind, true, recursive_exception);
1958 vmassert(recursive_exception || (handler_address == computed_address), "Handler address inconsistency: " PTR_FORMAT " != " PTR_FORMAT,
1959 p2i(handler_address), p2i(computed_address));
1960 #endif
1961 }
1962 }
1963
1964 current->set_exception_pc(pc);
1965 current->set_exception_handler_pc(handler_address);
1966
1967 // Check if the exception PC is a MethodHandle call site.
1968 current->set_is_method_handle_return(nm->is_method_handle_return(pc));
1969 }
1970
1971 // Restore correct return pc. Was saved above.
1972 current->set_exception_oop(exception());
1973 return handler_address;
1974
1975 JRT_END
1976
1977 // We are entering here from exception_blob
1978 // If there is a compiled exception handler in this method, we will continue there;
1979 // otherwise we will unwind the stack and continue at the caller of top frame method
1980 // Note we enter without the usual JRT wrapper. We will call a helper routine that
1981 // will do the normal VM entry. We do it this way so that we can see if the nmethod
1982 // we looked up the handler for has been deoptimized in the meantime. If it has been
1983 // we must not use the handler and instead return the deopt blob.
1984 address OptoRuntime::handle_exception_C(JavaThread* current) {
1985 //
1986 // We are in Java not VM and in debug mode we have a NoHandleMark
1987 //
1988 #ifndef PRODUCT
1989 SharedRuntime::_find_handler_ctr++; // find exception handler
1990 #endif
1991 DEBUG_ONLY(NoHandleMark __hm;)
1992 nmethod* nm = nullptr;
1993 address handler_address = nullptr;
1994 {
1995 // Enter the VM
1996
1997 ResetNoHandleMark rnhm;
1998 handler_address = handle_exception_C_helper(current, nm);
1999 }
2000
2001 // Back in java: Use no oops, DON'T safepoint
2002
2003 // Now check to see if the handler we are returning is in a now
2004 // deoptimized frame
2005
2006 if (nm != nullptr) {
2007 RegisterMap map(current,
2008 RegisterMap::UpdateMap::skip,
2009 RegisterMap::ProcessFrames::skip,
2010 RegisterMap::WalkContinuation::skip);
2011 frame caller = current->last_frame().sender(&map);
2012 #ifdef ASSERT
2013 assert(caller.is_compiled_frame(), "must be");
2014 #endif // ASSERT
2015 if (caller.is_deoptimized_frame()) {
2016 handler_address = SharedRuntime::deopt_blob()->unpack_with_exception();
2017 }
2018 }
2019 return handler_address;
2020 }
2021
2022 //------------------------------rethrow----------------------------------------
2023 // We get here after compiled code has executed a 'RethrowNode'. The callee
2024 // is either throwing or rethrowing an exception. The callee-save registers
2025 // have been restored, synchronized objects have been unlocked and the callee
2026 // stack frame has been removed. The return address was passed in.
2027 // Exception oop is passed as the 1st argument. This routine is then called
2028 // from the stub. On exit, we know where to jump in the caller's code.
2029 // After this C code exits, the stub will pop his frame and end in a jump
2030 // (instead of a return). We enter the caller's default handler.
2031 //
2032 // This must be JRT_LEAF:
2033 // - caller will not change its state as we cannot block on exit,
2034 // therefore raw_exception_handler_for_return_address is all it takes
2035 // to handle deoptimized blobs
2036 //
2037 // However, there needs to be a safepoint check in the middle! So compiled
2038 // safepoints are completely watertight.
2039 //
2040 // Thus, it cannot be a leaf since it contains the NoSafepointVerifier.
2041 //
2042 // *THIS IS NOT RECOMMENDED PROGRAMMING STYLE*
2043 //
2044 address OptoRuntime::rethrow_C(oopDesc* exception, JavaThread* thread, address ret_pc) {
2045 // ret_pc will have been loaded from the stack, so for AArch64 will be signed.
2046 AARCH64_PORT_ONLY(ret_pc = pauth_strip_verifiable(ret_pc));
2047
2048 #ifndef PRODUCT
2049 SharedRuntime::_rethrow_ctr++; // count rethrows
2050 #endif
2051 assert (exception != nullptr, "should have thrown a NullPointerException");
2052 #ifdef ASSERT
2053 if (!(exception->is_a(vmClasses::Throwable_klass()))) {
2054 // should throw an exception here
2055 ShouldNotReachHere();
2056 }
2057 #endif
2058
2059 thread->set_vm_result_oop(exception);
2060 // Frame not compiled (handles deoptimization blob)
2061 return SharedRuntime::raw_exception_handler_for_return_address(thread, ret_pc);
2062 }
2063
2064 static const TypeFunc* make_rethrow_Type() {
2065 // create input type (domain)
2066 const Type **fields = TypeTuple::fields(1);
2067 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Exception oop
2068 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1,fields);
2069
2070 // create result type (range)
2071 fields = TypeTuple::fields(1);
2072 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // Exception oop
2073 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1, fields);
2074
2075 return TypeFunc::make(domain, range);
2076 }
2077
2078
2079 void OptoRuntime::deoptimize_caller_frame(JavaThread *thread, bool doit) {
2080 // Deoptimize the caller before continuing, as the compiled
2081 // exception handler table may not be valid.
2082 if (DeoptimizeOnAllocationException && doit) {
2083 deoptimize_caller_frame(thread);
2084 }
2085 }
2086
2087 void OptoRuntime::deoptimize_caller_frame(JavaThread *thread) {
2088 // Called from within the owner thread, so no need for safepoint
2089 RegisterMap reg_map(thread,
2090 RegisterMap::UpdateMap::include,
2091 RegisterMap::ProcessFrames::include,
2092 RegisterMap::WalkContinuation::skip);
2093 frame stub_frame = thread->last_frame();
2094 assert(stub_frame.is_runtime_frame() || exception_blob()->contains(stub_frame.pc()), "sanity check");
2095 frame caller_frame = stub_frame.sender(®_map);
2096
2097 // Deoptimize the caller frame.
2098 Deoptimization::deoptimize_frame(thread, caller_frame.id());
2099 }
2100
2101
2102 bool OptoRuntime::is_deoptimized_caller_frame(JavaThread *thread) {
2103 // Called from within the owner thread, so no need for safepoint
2104 RegisterMap reg_map(thread,
2105 RegisterMap::UpdateMap::include,
2106 RegisterMap::ProcessFrames::include,
2107 RegisterMap::WalkContinuation::skip);
2108 frame stub_frame = thread->last_frame();
2109 assert(stub_frame.is_runtime_frame() || exception_blob()->contains(stub_frame.pc()), "sanity check");
2110 frame caller_frame = stub_frame.sender(®_map);
2111 return caller_frame.is_deoptimized_frame();
2112 }
2113
2114 static const TypeFunc* make_register_finalizer_Type() {
2115 // create input type (domain)
2116 const Type **fields = TypeTuple::fields(1);
2117 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // oop; Receiver
2118 // // The JavaThread* is passed to each routine as the last argument
2119 // fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // JavaThread *; Executing thread
2120 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1,fields);
2121
2122 // create result type (range)
2123 fields = TypeTuple::fields(0);
2124
2125 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2126
2127 return TypeFunc::make(domain, range);
2128 }
2129
2130 #if INCLUDE_JFR
2131 static const TypeFunc* make_class_id_load_barrier_Type() {
2132 // create input type (domain)
2133 const Type **fields = TypeTuple::fields(1);
2134 fields[TypeFunc::Parms+0] = TypeInstPtr::KLASS;
2135 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms + 1, fields);
2136
2137 // create result type (range)
2138 fields = TypeTuple::fields(0);
2139
2140 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms + 0, fields);
2141
2142 return TypeFunc::make(domain,range);
2143 }
2144 #endif // INCLUDE_JFR
2145
2146 //-----------------------------------------------------------------------------
2147 static const TypeFunc* make_dtrace_method_entry_exit_Type() {
2148 // create input type (domain)
2149 const Type **fields = TypeTuple::fields(2);
2150 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2151 fields[TypeFunc::Parms+1] = TypeMetadataPtr::BOTTOM; // Method*; Method we are entering
2152 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
2153
2154 // create result type (range)
2155 fields = TypeTuple::fields(0);
2156
2157 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2158
2159 return TypeFunc::make(domain, range);
2160 }
2161
2162 static const TypeFunc* make_dtrace_object_alloc_Type() {
2163 // create input type (domain)
2164 const Type **fields = TypeTuple::fields(2);
2165 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2166 fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL; // oop; newly allocated object
2167
2168 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
2169
2170 // create result type (range)
2171 fields = TypeTuple::fields(0);
2172
2173 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2174
2175 return TypeFunc::make(domain, range);
2176 }
2177
2178 JRT_ENTRY_NO_ASYNC(void, OptoRuntime::register_finalizer_C(oopDesc* obj, JavaThread* current))
2179 assert(oopDesc::is_oop(obj), "must be a valid oop");
2180 assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
2181 InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
2182 JRT_END
2183
2184 //-----------------------------------------------------------------------------
2185
2186 NamedCounter * volatile OptoRuntime::_named_counters = nullptr;
2187
2188 //
2189 // dump the collected NamedCounters.
2190 //
2191 void OptoRuntime::print_named_counters() {
2192 int total_lock_count = 0;
2193 int eliminated_lock_count = 0;
2194
2195 NamedCounter* c = _named_counters;
2196 while (c) {
2197 if (c->tag() == NamedCounter::LockCounter || c->tag() == NamedCounter::EliminatedLockCounter) {
2198 int count = c->count();
2199 if (count > 0) {
2200 bool eliminated = c->tag() == NamedCounter::EliminatedLockCounter;
2201 if (Verbose) {
2202 tty->print_cr("%d %s%s", count, c->name(), eliminated ? " (eliminated)" : "");
2203 }
2204 total_lock_count += count;
2205 if (eliminated) {
2206 eliminated_lock_count += count;
2207 }
2208 }
2209 }
2210 c = c->next();
2211 }
2212 if (total_lock_count > 0) {
2213 tty->print_cr("dynamic locks: %d", total_lock_count);
2214 if (eliminated_lock_count) {
2215 tty->print_cr("eliminated locks: %d (%d%%)", eliminated_lock_count,
2216 (int)(eliminated_lock_count * 100.0 / total_lock_count));
2217 }
2218 }
2219 }
2220
2221 //
2222 // Allocate a new NamedCounter. The JVMState is used to generate the
2223 // name which consists of method@line for the inlining tree.
2224 //
2225
2226 NamedCounter* OptoRuntime::new_named_counter(JVMState* youngest_jvms, NamedCounter::CounterTag tag) {
2227 int max_depth = youngest_jvms->depth();
2228
2229 // Visit scopes from youngest to oldest.
2230 bool first = true;
2231 stringStream st;
2232 for (int depth = max_depth; depth >= 1; depth--) {
2233 JVMState* jvms = youngest_jvms->of_depth(depth);
2234 ciMethod* m = jvms->has_method() ? jvms->method() : nullptr;
2235 if (!first) {
2236 st.print(" ");
2237 } else {
2238 first = false;
2239 }
2240 int bci = jvms->bci();
2241 if (bci < 0) bci = 0;
2242 if (m != nullptr) {
2243 st.print("%s.%s", m->holder()->name()->as_utf8(), m->name()->as_utf8());
2244 } else {
2245 st.print("no method");
2246 }
2247 st.print("@%d", bci);
2248 // To print linenumbers instead of bci use: m->line_number_from_bci(bci)
2249 }
2250 NamedCounter* c = new NamedCounter(st.freeze(), tag);
2251
2252 // atomically add the new counter to the head of the list. We only
2253 // add counters so this is safe.
2254 NamedCounter* head;
2255 do {
2256 c->set_next(nullptr);
2257 head = _named_counters;
2258 c->set_next(head);
2259 } while (Atomic::cmpxchg(&_named_counters, head, c) != head);
2260 return c;
2261 }
2262
2263 void OptoRuntime::initialize_types() {
2264 _new_instance_Type = make_new_instance_Type();
2265 _new_array_Type = make_new_array_Type();
2266 _new_array_nozero_Type = make_new_array_nozero_Type();
2267 _multianewarray2_Type = multianewarray_Type(2);
2268 _multianewarray3_Type = multianewarray_Type(3);
2269 _multianewarray4_Type = multianewarray_Type(4);
2270 _multianewarray5_Type = multianewarray_Type(5);
2271 _multianewarrayN_Type = make_multianewarrayN_Type();
2272 _complete_monitor_enter_Type = make_complete_monitor_enter_Type();
2273 _complete_monitor_exit_Type = make_complete_monitor_exit_Type();
2274 _monitor_notify_Type = make_monitor_notify_Type();
2275 _uncommon_trap_Type = make_uncommon_trap_Type();
2276 _athrow_Type = make_athrow_Type();
2277 _rethrow_Type = make_rethrow_Type();
2278 _Math_D_D_Type = make_Math_D_D_Type();
2279 _Math_DD_D_Type = make_Math_DD_D_Type();
2280 _modf_Type = make_modf_Type();
2281 _l2f_Type = make_l2f_Type();
2282 _void_long_Type = make_void_long_Type();
2283 _void_void_Type = make_void_void_Type();
2284 _jfr_write_checkpoint_Type = make_jfr_write_checkpoint_Type();
2285 _flush_windows_Type = make_flush_windows_Type();
2286 _fast_arraycopy_Type = make_arraycopy_Type(ac_fast);
2287 _checkcast_arraycopy_Type = make_arraycopy_Type(ac_checkcast);
2288 _generic_arraycopy_Type = make_arraycopy_Type(ac_generic);
2289 _slow_arraycopy_Type = make_arraycopy_Type(ac_slow);
2290 _unsafe_setmemory_Type = make_setmemory_Type();
2291 _array_fill_Type = make_array_fill_Type();
2292 _array_sort_Type = make_array_sort_Type();
2293 _array_partition_Type = make_array_partition_Type();
2294 _aescrypt_block_Type = make_aescrypt_block_Type();
2295 _cipherBlockChaining_aescrypt_Type = make_cipherBlockChaining_aescrypt_Type();
2296 _electronicCodeBook_aescrypt_Type = make_electronicCodeBook_aescrypt_Type();
2297 _counterMode_aescrypt_Type = make_counterMode_aescrypt_Type();
2298 _galoisCounterMode_aescrypt_Type = make_galoisCounterMode_aescrypt_Type();
2299 _digestBase_implCompress_with_sha3_Type = make_digestBase_implCompress_Type( /* is_sha3= */ true);
2300 _digestBase_implCompress_without_sha3_Type = make_digestBase_implCompress_Type( /* is_sha3= */ false);;
2301 _digestBase_implCompressMB_with_sha3_Type = make_digestBase_implCompressMB_Type(/* is_sha3= */ true);
2302 _digestBase_implCompressMB_without_sha3_Type = make_digestBase_implCompressMB_Type(/* is_sha3= */ false);
2303 _double_keccak_Type = make_double_keccak_Type();
2304 _multiplyToLen_Type = make_multiplyToLen_Type();
2305 _montgomeryMultiply_Type = make_montgomeryMultiply_Type();
2306 _montgomerySquare_Type = make_montgomerySquare_Type();
2307 _squareToLen_Type = make_squareToLen_Type();
2308 _mulAdd_Type = make_mulAdd_Type();
2309 _bigIntegerShift_Type = make_bigIntegerShift_Type();
2310 _vectorizedMismatch_Type = make_vectorizedMismatch_Type();
2311 _ghash_processBlocks_Type = make_ghash_processBlocks_Type();
2312 _chacha20Block_Type = make_chacha20Block_Type();
2313 _kyberNtt_Type = make_kyberNtt_Type();
2314 _kyberInverseNtt_Type = make_kyberInverseNtt_Type();
2315 _kyberNttMult_Type = make_kyberNttMult_Type();
2316 _kyberAddPoly_2_Type = make_kyberAddPoly_2_Type();
2317 _kyberAddPoly_3_Type = make_kyberAddPoly_3_Type();
2318 _kyber12To16_Type = make_kyber12To16_Type();
2319 _kyberBarrettReduce_Type = make_kyberBarrettReduce_Type();
2320 _dilithiumAlmostNtt_Type = make_dilithiumAlmostNtt_Type();
2321 _dilithiumAlmostInverseNtt_Type = make_dilithiumAlmostInverseNtt_Type();
2322 _dilithiumNttMult_Type = make_dilithiumNttMult_Type();
2323 _dilithiumMontMulByConstant_Type = make_dilithiumMontMulByConstant_Type();
2324 _dilithiumDecomposePoly_Type = make_dilithiumDecomposePoly_Type();
2325 _base64_encodeBlock_Type = make_base64_encodeBlock_Type();
2326 _base64_decodeBlock_Type = make_base64_decodeBlock_Type();
2327 _string_IndexOf_Type = make_string_IndexOf_Type();
2328 _poly1305_processBlocks_Type = make_poly1305_processBlocks_Type();
2329 _intpoly_montgomeryMult_P256_Type = make_intpoly_montgomeryMult_P256_Type();
2330 _intpoly_assign_Type = make_intpoly_assign_Type();
2331 _updateBytesCRC32_Type = make_updateBytesCRC32_Type();
2332 _updateBytesCRC32C_Type = make_updateBytesCRC32C_Type();
2333 _updateBytesAdler32_Type = make_updateBytesAdler32_Type();
2334 _osr_end_Type = make_osr_end_Type();
2335 _register_finalizer_Type = make_register_finalizer_Type();
2336 JFR_ONLY(
2337 _class_id_load_barrier_Type = make_class_id_load_barrier_Type();
2338 )
2339 #if INCLUDE_JVMTI
2340 _notify_jvmti_vthread_Type = make_notify_jvmti_vthread_Type();
2341 #endif // INCLUDE_JVMTI
2342 _dtrace_method_entry_exit_Type = make_dtrace_method_entry_exit_Type();
2343 _dtrace_object_alloc_Type = make_dtrace_object_alloc_Type();
2344 }
2345
2346 int trace_exception_counter = 0;
2347 static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg) {
2348 trace_exception_counter++;
2349 stringStream tempst;
2350
2351 tempst.print("%d [Exception (%s): ", trace_exception_counter, msg);
2352 exception_oop->print_value_on(&tempst);
2353 tempst.print(" in ");
2354 CodeBlob* blob = CodeCache::find_blob(exception_pc);
2355 if (blob->is_nmethod()) {
2356 blob->as_nmethod()->method()->print_value_on(&tempst);
2357 } else if (blob->is_runtime_stub()) {
2358 tempst.print("<runtime-stub>");
2359 } else {
2360 tempst.print("<unknown>");
2361 }
2362 tempst.print(" at " INTPTR_FORMAT, p2i(exception_pc));
2363 tempst.print("]");
2364
2365 st->print_raw_cr(tempst.freeze());
2366 }
2367
2368 const TypeFunc *OptoRuntime::store_inline_type_fields_Type() {
2369 // create input type (domain)
2370 uint total = SharedRuntime::java_return_convention_max_int + SharedRuntime::java_return_convention_max_float*2;
2371 const Type **fields = TypeTuple::fields(total);
2372 // We don't know the number of returned values and their
2373 // types. Assume all registers available to the return convention
2374 // are used.
2375 fields[TypeFunc::Parms] = TypePtr::BOTTOM;
2376 uint i = 1;
2377 for (; i < SharedRuntime::java_return_convention_max_int; i++) {
2378 fields[TypeFunc::Parms+i] = TypeInt::INT;
2379 }
2380 for (; i < total; i+=2) {
2381 fields[TypeFunc::Parms+i] = Type::DOUBLE;
2382 fields[TypeFunc::Parms+i+1] = Type::HALF;
2383 }
2384 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + total, fields);
2385
2386 // create result type (range)
2387 fields = TypeTuple::fields(1);
2388 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
2389
2390 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1,fields);
2391
2392 return TypeFunc::make(domain, range);
2393 }
2394
2395 const TypeFunc *OptoRuntime::pack_inline_type_Type() {
2396 // create input type (domain)
2397 uint total = 1 + SharedRuntime::java_return_convention_max_int + SharedRuntime::java_return_convention_max_float*2;
2398 const Type **fields = TypeTuple::fields(total);
2399 // We don't know the number of returned values and their
2400 // types. Assume all registers available to the return convention
2401 // are used.
2402 fields[TypeFunc::Parms] = TypeRawPtr::BOTTOM;
2403 fields[TypeFunc::Parms+1] = TypeRawPtr::BOTTOM;
2404 uint i = 2;
2405 for (; i < SharedRuntime::java_return_convention_max_int+1; i++) {
2406 fields[TypeFunc::Parms+i] = TypeInt::INT;
2407 }
2408 for (; i < total; i+=2) {
2409 fields[TypeFunc::Parms+i] = Type::DOUBLE;
2410 fields[TypeFunc::Parms+i+1] = Type::HALF;
2411 }
2412 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + total, fields);
2413
2414 // create result type (range)
2415 fields = TypeTuple::fields(1);
2416 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL;
2417
2418 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+1,fields);
2419
2420 return TypeFunc::make(domain, range);
2421 }
2422
2423 JRT_BLOCK_ENTRY(void, OptoRuntime::load_unknown_inline_C(flatArrayOopDesc* array, int index, JavaThread* current))
2424 JRT_BLOCK;
2425 oop buffer = array->read_value_from_flat_array(index, THREAD);
2426 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
2427 current->set_vm_result_oop(buffer);
2428 JRT_BLOCK_END;
2429 JRT_END
2430
2431 const TypeFunc* OptoRuntime::load_unknown_inline_Type() {
2432 // create input type (domain)
2433 const Type** fields = TypeTuple::fields(2);
2434 fields[TypeFunc::Parms] = TypeOopPtr::NOTNULL;
2435 fields[TypeFunc::Parms+1] = TypeInt::POS;
2436
2437 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+2, fields);
2438
2439 // create result type (range)
2440 fields = TypeTuple::fields(1);
2441 fields[TypeFunc::Parms] = TypeInstPtr::BOTTOM;
2442
2443 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms+1, fields);
2444
2445 return TypeFunc::make(domain, range);
2446 }
2447
2448 JRT_BLOCK_ENTRY(void, OptoRuntime::store_unknown_inline_C(instanceOopDesc* buffer, flatArrayOopDesc* array, int index, JavaThread* current))
2449 JRT_BLOCK;
2450 array->write_value_to_flat_array(buffer, index, THREAD);
2451 if (HAS_PENDING_EXCEPTION) {
2452 fatal("This entry must be changed to be a non-leaf entry because writing to a flat array can now throw an exception");
2453 }
2454 JRT_BLOCK_END;
2455 JRT_END
2456
2457 const TypeFunc* OptoRuntime::store_unknown_inline_Type() {
2458 // create input type (domain)
2459 const Type** fields = TypeTuple::fields(3);
2460 fields[TypeFunc::Parms] = TypeInstPtr::NOTNULL;
2461 fields[TypeFunc::Parms+1] = TypeOopPtr::NOTNULL;
2462 fields[TypeFunc::Parms+2] = TypeInt::POS;
2463
2464 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+3, fields);
2465
2466 // create result type (range)
2467 fields = TypeTuple::fields(0);
2468 const TypeTuple* range = TypeTuple::make(TypeFunc::Parms, fields);
2469
2470 return TypeFunc::make(domain, range);
2471 }