1 /*
  2  * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.
  8  *
  9  * This code is distributed in the hope that it will be useful, but WITHOUT
 10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 12  * version 2 for more details (a copy is included in the LICENSE file that
 13  * accompanied this code).
 14  *
 15  * You should have received a copy of the GNU General Public License version
 16  * 2 along with this work; if not, write to the Free Software Foundation,
 17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 18  *
 19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 20  * or visit www.oracle.com if you need additional information or have any
 21  * questions.
 22  *
 23  */
 24 
 25 #ifndef SHARE_OPTO_RUNTIME_HPP
 26 #define SHARE_OPTO_RUNTIME_HPP
 27 
 28 #include "code/codeBlob.hpp"
 29 #include "opto/machnode.hpp"
 30 #include "opto/optoreg.hpp"
 31 #include "runtime/stubDeclarations.hpp"
 32 #include "runtime/stubInfo.hpp"
 33 #include "runtime/vframe.hpp"
 34 
 35 //------------------------------OptoRuntime------------------------------------
 36 // Opto compiler runtime routines
 37 //
 38 // These are all generated from Ideal graphs.  They are called with the
 39 // Java calling convention.  Internally they call C++.  They are made once at
 40 // startup time and Opto compiles calls to them later.
 41 // Things are broken up into quads: the signature they will be called with,
 42 // the address of the generated code, the corresponding C++ code and an
 43 // nmethod.
 44 
 45 // The signature (returned by "xxx_Type()") is used at startup time by the
 46 // Generator to make the generated code "xxx_Java".  Opto compiles calls
 47 // to the generated code "xxx_Java".  When the compiled code gets executed,
 48 // it calls the C++ code "xxx_C".  The generated nmethod is saved in the
 49 // CodeCache.  Exception handlers use the nmethod to get the callee-save
 50 // register OopMaps.
 51 class CallInfo;
 52 
 53 //
 54 // NamedCounters are tagged counters which can be used for profiling
 55 // code in various ways.  Currently they are used by the lock coarsening code
 56 //
 57 
 58 class NamedCounter : public CHeapObj<mtCompiler> {
 59 public:
 60     enum CounterTag {
 61     NoTag,
 62     LockCounter,
 63     EliminatedLockCounter
 64   };
 65 
 66 private:
 67   const char *  _name;
 68   int           _count;
 69   CounterTag    _tag;
 70   NamedCounter* _next;
 71 
 72  public:
 73   NamedCounter(const char *n, CounterTag tag = NoTag):
 74     _name(n == nullptr ? nullptr : os::strdup(n)),
 75     _count(0),
 76     _tag(tag),
 77     _next(nullptr) {}
 78 
 79   ~NamedCounter() {
 80     if (_name != nullptr) {
 81       os::free((void*)_name);
 82     }
 83   }
 84 
 85   const char * name() const     { return _name; }
 86   int count() const             { return _count; }
 87   address addr()                { return (address)&_count; }
 88   CounterTag tag() const        { return _tag; }
 89   void set_tag(CounterTag tag)  { _tag = tag; }
 90 
 91   NamedCounter* next() const    { return _next; }
 92   void set_next(NamedCounter* next) {
 93     assert(_next == nullptr || next == nullptr, "already set");
 94     _next = next;
 95   }
 96 
 97 };
 98 
 99 typedef const TypeFunc*(*TypeFunc_generator)();
100 
101 class OptoRuntime : public AllStatic {
102   friend class Matcher;  // allow access to stub names
103   friend class AOTCodeAddressTable;
104 
105  private:
106   // declare opto stub address/blob holder static fields
107 #define C2_BLOB_FIELD_DECLARE(name, type) \
108   static type*       BLOB_FIELD_NAME(name);
109 #define C2_STUB_FIELD_NAME(name) _ ## name ## _Java
110 #define C2_STUB_FIELD_DECLARE(name, f, t, r) \
111   static address     C2_STUB_FIELD_NAME(name) ;
112 #define C2_JVMTI_STUB_FIELD_DECLARE(name) \
113   static address     STUB_FIELD_NAME(name);
114 
115   C2_STUBS_DO(C2_BLOB_FIELD_DECLARE, C2_STUB_FIELD_DECLARE, C2_JVMTI_STUB_FIELD_DECLARE)
116 
117 #undef C2_BLOB_FIELD_DECLARE
118 #undef C2_STUB_FIELD_NAME
119 #undef C2_STUB_FIELD_DECLARE
120 #undef C2_JVMTI_STUB_FIELD_DECLARE
121 
122   // static TypeFunc* data members
123   static const TypeFunc* _new_instance_Type;
124   static const TypeFunc* _new_array_Type;
125   static const TypeFunc* _new_array_nozero_Type;
126   static const TypeFunc* _multianewarray2_Type;
127   static const TypeFunc* _multianewarray3_Type;
128   static const TypeFunc* _multianewarray4_Type;
129   static const TypeFunc* _multianewarray5_Type;
130   static const TypeFunc* _multianewarrayN_Type;
131   static const TypeFunc* _complete_monitor_enter_Type;
132   static const TypeFunc* _complete_monitor_exit_Type;
133   static const TypeFunc* _monitor_notify_Type;
134   static const TypeFunc* _uncommon_trap_Type;
135   static const TypeFunc* _athrow_Type;
136   static const TypeFunc* _rethrow_Type;
137   static const TypeFunc* _Math_D_D_Type;
138   static const TypeFunc* _Math_DD_D_Type;
139   static const TypeFunc* _modf_Type;
140   static const TypeFunc* _l2f_Type;
141   static const TypeFunc* _void_long_Type;
142   static const TypeFunc* _void_void_Type;
143   static const TypeFunc* _jfr_write_checkpoint_Type;
144   static const TypeFunc* _flush_windows_Type;
145   static const TypeFunc* _fast_arraycopy_Type;
146   static const TypeFunc* _checkcast_arraycopy_Type;
147   static const TypeFunc* _generic_arraycopy_Type;
148   static const TypeFunc* _slow_arraycopy_Type;
149   static const TypeFunc* _unsafe_setmemory_Type;
150   static const TypeFunc* _array_fill_Type;
151   static const TypeFunc* _array_sort_Type;
152   static const TypeFunc* _array_partition_Type;
153   static const TypeFunc* _aescrypt_block_Type;
154   static const TypeFunc* _cipherBlockChaining_aescrypt_Type;
155   static const TypeFunc* _electronicCodeBook_aescrypt_Type;
156   static const TypeFunc* _counterMode_aescrypt_Type;
157   static const TypeFunc* _galoisCounterMode_aescrypt_Type;
158   static const TypeFunc* _digestBase_implCompress_with_sha3_Type;
159   static const TypeFunc* _digestBase_implCompress_without_sha3_Type;
160   static const TypeFunc* _digestBase_implCompressMB_with_sha3_Type;
161   static const TypeFunc* _digestBase_implCompressMB_without_sha3_Type;
162   static const TypeFunc* _double_keccak_Type;
163   static const TypeFunc* _multiplyToLen_Type;
164   static const TypeFunc* _montgomeryMultiply_Type;
165   static const TypeFunc* _montgomerySquare_Type;
166   static const TypeFunc* _squareToLen_Type;
167   static const TypeFunc* _mulAdd_Type;
168   static const TypeFunc* _bigIntegerShift_Type;
169   static const TypeFunc* _vectorizedMismatch_Type;
170   static const TypeFunc* _ghash_processBlocks_Type;
171   static const TypeFunc* _chacha20Block_Type;
172   static const TypeFunc* _kyberNtt_Type;
173   static const TypeFunc* _kyberInverseNtt_Type;
174   static const TypeFunc* _kyberNttMult_Type;
175   static const TypeFunc* _kyberAddPoly_2_Type;
176   static const TypeFunc* _kyberAddPoly_3_Type;
177   static const TypeFunc* _kyber12To16_Type;
178   static const TypeFunc* _kyberBarrettReduce_Type;
179   static const TypeFunc* _dilithiumAlmostNtt_Type;
180   static const TypeFunc* _dilithiumAlmostInverseNtt_Type;
181   static const TypeFunc* _dilithiumNttMult_Type;
182   static const TypeFunc* _dilithiumMontMulByConstant_Type;
183   static const TypeFunc* _dilithiumDecomposePoly_Type;
184   static const TypeFunc* _base64_encodeBlock_Type;
185   static const TypeFunc* _base64_decodeBlock_Type;
186   static const TypeFunc* _string_IndexOf_Type;
187   static const TypeFunc* _poly1305_processBlocks_Type;
188   static const TypeFunc* _intpoly_montgomeryMult_P256_Type;
189   static const TypeFunc* _intpoly_assign_Type;
190   static const TypeFunc* _updateBytesCRC32_Type;
191   static const TypeFunc* _updateBytesCRC32C_Type;
192   static const TypeFunc* _updateBytesAdler32_Type;
193   static const TypeFunc* _osr_end_Type;
194   static const TypeFunc* _register_finalizer_Type;
195 #if INCLUDE_JFR
196   static const TypeFunc* _class_id_load_barrier_Type;
197 #endif // INCLUDE_JFR
198 #if INCLUDE_JVMTI
199   static const TypeFunc* _notify_jvmti_vthread_Type;
200 #endif // INCLUDE_JVMTI
201   static const TypeFunc* _dtrace_method_entry_exit_Type;
202   static const TypeFunc* _dtrace_object_alloc_Type;
203 
204   // define stubs
205   static address generate_stub(ciEnv* ci_env, TypeFunc_generator gen, address C_function, const char* name, StubId stub_id, int is_fancy_jump, bool pass_tls, bool return_pc);
206 
207   //
208   // Implementation of runtime methods
209   // =================================
210 
211   // Allocate storage for a Java instance.
212   static void new_instance_C(Klass* instance_klass, bool is_larval, JavaThread* current);
213 
214   // Allocate storage for a objArray or typeArray
215   static void new_array_C(Klass* array_klass, int len, oopDesc* init_val, JavaThread* current);
216   static void new_array_nozero_C(Klass* array_klass, int len, JavaThread* current);
217 
218   // Allocate storage for a multi-dimensional arrays
219   // Note: needs to be fixed for arbitrary number of dimensions
220   static void multianewarray2_C(Klass* klass, int len1, int len2, JavaThread* current);
221   static void multianewarray3_C(Klass* klass, int len1, int len2, int len3, JavaThread* current);
222   static void multianewarray4_C(Klass* klass, int len1, int len2, int len3, int len4, JavaThread* current);
223   static void multianewarray5_C(Klass* klass, int len1, int len2, int len3, int len4, int len5, JavaThread* current);
224   static void multianewarrayN_C(Klass* klass, arrayOopDesc* dims, JavaThread* current);
225 
226   // local methods passed as arguments to stub generator that forward
227   // control to corresponding JRT methods of SharedRuntime
228   static void slow_arraycopy_C(oopDesc* src,  jint src_pos,
229                                oopDesc* dest, jint dest_pos,
230                                jint length, JavaThread* thread);
231   static void complete_monitor_locking_C(oopDesc* obj, BasicLock* lock, JavaThread* current);
232 
233 public:
234   static void monitor_notify_C(oopDesc* obj, JavaThread* current);
235   static void monitor_notifyAll_C(oopDesc* obj, JavaThread* current);
236 
237 private:
238 
239   // Implicit exception support
240   static void throw_null_exception_C(JavaThread* thread);
241 
242   // Exception handling
243   static address handle_exception_C       (JavaThread* current);
244   static address handle_exception_C_helper(JavaThread* current, nmethod*& nm);
245   static address rethrow_C                (oopDesc* exception, JavaThread *thread, address return_pc );
246   static void deoptimize_caller_frame     (JavaThread *thread);
247   static void deoptimize_caller_frame     (JavaThread *thread, bool doit);
248   static bool is_deoptimized_caller_frame (JavaThread *thread);
249 
250   // CodeBlob support
251   // ===================================================================
252 
253   static UncommonTrapBlob* generate_uncommon_trap_blob(void);
254   static ExceptionBlob* generate_exception_blob();
255 
256   static void register_finalizer_C(oopDesc* obj, JavaThread* current);
257 
258  public:
259   static void load_unknown_inline_C(flatArrayOopDesc* array, int index, JavaThread* current);
260   static void store_unknown_inline_C(instanceOopDesc* buffer, flatArrayOopDesc* array, int index, JavaThread* current);
261 
262   static bool is_callee_saved_register(MachRegisterNumbers reg);
263 
264   // One time only generate runtime code stubs. Returns true
265   // when runtime stubs have been generated successfully and
266   // false otherwise.
267   static bool generate(ciEnv* env);
268 
269   // Returns the name of a stub
270   static const char* stub_name(address entry);
271 
272   // Returns the name associated with a given stub id
273   static const char* stub_name(StubId id) {
274     assert(StubInfo::is_c2(id), "not a C2 stub %s", StubInfo::name(id));
275     return StubInfo::name(id);
276   }
277 
278   // access to runtime stubs entry points for java code
279   static address new_instance_Java()                     { return _new_instance_Java; }
280   static address new_array_Java()                        { return _new_array_Java; }
281   static address new_array_nozero_Java()                 { return _new_array_nozero_Java; }
282   static address multianewarray2_Java()                  { return _multianewarray2_Java; }
283   static address multianewarray3_Java()                  { return _multianewarray3_Java; }
284   static address multianewarray4_Java()                  { return _multianewarray4_Java; }
285   static address multianewarray5_Java()                  { return _multianewarray5_Java; }
286   static address multianewarrayN_Java()                  { return _multianewarrayN_Java; }
287   static address complete_monitor_locking_Java()         { return _complete_monitor_locking_Java; }
288   static address monitor_notify_Java()                   { return _monitor_notify_Java; }
289   static address monitor_notifyAll_Java()                { return _monitor_notifyAll_Java; }
290 
291   static address slow_arraycopy_Java()                   { return _slow_arraycopy_Java; }
292   static address register_finalizer_Java()               { return _register_finalizer_Java; }
293   static address load_unknown_inline_Java()              { return _load_unknown_inline_Java; }
294   static address store_unknown_inline_Java()             { return _store_unknown_inline_Java; }
295 #if INCLUDE_JVMTI
296   static address notify_jvmti_vthread_start()            { return _notify_jvmti_vthread_start; }
297   static address notify_jvmti_vthread_end()              { return _notify_jvmti_vthread_end; }
298   static address notify_jvmti_vthread_mount()            { return _notify_jvmti_vthread_mount; }
299   static address notify_jvmti_vthread_unmount()          { return _notify_jvmti_vthread_unmount; }
300 #endif
301 
302   static UncommonTrapBlob* uncommon_trap_blob()                  { return _uncommon_trap_blob; }
303   static ExceptionBlob*    exception_blob()                      { return _exception_blob; }
304 
305   // Implicit exception support
306   static void throw_div0_exception_C      (JavaThread* thread);
307   static void throw_stack_overflow_error_C(JavaThread* thread);
308 
309   // Exception handling
310   static address rethrow_stub()             { return _rethrow_Java; }
311 
312 
313   // Type functions
314   // ======================================================
315 
316   static inline const TypeFunc* new_instance_Type() {
317     assert(_new_instance_Type != nullptr, "should be initialized");
318     return _new_instance_Type;
319   }
320 
321   static inline const TypeFunc* new_array_Type() {
322     assert(_new_array_Type != nullptr, "should be initialized");
323     return _new_array_Type;
324   }
325 
326   static inline const TypeFunc* new_array_nozero_Type() {
327     assert(_new_array_nozero_Type != nullptr, "should be initialized");
328     return _new_array_nozero_Type;
329   }
330 
331   static const TypeFunc* multianewarray_Type(int ndim); // multianewarray
332 
333   static inline const TypeFunc* multianewarray2_Type() {
334     assert(_multianewarray2_Type != nullptr, "should be initialized");
335     return _multianewarray2_Type;
336   }
337 
338   static inline const TypeFunc* multianewarray3_Type() {
339     assert(_multianewarray3_Type != nullptr, "should be initialized");
340     return _multianewarray3_Type;
341   }
342 
343   static inline const TypeFunc* multianewarray4_Type() {
344     assert(_multianewarray4_Type != nullptr, "should be initialized");
345     return _multianewarray4_Type;
346   }
347 
348   static inline const TypeFunc* multianewarray5_Type() {
349     assert(_multianewarray5_Type != nullptr, "should be initialized");
350     return _multianewarray5_Type;
351   }
352 
353   static inline const TypeFunc* multianewarrayN_Type() {
354     assert(_multianewarrayN_Type != nullptr, "should be initialized");
355     return _multianewarrayN_Type;
356   }
357 
358   static inline const TypeFunc* complete_monitor_enter_Type() {
359     assert(_complete_monitor_enter_Type != nullptr, "should be initialized");
360     return _complete_monitor_enter_Type;
361   }
362 
363   static inline const TypeFunc* complete_monitor_locking_Type() {
364     return complete_monitor_enter_Type();
365   }
366 
367   static inline const TypeFunc* complete_monitor_exit_Type() {
368     assert(_complete_monitor_exit_Type != nullptr, "should be initialized");
369     return _complete_monitor_exit_Type;
370   }
371 
372   static inline const TypeFunc* monitor_notify_Type() {
373     assert(_monitor_notify_Type != nullptr, "should be initialized");
374     return _monitor_notify_Type;
375   }
376 
377   static inline const TypeFunc* monitor_notifyAll_Type() {
378     return monitor_notify_Type();
379   }
380 
381   static inline const TypeFunc* uncommon_trap_Type() {
382     assert(_uncommon_trap_Type != nullptr, "should be initialized");
383     return _uncommon_trap_Type;
384   }
385 
386   static inline const TypeFunc* athrow_Type() {
387     assert(_athrow_Type != nullptr, "should be initialized");
388     return _athrow_Type;
389   }
390 
391   static inline const TypeFunc* rethrow_Type() {
392     assert(_rethrow_Type != nullptr, "should be initialized");
393     return _rethrow_Type;
394   }
395 
396   static inline const TypeFunc* Math_D_D_Type() {
397     assert(_Math_D_D_Type != nullptr, "should be initialized");
398     return _Math_D_D_Type;
399   }
400 
401   static inline const TypeFunc* Math_DD_D_Type() {
402     assert(_Math_DD_D_Type != nullptr, "should be initialized");
403     return _Math_DD_D_Type;
404   }
405 
406   static const TypeFunc* Math_Vector_Vector_Type(uint num_arg, const TypeVect* in_type, const TypeVect* out_type);
407 
408   static inline const TypeFunc* modf_Type() {
409     assert(_modf_Type != nullptr, "should be initialized");
410     return _modf_Type;
411   }
412 
413   static inline const TypeFunc* l2f_Type() {
414     assert(_l2f_Type != nullptr, "should be initialized");
415     return _l2f_Type;
416   }
417 
418   static inline const TypeFunc* void_long_Type() {
419     assert(_void_long_Type != nullptr, "should be initialized");
420     return _void_long_Type;
421   }
422 
423   static inline const TypeFunc* void_void_Type() {
424     assert(_void_void_Type != nullptr, "should be initialized");
425     return _void_void_Type;
426   }
427 
428   static const TypeFunc* jfr_write_checkpoint_Type() {
429     assert(_jfr_write_checkpoint_Type != nullptr, "should be initialized");
430     return _jfr_write_checkpoint_Type;
431   }
432 
433   static const TypeFunc* flush_windows_Type() {
434     assert(_flush_windows_Type != nullptr, "should be initialized");
435     return _flush_windows_Type;
436   }
437 
438   // arraycopy routine types
439   static inline const TypeFunc* fast_arraycopy_Type() {
440     assert(_fast_arraycopy_Type != nullptr, "should be initialized");
441     // This signature is simple:  Two base pointers and a size_t.
442     return _fast_arraycopy_Type;
443   }
444 
445   static inline const TypeFunc* checkcast_arraycopy_Type() {
446     assert(_checkcast_arraycopy_Type != nullptr, "should be initialized");
447     // An extension of fast_arraycopy_Type which adds type checking.
448     return _checkcast_arraycopy_Type;
449   }
450 
451   static inline const TypeFunc* generic_arraycopy_Type() {
452     assert(_generic_arraycopy_Type != nullptr, "should be initialized");
453     // This signature is like System.arraycopy, except that it returns status.
454     return _generic_arraycopy_Type;
455   }
456 
457   static inline const TypeFunc* slow_arraycopy_Type() {
458     assert(_slow_arraycopy_Type != nullptr, "should be initialized");
459     // This signature is exactly the same as System.arraycopy.
460     // There are no intptr_t (int/long) arguments.
461     return _slow_arraycopy_Type;
462   }   // the full routine
463 
464   static inline const TypeFunc* unsafe_setmemory_Type() {
465     assert(_unsafe_setmemory_Type != nullptr, "should be initialized");
466     return _unsafe_setmemory_Type;
467   }
468 
469 //  static const TypeFunc* digestBase_implCompress_Type(bool is_sha3);
470 //  static const TypeFunc* digestBase_implCompressMB_Type(bool is_sha3);
471 //  static const TypeFunc* double_keccak_Type();
472 
473   static inline const TypeFunc* array_fill_Type() {
474     assert(_array_fill_Type != nullptr, "should be initialized");
475     return _array_fill_Type;
476   }
477 
478   static inline const TypeFunc* array_sort_Type() {
479     assert(_array_sort_Type != nullptr, "should be initialized");
480     return _array_sort_Type;
481   }
482 
483   static inline const TypeFunc* array_partition_Type() {
484     assert(_array_partition_Type != nullptr, "should be initialized");
485     return _array_partition_Type;
486   }
487 
488   // for aescrypt encrypt/decrypt operations, just three pointers returning void (length is constant)
489   static inline const TypeFunc* aescrypt_block_Type() {
490     assert(_aescrypt_block_Type != nullptr, "should be initialized");
491     return _aescrypt_block_Type;
492   }
493 
494   // for cipherBlockChaining calls of aescrypt encrypt/decrypt, four pointers and a length, returning int
495   static inline const TypeFunc* cipherBlockChaining_aescrypt_Type() {
496     assert(_cipherBlockChaining_aescrypt_Type != nullptr, "should be initialized");
497     return _cipherBlockChaining_aescrypt_Type;
498   }
499 
500   // for electronicCodeBook calls of aescrypt encrypt/decrypt, three pointers and a length, returning int
501   static inline const TypeFunc* electronicCodeBook_aescrypt_Type() {
502     assert(_electronicCodeBook_aescrypt_Type != nullptr, "should be initialized");
503     return _electronicCodeBook_aescrypt_Type;
504   }
505 
506   //for counterMode calls of aescrypt encrypt/decrypt, four pointers and a length, returning int
507   static inline const TypeFunc* counterMode_aescrypt_Type() {
508     assert(_counterMode_aescrypt_Type != nullptr, "should be initialized");
509     return _counterMode_aescrypt_Type;
510   }
511 
512   //for counterMode calls of aescrypt encrypt/decrypt, four pointers and a length, returning int
513   static inline const TypeFunc* galoisCounterMode_aescrypt_Type() {
514     assert(_galoisCounterMode_aescrypt_Type != nullptr, "should be initialized");
515     return _galoisCounterMode_aescrypt_Type;
516   }
517 
518   /*
519    * void implCompress(byte[] buf, int ofs)
520    */
521   static inline const TypeFunc* digestBase_implCompress_Type(bool is_sha3) {
522     assert((_digestBase_implCompress_with_sha3_Type != nullptr) &&
523            (_digestBase_implCompress_without_sha3_Type != nullptr), "should be initialized");
524     return is_sha3 ? _digestBase_implCompress_with_sha3_Type : _digestBase_implCompress_without_sha3_Type;
525   }
526 
527   /*
528    * int implCompressMultiBlock(byte[] b, int ofs, int limit)
529    */
530   static inline const TypeFunc* digestBase_implCompressMB_Type(bool is_sha3) {
531     assert((_digestBase_implCompressMB_with_sha3_Type != nullptr) &&
532            (_digestBase_implCompressMB_without_sha3_Type != nullptr), "should be initialized");
533     return is_sha3 ? _digestBase_implCompressMB_with_sha3_Type : _digestBase_implCompressMB_without_sha3_Type;
534   }
535 
536   static inline const TypeFunc* double_keccak_Type() {
537     assert(_double_keccak_Type != nullptr, "should be initialized");
538     return _double_keccak_Type;
539   }
540 
541   static inline const TypeFunc* multiplyToLen_Type() {
542     assert(_multiplyToLen_Type != nullptr, "should be initialized");
543     return _multiplyToLen_Type;
544   }
545 
546   static inline const TypeFunc* montgomeryMultiply_Type() {
547     assert(_montgomeryMultiply_Type != nullptr, "should be initialized");
548     return _montgomeryMultiply_Type;
549   }
550 
551   static inline const TypeFunc* montgomerySquare_Type() {
552     assert(_montgomerySquare_Type != nullptr, "should be initialized");
553     return _montgomerySquare_Type;
554   }
555 
556   static inline const TypeFunc* squareToLen_Type() {
557     assert(_squareToLen_Type != nullptr, "should be initialized");
558     return _squareToLen_Type;
559   }
560 
561   // for mulAdd calls, 2 pointers and 3 ints, returning int
562   static inline const TypeFunc* mulAdd_Type() {
563     assert(_mulAdd_Type != nullptr, "should be initialized");
564     return _mulAdd_Type;
565   }
566 
567   static inline const TypeFunc* bigIntegerShift_Type() {
568     assert(_bigIntegerShift_Type != nullptr, "should be initialized");
569     return _bigIntegerShift_Type;
570   }
571 
572   static inline const TypeFunc* vectorizedMismatch_Type() {
573     assert(_vectorizedMismatch_Type != nullptr, "should be initialized");
574     return _vectorizedMismatch_Type;
575   }
576 
577   // GHASH block processing
578   static inline const TypeFunc* ghash_processBlocks_Type() {
579     assert(_ghash_processBlocks_Type != nullptr, "should be initialized");
580     return _ghash_processBlocks_Type;
581   }
582 
583   // ChaCha20 Block function
584   static inline const TypeFunc* chacha20Block_Type() {
585     assert(_chacha20Block_Type != nullptr, "should be initialized");
586     return _chacha20Block_Type;
587   }
588 
589   static inline const TypeFunc* kyberNtt_Type() {
590     assert(_kyberNtt_Type != nullptr, "should be initialized");
591     return _kyberNtt_Type;
592   }
593 
594   static inline const TypeFunc* kyberInverseNtt_Type() {
595     assert(_kyberInverseNtt_Type != nullptr, "should be initialized");
596     return _kyberInverseNtt_Type;
597   }
598 
599   static inline const TypeFunc* kyberNttMult_Type() {
600     assert(_kyberNttMult_Type != nullptr, "should be initialized");
601     return _kyberNttMult_Type;
602   }
603 
604   static inline const TypeFunc* kyberAddPoly_2_Type() {
605     assert(_kyberAddPoly_2_Type != nullptr, "should be initialized");
606     return _kyberAddPoly_2_Type;
607   }
608 
609   static inline const TypeFunc* kyberAddPoly_3_Type() {
610     assert(_kyberAddPoly_3_Type != nullptr, "should be initialized");
611     return _kyberAddPoly_3_Type;
612   }
613 
614   static inline const TypeFunc* kyber12To16_Type() {
615     assert(_kyber12To16_Type != nullptr, "should be initialized");
616     return _kyber12To16_Type;
617   }
618 
619   static inline const TypeFunc* kyberBarrettReduce_Type() {
620     assert(_kyberBarrettReduce_Type != nullptr, "should be initialized");
621     return _kyberBarrettReduce_Type;
622   }
623 
624   static inline const TypeFunc* dilithiumAlmostNtt_Type() {
625     assert(_dilithiumAlmostNtt_Type != nullptr, "should be initialized");
626     return _dilithiumAlmostNtt_Type;
627   }
628 
629   static inline const TypeFunc* dilithiumAlmostInverseNtt_Type() {
630     assert(_dilithiumAlmostInverseNtt_Type != nullptr, "should be initialized");
631     return _dilithiumAlmostInverseNtt_Type;
632   }
633 
634   static inline const TypeFunc* dilithiumNttMult_Type() {
635     assert(_dilithiumNttMult_Type != nullptr, "should be initialized");
636     return _dilithiumNttMult_Type;
637   }
638 
639   static inline const TypeFunc* dilithiumMontMulByConstant_Type() {
640     assert(_dilithiumMontMulByConstant_Type != nullptr, "should be initialized");
641     return _dilithiumMontMulByConstant_Type;
642   }
643 
644   static inline const TypeFunc* dilithiumDecomposePoly_Type() {
645     assert(_dilithiumDecomposePoly_Type != nullptr, "should be initialized");
646     return _dilithiumDecomposePoly_Type;
647   }
648 
649   // Base64 encode function
650   static inline const TypeFunc* base64_encodeBlock_Type() {
651     assert(_base64_encodeBlock_Type != nullptr, "should be initialized");
652     return _base64_encodeBlock_Type;
653   }
654 
655   // Base64 decode function
656   static inline const TypeFunc* base64_decodeBlock_Type() {
657     assert(_base64_decodeBlock_Type != nullptr, "should be initialized");
658     return _base64_decodeBlock_Type;
659   }
660 
661   // String IndexOf function
662   static inline const TypeFunc* string_IndexOf_Type() {
663     assert(_string_IndexOf_Type != nullptr, "should be initialized");
664     return _string_IndexOf_Type;
665   }
666 
667   // Poly1305 processMultipleBlocks function
668   static inline const TypeFunc* poly1305_processBlocks_Type() {
669     assert(_poly1305_processBlocks_Type != nullptr, "should be initialized");
670     return _poly1305_processBlocks_Type;
671   }
672 
673   // MontgomeryIntegerPolynomialP256 multiply function
674   static inline const TypeFunc* intpoly_montgomeryMult_P256_Type() {
675     assert(_intpoly_montgomeryMult_P256_Type != nullptr, "should be initialized");
676     return _intpoly_montgomeryMult_P256_Type;
677   }
678 
679   // IntegerPolynomial constant time assignment function
680   static inline const TypeFunc* intpoly_assign_Type() {
681     assert(_intpoly_assign_Type != nullptr, "should be initialized");
682     return _intpoly_assign_Type;
683   }
684 
685   /**
686    * int updateBytesCRC32(int crc, byte* b, int len)
687    */
688   static inline const TypeFunc* updateBytesCRC32_Type() {
689     assert(_updateBytesCRC32_Type != nullptr, "should be initialized");
690     return _updateBytesCRC32_Type;
691   }
692 
693   /**
694    * int updateBytesCRC32C(int crc, byte* buf, int len, int* table)
695    */
696   static inline const TypeFunc* updateBytesCRC32C_Type() {
697     assert(_updateBytesCRC32C_Type != nullptr, "should be initialized");
698     return _updateBytesCRC32C_Type;
699   }
700 
701   /**
702    *  int updateBytesAdler32(int adler, bytes* b, int off, int len)
703    */
704   static inline const TypeFunc* updateBytesAdler32_Type() {
705     assert(_updateBytesAdler32_Type != nullptr, "should be initialized");
706     return _updateBytesAdler32_Type;
707   }
708 
709 
710   // leaf on stack replacement interpreter accessor types
711   static inline const TypeFunc* osr_end_Type() {
712     assert(_osr_end_Type != nullptr, "should be initialized");
713     return _osr_end_Type;
714   }
715 
716   static inline const TypeFunc* register_finalizer_Type() {
717     assert(_register_finalizer_Type != nullptr, "should be initialized");
718     return _register_finalizer_Type;
719   }
720 
721 #if INCLUDE_JFR
722   static inline const TypeFunc* class_id_load_barrier_Type() {
723     assert(_class_id_load_barrier_Type != nullptr, "should be initialized");
724     return _class_id_load_barrier_Type;
725   }
726 #endif // INCLUDE_JFR
727 
728   static const TypeFunc* load_unknown_inline_Type();
729   static const TypeFunc* store_unknown_inline_Type();
730 
731   static const TypeFunc* store_inline_type_fields_Type();
732   static const TypeFunc* pack_inline_type_Type();
733 
734 #if INCLUDE_JVMTI
735   static inline const TypeFunc* notify_jvmti_vthread_Type() {
736     assert(_notify_jvmti_vthread_Type != nullptr, "should be initialized");
737     return _notify_jvmti_vthread_Type;
738   }
739 #endif
740 
741   // Dtrace support. entry and exit probes have the same signature
742   static inline const TypeFunc* dtrace_method_entry_exit_Type() {
743     assert(_dtrace_method_entry_exit_Type != nullptr, "should be initialized");
744     return _dtrace_method_entry_exit_Type;
745   }
746 
747   static inline const TypeFunc* dtrace_object_alloc_Type() {
748     assert(_dtrace_object_alloc_Type != nullptr, "should be initialized");
749     return _dtrace_object_alloc_Type;
750   }
751 
752  private:
753  static NamedCounter * volatile _named_counters;
754 
755  public:
756  // helper function which creates a named counter labeled with the
757  // if they are available
758  static NamedCounter* new_named_counter(JVMState* jvms, NamedCounter::CounterTag tag);
759 
760  // dumps all the named counters
761  static void          print_named_counters();
762 
763  static void          initialize_types();
764 };
765 
766 #endif // SHARE_OPTO_RUNTIME_HPP