< prev index next >

src/hotspot/share/opto/runtime.cpp

Print this page
*** 1,7 ***
  /*
!  * Copyright (c) 1998, 2023, Oracle and/or its affiliates. All rights reserved.
   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   *
   * This code is free software; you can redistribute it and/or modify it
   * under the terms of the GNU General Public License version 2 only, as
   * published by the Free Software Foundation.
--- 1,7 ---
  /*
!  * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
   * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   *
   * This code is free software; you can redistribute it and/or modify it
   * under the terms of the GNU General Public License version 2 only, as
   * published by the Free Software Foundation.

*** 31,10 ***
--- 31,11 ---
  #include "code/nmethod.hpp"
  #include "code/pcDesc.hpp"
  #include "code/scopeDesc.hpp"
  #include "code/vtableStubs.hpp"
  #include "compiler/compileBroker.hpp"
+ #include "compiler/compilerDefinitions.inline.hpp"
  #include "compiler/oopMap.hpp"
  #include "gc/g1/heapRegion.hpp"
  #include "gc/shared/barrierSet.hpp"
  #include "gc/shared/collectedHeap.hpp"
  #include "gc/shared/gcLocker.hpp"

*** 64,20 ***
--- 65,23 ---
  #include "prims/jvmtiExport.hpp"
  #include "runtime/atomic.hpp"
  #include "runtime/frame.inline.hpp"
  #include "runtime/handles.inline.hpp"
  #include "runtime/interfaceSupport.inline.hpp"
+ #include "runtime/java.hpp"
  #include "runtime/javaCalls.hpp"
+ #include "runtime/perfData.inline.hpp"
  #include "runtime/sharedRuntime.hpp"
  #include "runtime/signature.hpp"
  #include "runtime/stackWatermarkSet.hpp"
  #include "runtime/synchronizer.hpp"
  #include "runtime/threadCritical.hpp"
  #include "runtime/threadWXSetters.inline.hpp"
  #include "runtime/vframe.hpp"
  #include "runtime/vframeArray.hpp"
  #include "runtime/vframe_hp.hpp"
+ #include "services/management.hpp"
  #include "utilities/copy.hpp"
  #include "utilities/preserveException.hpp"
  
  
  // For debugging purposes:

*** 106,19 ***
--- 110,22 ---
  address OptoRuntime::_monitor_notifyAll_Java                      = nullptr;
  address OptoRuntime::_rethrow_Java                                = nullptr;
  
  address OptoRuntime::_slow_arraycopy_Java                         = nullptr;
  address OptoRuntime::_register_finalizer_Java                     = nullptr;
+ address OptoRuntime::_class_init_barrier_Java                     = nullptr;
  #if INCLUDE_JVMTI
  address OptoRuntime::_notify_jvmti_vthread_start                  = nullptr;
  address OptoRuntime::_notify_jvmti_vthread_end                    = nullptr;
  address OptoRuntime::_notify_jvmti_vthread_mount                  = nullptr;
  address OptoRuntime::_notify_jvmti_vthread_unmount                = nullptr;
  #endif
  
  ExceptionBlob* OptoRuntime::_exception_blob;
  
+ PerfCounter* _perf_OptoRuntime_class_init_barrier_redundant_count = nullptr;
+ 
  // This should be called in an assertion at the start of OptoRuntime routines
  // which are entered from compiled code (all of them)
  #ifdef ASSERT
  static bool check_compiled_frame(JavaThread* thread) {
    assert(thread->last_frame().is_runtime_frame(), "cannot call runtime directly from compiled code");

*** 136,10 ***
--- 143,11 ---
  #define gen(env, var, type_func_gen, c_func, fancy_jump, pass_tls, return_pc) \
    var = generate_stub(env, type_func_gen, CAST_FROM_FN_PTR(address, c_func), #var, fancy_jump, pass_tls, return_pc); \
    if (var == nullptr) { return false; }
  
  bool OptoRuntime::generate(ciEnv* env) {
+   init_counters();
  
    generate_exception_blob();
  
    // Note: tls: Means fetching the return oop out of the thread-local storage
    //

*** 164,10 ***
--- 172,11 ---
    gen(env, _monitor_notifyAll_Java         , monitor_notify_Type          , monitor_notifyAll_C             ,    0 , false, false);
    gen(env, _rethrow_Java                   , rethrow_Type                 , rethrow_C                       ,    2 , true , true );
  
    gen(env, _slow_arraycopy_Java            , slow_arraycopy_Type          , SharedRuntime::slow_arraycopy_C ,    0 , false, false);
    gen(env, _register_finalizer_Java        , register_finalizer_Type      , register_finalizer              ,    0 , false, false);
+   gen(env, _class_init_barrier_Java        , class_init_barrier_Type      , class_init_barrier              ,    0 , false, false);
  
    return true;
  }
  
  #undef gen

*** 179,11 ***
                                     const char *name, int is_fancy_jump,
                                     bool pass_tls,
                                     bool return_pc) {
  
    // Matching the default directive, we currently have no method to match.
!   DirectiveSet* directive = DirectivesStack::getDefaultDirective(CompileBroker::compiler(CompLevel_full_optimization));
    ResourceMark rm;
    Compile C(env, gen, C_function, name, is_fancy_jump, pass_tls, return_pc, directive);
    DirectivesStack::release(directive);
    return  C.stub_entry_point();
  }
--- 188,11 ---
                                     const char *name, int is_fancy_jump,
                                     bool pass_tls,
                                     bool return_pc) {
  
    // Matching the default directive, we currently have no method to match.
!   DirectiveSet* directive = DirectivesStack::getDefaultDirective(CompilerThread::current()->compiler());
    ResourceMark rm;
    Compile C(env, gen, C_function, name, is_fancy_jump, pass_tls, return_pc, directive);
    DirectivesStack::release(directive);
    return  C.stub_entry_point();
  }

*** 209,11 ***
  //=============================allocation======================================
  // We failed the fast-path allocation.  Now we need to do a scavenge or GC
  // and try allocation again.
  
  // object allocation
! JRT_BLOCK_ENTRY(void, OptoRuntime::new_instance_C(Klass* klass, JavaThread* current))
    JRT_BLOCK;
  #ifndef PRODUCT
    SharedRuntime::_new_instance_ctr++;         // new instance requires GC
  #endif
    assert(check_compiled_frame(current), "incorrect caller");
--- 218,11 ---
  //=============================allocation======================================
  // We failed the fast-path allocation.  Now we need to do a scavenge or GC
  // and try allocation again.
  
  // object allocation
! JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, new_instance_C, OptoRuntime::new_instance_C(Klass* klass, JavaThread* current))
    JRT_BLOCK;
  #ifndef PRODUCT
    SharedRuntime::_new_instance_ctr++;         // new instance requires GC
  #endif
    assert(check_compiled_frame(current), "incorrect caller");

*** 247,11 ***
    SharedRuntime::on_slowpath_allocation_exit(current);
  JRT_END
  
  
  // array allocation
! JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_C(Klass* array_type, int len, JavaThread* current))
    JRT_BLOCK;
  #ifndef PRODUCT
    SharedRuntime::_new_array_ctr++;            // new array requires GC
  #endif
    assert(check_compiled_frame(current), "incorrect caller");
--- 256,11 ---
    SharedRuntime::on_slowpath_allocation_exit(current);
  JRT_END
  
  
  // array allocation
! JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, new_array_C, OptoRuntime::new_array_C(Klass* array_type, int len, JavaThread* current))
    JRT_BLOCK;
  #ifndef PRODUCT
    SharedRuntime::_new_array_ctr++;            // new array requires GC
  #endif
    assert(check_compiled_frame(current), "incorrect caller");

*** 284,11 ***
    // inform GC that we won't do card marks for initializing writes.
    SharedRuntime::on_slowpath_allocation_exit(current);
  JRT_END
  
  // array allocation without zeroing
! JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(Klass* array_type, int len, JavaThread* current))
    JRT_BLOCK;
  #ifndef PRODUCT
    SharedRuntime::_new_array_ctr++;            // new array requires GC
  #endif
    assert(check_compiled_frame(current), "incorrect caller");
--- 293,11 ---
    // inform GC that we won't do card marks for initializing writes.
    SharedRuntime::on_slowpath_allocation_exit(current);
  JRT_END
  
  // array allocation without zeroing
! JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, new_array_nozero_C, OptoRuntime::new_array_nozero_C(Klass* array_type, int len, JavaThread* current))
    JRT_BLOCK;
  #ifndef PRODUCT
    SharedRuntime::_new_array_ctr++;            // new array requires GC
  #endif
    assert(check_compiled_frame(current), "incorrect caller");

*** 333,11 ***
  JRT_END
  
  // Note: multianewarray for one dimension is handled inline by GraphKit::new_array.
  
  // multianewarray for 2 dimensions
! JRT_ENTRY(void, OptoRuntime::multianewarray2_C(Klass* elem_type, int len1, int len2, JavaThread* current))
  #ifndef PRODUCT
    SharedRuntime::_multi2_ctr++;                // multianewarray for 1 dimension
  #endif
    assert(check_compiled_frame(current), "incorrect caller");
    assert(elem_type->is_klass(), "not a class");
--- 342,11 ---
  JRT_END
  
  // Note: multianewarray for one dimension is handled inline by GraphKit::new_array.
  
  // multianewarray for 2 dimensions
! JRT_ENTRY_PROF(void, OptoRuntime, multianewarray2_C, OptoRuntime::multianewarray2_C(Klass* elem_type, int len1, int len2, JavaThread* current))
  #ifndef PRODUCT
    SharedRuntime::_multi2_ctr++;                // multianewarray for 1 dimension
  #endif
    assert(check_compiled_frame(current), "incorrect caller");
    assert(elem_type->is_klass(), "not a class");

*** 349,11 ***
    deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
    current->set_vm_result(obj);
  JRT_END
  
  // multianewarray for 3 dimensions
! JRT_ENTRY(void, OptoRuntime::multianewarray3_C(Klass* elem_type, int len1, int len2, int len3, JavaThread* current))
  #ifndef PRODUCT
    SharedRuntime::_multi3_ctr++;                // multianewarray for 1 dimension
  #endif
    assert(check_compiled_frame(current), "incorrect caller");
    assert(elem_type->is_klass(), "not a class");
--- 358,11 ---
    deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
    current->set_vm_result(obj);
  JRT_END
  
  // multianewarray for 3 dimensions
! JRT_ENTRY_PROF(void, OptoRuntime, multianewarray3_C, OptoRuntime::multianewarray3_C(Klass* elem_type, int len1, int len2, int len3, JavaThread* current))
  #ifndef PRODUCT
    SharedRuntime::_multi3_ctr++;                // multianewarray for 1 dimension
  #endif
    assert(check_compiled_frame(current), "incorrect caller");
    assert(elem_type->is_klass(), "not a class");

*** 366,11 ***
    deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
    current->set_vm_result(obj);
  JRT_END
  
  // multianewarray for 4 dimensions
! JRT_ENTRY(void, OptoRuntime::multianewarray4_C(Klass* elem_type, int len1, int len2, int len3, int len4, JavaThread* current))
  #ifndef PRODUCT
    SharedRuntime::_multi4_ctr++;                // multianewarray for 1 dimension
  #endif
    assert(check_compiled_frame(current), "incorrect caller");
    assert(elem_type->is_klass(), "not a class");
--- 375,11 ---
    deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
    current->set_vm_result(obj);
  JRT_END
  
  // multianewarray for 4 dimensions
! JRT_ENTRY_PROF(void, OptoRuntime, multianewarray4_C, OptoRuntime::multianewarray4_C(Klass* elem_type, int len1, int len2, int len3, int len4, JavaThread* current))
  #ifndef PRODUCT
    SharedRuntime::_multi4_ctr++;                // multianewarray for 1 dimension
  #endif
    assert(check_compiled_frame(current), "incorrect caller");
    assert(elem_type->is_klass(), "not a class");

*** 402,11 ***
    oop obj = ArrayKlass::cast(elem_type)->multi_allocate(5, dims, THREAD);
    deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
    current->set_vm_result(obj);
  JRT_END
  
! JRT_ENTRY(void, OptoRuntime::multianewarrayN_C(Klass* elem_type, arrayOopDesc* dims, JavaThread* current))
    assert(check_compiled_frame(current), "incorrect caller");
    assert(elem_type->is_klass(), "not a class");
    assert(oop(dims)->is_typeArray(), "not an array");
  
    ResourceMark rm;
--- 411,11 ---
    oop obj = ArrayKlass::cast(elem_type)->multi_allocate(5, dims, THREAD);
    deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
    current->set_vm_result(obj);
  JRT_END
  
! JRT_ENTRY_PROF(void, OptoRuntime, multianewarrayN_C, OptoRuntime::multianewarrayN_C(Klass* elem_type, arrayOopDesc* dims, JavaThread* current))
    assert(check_compiled_frame(current), "incorrect caller");
    assert(elem_type->is_klass(), "not a class");
    assert(oop(dims)->is_typeArray(), "not an array");
  
    ResourceMark rm;

*** 420,11 ***
    oop obj = ArrayKlass::cast(elem_type)->multi_allocate(len, c_dims, THREAD);
    deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
    current->set_vm_result(obj);
  JRT_END
  
! JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notify_C(oopDesc* obj, JavaThread* current))
  
    // Very few notify/notifyAll operations find any threads on the waitset, so
    // the dominant fast-path is to simply return.
    // Relatedly, it's critical that notify/notifyAll be fast in order to
    // reduce lock hold times.
--- 429,11 ---
    oop obj = ArrayKlass::cast(elem_type)->multi_allocate(len, c_dims, THREAD);
    deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
    current->set_vm_result(obj);
  JRT_END
  
! JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, monitor_notify_C, OptoRuntime::monitor_notify_C(oopDesc* obj, JavaThread* current))
  
    // Very few notify/notifyAll operations find any threads on the waitset, so
    // the dominant fast-path is to simply return.
    // Relatedly, it's critical that notify/notifyAll be fast in order to
    // reduce lock hold times.

*** 442,11 ***
    Handle h_obj(current, obj);
    ObjectSynchronizer::notify(h_obj, CHECK);
    JRT_BLOCK_END;
  JRT_END
  
! JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notifyAll_C(oopDesc* obj, JavaThread* current))
  
    if (!SafepointSynchronize::is_synchronizing() ) {
      if (ObjectSynchronizer::quick_notify(obj, current, true)) {
        return;
      }
--- 451,11 ---
    Handle h_obj(current, obj);
    ObjectSynchronizer::notify(h_obj, CHECK);
    JRT_BLOCK_END;
  JRT_END
  
! JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, monitor_notifyAll_C, OptoRuntime::monitor_notifyAll_C(oopDesc* obj, JavaThread* current))
  
    if (!SafepointSynchronize::is_synchronizing() ) {
      if (ObjectSynchronizer::quick_notify(obj, current, true)) {
        return;
      }

*** 1413,11 ***
  static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg);
  
  // The method is an entry that is always called by a C++ method not
  // directly from compiled code. Compiled code will call the C++ method following.
  // We can't allow async exception to be installed during  exception processing.
! JRT_ENTRY_NO_ASYNC(address, OptoRuntime::handle_exception_C_helper(JavaThread* current, nmethod* &nm))
    // The frame we rethrow the exception to might not have been processed by the GC yet.
    // The stack watermark barrier takes care of detecting that and ensuring the frame
    // has updated oops.
    StackWatermarkSet::after_unwind(current);
  
--- 1422,11 ---
  static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg);
  
  // The method is an entry that is always called by a C++ method not
  // directly from compiled code. Compiled code will call the C++ method following.
  // We can't allow async exception to be installed during  exception processing.
! JRT_ENTRY_NO_ASYNC_PROF(address, OptoRuntime, handle_exception_C_helper, OptoRuntime::handle_exception_C_helper(JavaThread* current, nmethod* &nm))
    // The frame we rethrow the exception to might not have been processed by the GC yet.
    // The stack watermark barrier takes care of detecting that and ensuring the frame
    // has updated oops.
    StackWatermarkSet::after_unwind(current);
  

*** 1689,10 ***
--- 1698,24 ---
    const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
  
    return TypeFunc::make(domain,range);
  }
  
+ const TypeFunc *OptoRuntime::class_init_barrier_Type() {
+   // create input type (domain)
+   const Type** fields = TypeTuple::fields(1);
+   fields[TypeFunc::Parms+0] = TypeKlassPtr::NOTNULL;
+   // // The JavaThread* is passed to each routine as the last argument
+   // fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL;  // JavaThread *; Executing thread
+   const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+1, fields);
+ 
+   // create result type (range)
+   fields = TypeTuple::fields(0);
+   const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
+   return TypeFunc::make(domain,range);
+ }
+ 
  #if INCLUDE_JFR
  const TypeFunc *OptoRuntime::class_id_load_barrier_Type() {
    // create input type (domain)
    const Type **fields = TypeTuple::fields(1);
    fields[TypeFunc::Parms+0] = TypeInstPtr::KLASS;

*** 1739,16 ***
  
    return TypeFunc::make(domain,range);
  }
  
  
! JRT_ENTRY_NO_ASYNC(void, OptoRuntime::register_finalizer(oopDesc* obj, JavaThread* current))
    assert(oopDesc::is_oop(obj), "must be a valid oop");
    assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
    InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
  JRT_END
  
  //-----------------------------------------------------------------------------
  
  NamedCounter * volatile OptoRuntime::_named_counters = nullptr;
  
  //
--- 1762,25 ---
  
    return TypeFunc::make(domain,range);
  }
  
  
! JRT_ENTRY_NO_ASYNC_PROF(void, OptoRuntime, register_finalizer, OptoRuntime::register_finalizer(oopDesc* obj, JavaThread* current))
    assert(oopDesc::is_oop(obj), "must be a valid oop");
    assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
    InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
  JRT_END
  
+ JRT_ENTRY_NO_ASYNC_PROF(void, OptoRuntime, class_init_barrier, OptoRuntime::class_init_barrier(Klass* k, JavaThread* current))
+   InstanceKlass* ik = InstanceKlass::cast(k);
+   if (ik->should_be_initialized()) {
+     ik->initialize(CHECK);
+   } else if (UsePerfData) {
+     _perf_OptoRuntime_class_init_barrier_redundant_count->inc();
+   }
+ JRT_END
+ 
  //-----------------------------------------------------------------------------
  
  NamedCounter * volatile OptoRuntime::_named_counters = nullptr;
  
  //

*** 1859,5 ***
--- 1891,71 ---
    tempst.print(" at " INTPTR_FORMAT,  p2i(exception_pc));
    tempst.print("]");
  
    st->print_raw_cr(tempst.freeze());
  }
+ 
+ #define DO_COUNTERS2(macro2, macro1) \
+   macro2(OptoRuntime, new_instance_C) \
+   macro2(OptoRuntime, new_array_C) \
+   macro2(OptoRuntime, new_array_nozero_C) \
+   macro2(OptoRuntime, multianewarray2_C) \
+   macro2(OptoRuntime, multianewarray3_C) \
+   macro2(OptoRuntime, multianewarray4_C) \
+   macro2(OptoRuntime, multianewarrayN_C) \
+   macro2(OptoRuntime, monitor_notify_C) \
+   macro2(OptoRuntime, monitor_notifyAll_C) \
+   macro2(OptoRuntime, handle_exception_C_helper) \
+   macro2(OptoRuntime, register_finalizer) \
+   macro2(OptoRuntime, class_init_barrier) \
+   macro1(OptoRuntime, class_init_barrier_redundant)
+ 
+ #define INIT_COUNTER_TIME_AND_CNT(sub, name) \
+   NEWPERFTICKCOUNTERS(_perf_##sub##_##name##_timer, SUN_CI, #sub "::" #name); \
+   NEWPERFEVENTCOUNTER(_perf_##sub##_##name##_count, SUN_CI, #sub "::" #name "_count");
+ 
+ #define INIT_COUNTER_CNT(sub, name) \
+   NEWPERFEVENTCOUNTER(_perf_##sub##_##name##_count, SUN_CI, #sub "::" #name "_count");
+ 
+ void OptoRuntime::init_counters() {
+   assert(CompilerConfig::is_c2_enabled(), "");
+ 
+   if (UsePerfData) {
+     EXCEPTION_MARK;
+ 
+     DO_COUNTERS2(INIT_COUNTER_TIME_AND_CNT, INIT_COUNTER_CNT)
+ 
+     if (HAS_PENDING_EXCEPTION) {
+       vm_exit_during_initialization("jvm_perf_init failed unexpectedly");
+     }
+   }
+ }
+ #undef INIT_COUNTER_TIME_AND_CNT
+ #undef INIT_COUNTER_CNT
+ 
+ #define PRINT_COUNTER_TIME_AND_CNT(sub, name) { \
+   jlong count = _perf_##sub##_##name##_count->get_value(); \
+   if (count > 0) { \
+     st->print_cr("  %-30s = %4ldms (elapsed) %4ldms (thread) (%5ld events)", #sub "::" #name, \
+                  _perf_##sub##_##name##_timer->elapsed_counter_value_ms(), \
+                  _perf_##sub##_##name##_timer->thread_counter_value_ms(), \
+                  count); \
+   }}
+ 
+ #define PRINT_COUNTER_CNT(sub, name) { \
+   jlong count = _perf_##sub##_##name##_count->get_value(); \
+   if (count > 0) { \
+     st->print_cr("  %-30s = %5ld events", #name, count); \
+   }}
+ 
+ void OptoRuntime::print_counters_on(outputStream* st) {
+   if (UsePerfData && ProfileRuntimeCalls && CompilerConfig::is_c2_enabled()) {
+     DO_COUNTERS2(PRINT_COUNTER_TIME_AND_CNT, PRINT_COUNTER_CNT)
+   } else {
+     st->print_cr("  OptoRuntime: no info (%s is disabled)",
+                  (!CompilerConfig::is_c2_enabled() ? "C2" : (UsePerfData ? "ProfileRuntimeCalls" : "UsePerfData")));
+   }
+ }
+ 
+ #undef PRINT_COUNTER_TIME_AND_CNT
+ #undef PRINT_COUNTER_CNT
+ #undef DO_COUNTERS2
< prev index next >