15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "classfile/vmClasses.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "code/codeCache.hpp"
28 #include "code/compiledIC.hpp"
29 #include "code/nmethod.hpp"
30 #include "code/pcDesc.hpp"
31 #include "code/scopeDesc.hpp"
32 #include "code/vtableStubs.hpp"
33 #include "compiler/compilationMemoryStatistic.hpp"
34 #include "compiler/compileBroker.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/g1/g1HeapRegion.hpp"
37 #include "gc/shared/barrierSet.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gcLocker.hpp"
40 #include "interpreter/bytecode.hpp"
41 #include "interpreter/interpreter.hpp"
42 #include "interpreter/linkResolver.hpp"
43 #include "logging/log.hpp"
44 #include "logging/logStream.hpp"
45 #include "memory/oopFactory.hpp"
46 #include "memory/resourceArea.hpp"
47 #include "oops/objArrayKlass.hpp"
48 #include "oops/klass.inline.hpp"
49 #include "oops/oop.inline.hpp"
50 #include "oops/typeArrayOop.inline.hpp"
51 #include "opto/ad.hpp"
52 #include "opto/addnode.hpp"
53 #include "opto/callnode.hpp"
54 #include "opto/cfgnode.hpp"
55 #include "opto/graphKit.hpp"
56 #include "opto/machnode.hpp"
57 #include "opto/matcher.hpp"
58 #include "opto/memnode.hpp"
59 #include "opto/mulnode.hpp"
60 #include "opto/output.hpp"
61 #include "opto/runtime.hpp"
62 #include "opto/subnode.hpp"
63 #include "prims/jvmtiExport.hpp"
64 #include "runtime/atomic.hpp"
65 #include "runtime/frame.inline.hpp"
66 #include "runtime/handles.inline.hpp"
67 #include "runtime/interfaceSupport.inline.hpp"
68 #include "runtime/javaCalls.hpp"
69 #include "runtime/sharedRuntime.hpp"
70 #include "runtime/signature.hpp"
71 #include "runtime/stackWatermarkSet.hpp"
72 #include "runtime/synchronizer.hpp"
73 #include "runtime/threadCritical.hpp"
74 #include "runtime/threadWXSetters.inline.hpp"
75 #include "runtime/vframe.hpp"
76 #include "runtime/vframeArray.hpp"
77 #include "runtime/vframe_hp.hpp"
78 #include "utilities/copy.hpp"
79 #include "utilities/preserveException.hpp"
80
81
82 // For debugging purposes:
83 // To force FullGCALot inside a runtime function, add the following two lines
84 //
85 // Universe::release_fullgc_alot_dummy();
86 // Universe::heap()->collect();
87 //
88 // At command line specify the parameters: -XX:+FullGCALot -XX:FullGCALotStart=100000000
89
90
91 #define C2_BLOB_FIELD_DEFINE(name, type) \
92 type OptoRuntime:: BLOB_FIELD_NAME(name) = nullptr;
93 #define C2_STUB_FIELD_NAME(name) _ ## name ## _Java
94 #define C2_STUB_FIELD_DEFINE(name, f, t, r) \
95 address OptoRuntime:: C2_STUB_FIELD_NAME(name) = nullptr;
96 #define C2_JVMTI_STUB_FIELD_DEFINE(name) \
97 address OptoRuntime:: STUB_FIELD_NAME(name) = nullptr;
98 C2_STUBS_DO(C2_BLOB_FIELD_DEFINE, C2_STUB_FIELD_DEFINE, C2_JVMTI_STUB_FIELD_DEFINE)
99 #undef C2_BLOB_FIELD_DEFINE
100 #undef C2_STUB_FIELD_DEFINE
101 #undef C2_JVMTI_STUB_FIELD_DEFINE
102
103 #define C2_BLOB_NAME_DEFINE(name, type) "C2 Runtime " # name "_blob",
104 #define C2_STUB_NAME_DEFINE(name, f, t, r) "C2 Runtime " # name,
105 #define C2_JVMTI_STUB_NAME_DEFINE(name) "C2 Runtime " # name,
106 const char* OptoRuntime::_stub_names[] = {
107 C2_STUBS_DO(C2_BLOB_NAME_DEFINE, C2_STUB_NAME_DEFINE, C2_JVMTI_STUB_NAME_DEFINE)
108 };
109 #undef C2_BLOB_NAME_DEFINE
110 #undef C2_STUB_NAME_DEFINE
111 #undef C2_JVMTI_STUB_NAME_DEFINE
112
113 // This should be called in an assertion at the start of OptoRuntime routines
114 // which are entered from compiled code (all of them)
115 #ifdef ASSERT
116 static bool check_compiled_frame(JavaThread* thread) {
117 assert(thread->last_frame().is_runtime_frame(), "cannot call runtime directly from compiled code");
118 RegisterMap map(thread,
119 RegisterMap::UpdateMap::skip,
120 RegisterMap::ProcessFrames::include,
121 RegisterMap::WalkContinuation::skip);
122 frame caller = thread->last_frame().sender(&map);
123 assert(caller.is_compiled_frame(), "not being called from compiled like code");
124 return true;
125 }
126 #endif // ASSERT
127
128 /*
129 #define gen(env, var, type_func_gen, c_func, fancy_jump, pass_tls, return_pc) \
130 var = generate_stub(env, type_func_gen, CAST_FROM_FN_PTR(address, c_func), #var, fancy_jump, pass_tls, return_pc); \
131 if (var == nullptr) { return false; }
132 */
159 C2_STUB_NAME(name), \
160 fancy_jump, \
161 pass_tls, \
162 pass_retpc); \
163 if (C2_STUB_FIELD_NAME(name) == nullptr) { return false; } \
164
165 #define C2_JVMTI_STUB_C_FUNC(name) CAST_FROM_FN_PTR(address, SharedRuntime::name)
166
167 #define GEN_C2_JVMTI_STUB(name) \
168 STUB_FIELD_NAME(name) = \
169 generate_stub(env, \
170 notify_jvmti_vthread_Type, \
171 C2_JVMTI_STUB_C_FUNC(name), \
172 C2_STUB_NAME(name), \
173 0, \
174 true, \
175 false); \
176 if (STUB_FIELD_NAME(name) == nullptr) { return false; } \
177
178 bool OptoRuntime::generate(ciEnv* env) {
179
180 C2_STUBS_DO(GEN_C2_BLOB, GEN_C2_STUB, GEN_C2_JVMTI_STUB)
181
182 return true;
183 }
184
185 #undef GEN_C2_BLOB
186
187 #undef C2_STUB_FIELD_NAME
188 #undef C2_STUB_TYPEFUNC
189 #undef C2_STUB_C_FUNC
190 #undef C2_STUB_NAME
191 #undef GEN_C2_STUB
192
193 #undef C2_JVMTI_STUB_C_FUNC
194 #undef GEN_C2_JVMTI_STUB
195 // #undef gen
196
197 const TypeFunc* OptoRuntime::_new_instance_Type = nullptr;
198 const TypeFunc* OptoRuntime::_new_array_Type = nullptr;
265 const TypeFunc* OptoRuntime::_updateBytesAdler32_Type = nullptr;
266 const TypeFunc* OptoRuntime::_osr_end_Type = nullptr;
267 const TypeFunc* OptoRuntime::_register_finalizer_Type = nullptr;
268 #if INCLUDE_JFR
269 const TypeFunc* OptoRuntime::_class_id_load_barrier_Type = nullptr;
270 #endif // INCLUDE_JFR
271 #if INCLUDE_JVMTI
272 const TypeFunc* OptoRuntime::_notify_jvmti_vthread_Type = nullptr;
273 #endif // INCLUDE_JVMTI
274 const TypeFunc* OptoRuntime::_dtrace_method_entry_exit_Type = nullptr;
275 const TypeFunc* OptoRuntime::_dtrace_object_alloc_Type = nullptr;
276
277 // Helper method to do generation of RunTimeStub's
278 address OptoRuntime::generate_stub(ciEnv* env,
279 TypeFunc_generator gen, address C_function,
280 const char *name, int is_fancy_jump,
281 bool pass_tls,
282 bool return_pc) {
283
284 // Matching the default directive, we currently have no method to match.
285 DirectiveSet* directive = DirectivesStack::getDefaultDirective(CompileBroker::compiler(CompLevel_full_optimization));
286 CompilationMemoryStatisticMark cmsm(directive);
287 ResourceMark rm;
288 Compile C(env, gen, C_function, name, is_fancy_jump, pass_tls, return_pc, directive);
289 DirectivesStack::release(directive);
290 return C.stub_entry_point();
291 }
292
293 const char* OptoRuntime::stub_name(address entry) {
294 #ifndef PRODUCT
295 CodeBlob* cb = CodeCache::find_blob(entry);
296 RuntimeStub* rs =(RuntimeStub *)cb;
297 assert(rs != nullptr && rs->is_runtime_stub(), "not a runtime stub");
298 return rs->name();
299 #else
300 // Fast implementation for product mode (maybe it should be inlined too)
301 return "runtime stub";
302 #endif
303 }
304
305 // local methods passed as arguments to stub generator that forward
309 oopDesc* dest, jint dest_pos,
310 jint length, JavaThread* thread) {
311 SharedRuntime::slow_arraycopy_C(src, src_pos, dest, dest_pos, length, thread);
312 }
313
314 void OptoRuntime::complete_monitor_locking_C(oopDesc* obj, BasicLock* lock, JavaThread* current) {
315 SharedRuntime::complete_monitor_locking_C(obj, lock, current);
316 }
317
318
319 //=============================================================================
320 // Opto compiler runtime routines
321 //=============================================================================
322
323
324 //=============================allocation======================================
325 // We failed the fast-path allocation. Now we need to do a scavenge or GC
326 // and try allocation again.
327
328 // object allocation
329 JRT_BLOCK_ENTRY(void, OptoRuntime::new_instance_C(Klass* klass, JavaThread* current))
330 JRT_BLOCK;
331 #ifndef PRODUCT
332 SharedRuntime::_new_instance_ctr++; // new instance requires GC
333 #endif
334 assert(check_compiled_frame(current), "incorrect caller");
335
336 // These checks are cheap to make and support reflective allocation.
337 int lh = klass->layout_helper();
338 if (Klass::layout_helper_needs_slow_path(lh) || !InstanceKlass::cast(klass)->is_initialized()) {
339 Handle holder(current, klass->klass_holder()); // keep the klass alive
340 klass->check_valid_for_instantiation(false, THREAD);
341 if (!HAS_PENDING_EXCEPTION) {
342 InstanceKlass::cast(klass)->initialize(THREAD);
343 }
344 }
345
346 if (!HAS_PENDING_EXCEPTION) {
347 // Scavenge and allocate an instance.
348 Handle holder(current, klass->klass_holder()); // keep the klass alive
349 oop result = InstanceKlass::cast(klass)->allocate_instance(THREAD);
350 current->set_vm_result_oop(result);
351
352 // Pass oops back through thread local storage. Our apparent type to Java
353 // is that we return an oop, but we can block on exit from this routine and
354 // a GC can trash the oop in C's return register. The generated stub will
355 // fetch the oop from TLS after any possible GC.
356 }
357
358 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
359 JRT_BLOCK_END;
360
361 // inform GC that we won't do card marks for initializing writes.
362 SharedRuntime::on_slowpath_allocation_exit(current);
363 JRT_END
364
365
366 // array allocation
367 JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_C(Klass* array_type, int len, JavaThread* current))
368 JRT_BLOCK;
369 #ifndef PRODUCT
370 SharedRuntime::_new_array_ctr++; // new array requires GC
371 #endif
372 assert(check_compiled_frame(current), "incorrect caller");
373
374 // Scavenge and allocate an instance.
375 oop result;
376
377 if (array_type->is_typeArray_klass()) {
378 // The oopFactory likes to work with the element type.
379 // (We could bypass the oopFactory, since it doesn't add much value.)
380 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
381 result = oopFactory::new_typeArray(elem_type, len, THREAD);
382 } else {
383 // Although the oopFactory likes to work with the elem_type,
384 // the compiler prefers the array_type, since it must already have
385 // that latter value in hand for the fast path.
386 Handle holder(current, array_type->klass_holder()); // keep the array klass alive
387 Klass* elem_type = ObjArrayKlass::cast(array_type)->element_klass();
388 result = oopFactory::new_objArray(elem_type, len, THREAD);
389 }
390
391 // Pass oops back through thread local storage. Our apparent type to Java
392 // is that we return an oop, but we can block on exit from this routine and
393 // a GC can trash the oop in C's return register. The generated stub will
394 // fetch the oop from TLS after any possible GC.
395 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
396 current->set_vm_result_oop(result);
397 JRT_BLOCK_END;
398
399 // inform GC that we won't do card marks for initializing writes.
400 SharedRuntime::on_slowpath_allocation_exit(current);
401 JRT_END
402
403 // array allocation without zeroing
404 JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(Klass* array_type, int len, JavaThread* current))
405 JRT_BLOCK;
406 #ifndef PRODUCT
407 SharedRuntime::_new_array_ctr++; // new array requires GC
408 #endif
409 assert(check_compiled_frame(current), "incorrect caller");
410
411 // Scavenge and allocate an instance.
412 oop result;
413
414 assert(array_type->is_typeArray_klass(), "should be called only for type array");
415 // The oopFactory likes to work with the element type.
416 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
417 result = oopFactory::new_typeArray_nozero(elem_type, len, THREAD);
418
419 // Pass oops back through thread local storage. Our apparent type to Java
420 // is that we return an oop, but we can block on exit from this routine and
421 // a GC can trash the oop in C's return register. The generated stub will
422 // fetch the oop from TLS after any possible GC.
423 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
424 current->set_vm_result_oop(result);
436 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
437 size_t hs_bytes = arrayOopDesc::base_offset_in_bytes(elem_type);
438 assert(is_aligned(hs_bytes, BytesPerInt), "must be 4 byte aligned");
439 HeapWord* obj = cast_from_oop<HeapWord*>(result);
440 if (!is_aligned(hs_bytes, BytesPerLong)) {
441 *reinterpret_cast<jint*>(reinterpret_cast<char*>(obj) + hs_bytes) = 0;
442 hs_bytes += BytesPerInt;
443 }
444
445 // Optimized zeroing.
446 assert(is_aligned(hs_bytes, BytesPerLong), "must be 8-byte aligned");
447 const size_t aligned_hs = hs_bytes / BytesPerLong;
448 Copy::fill_to_aligned_words(obj+aligned_hs, size-aligned_hs);
449 }
450
451 JRT_END
452
453 // Note: multianewarray for one dimension is handled inline by GraphKit::new_array.
454
455 // multianewarray for 2 dimensions
456 JRT_ENTRY(void, OptoRuntime::multianewarray2_C(Klass* elem_type, int len1, int len2, JavaThread* current))
457 #ifndef PRODUCT
458 SharedRuntime::_multi2_ctr++; // multianewarray for 1 dimension
459 #endif
460 assert(check_compiled_frame(current), "incorrect caller");
461 assert(elem_type->is_klass(), "not a class");
462 jint dims[2];
463 dims[0] = len1;
464 dims[1] = len2;
465 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
466 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(2, dims, THREAD);
467 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
468 current->set_vm_result_oop(obj);
469 JRT_END
470
471 // multianewarray for 3 dimensions
472 JRT_ENTRY(void, OptoRuntime::multianewarray3_C(Klass* elem_type, int len1, int len2, int len3, JavaThread* current))
473 #ifndef PRODUCT
474 SharedRuntime::_multi3_ctr++; // multianewarray for 1 dimension
475 #endif
476 assert(check_compiled_frame(current), "incorrect caller");
477 assert(elem_type->is_klass(), "not a class");
478 jint dims[3];
479 dims[0] = len1;
480 dims[1] = len2;
481 dims[2] = len3;
482 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
483 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(3, dims, THREAD);
484 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
485 current->set_vm_result_oop(obj);
486 JRT_END
487
488 // multianewarray for 4 dimensions
489 JRT_ENTRY(void, OptoRuntime::multianewarray4_C(Klass* elem_type, int len1, int len2, int len3, int len4, JavaThread* current))
490 #ifndef PRODUCT
491 SharedRuntime::_multi4_ctr++; // multianewarray for 1 dimension
492 #endif
493 assert(check_compiled_frame(current), "incorrect caller");
494 assert(elem_type->is_klass(), "not a class");
495 jint dims[4];
496 dims[0] = len1;
497 dims[1] = len2;
498 dims[2] = len3;
499 dims[3] = len4;
500 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
501 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(4, dims, THREAD);
502 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
503 current->set_vm_result_oop(obj);
504 JRT_END
505
506 // multianewarray for 5 dimensions
507 JRT_ENTRY(void, OptoRuntime::multianewarray5_C(Klass* elem_type, int len1, int len2, int len3, int len4, int len5, JavaThread* current))
508 #ifndef PRODUCT
509 SharedRuntime::_multi5_ctr++; // multianewarray for 1 dimension
510 #endif
511 assert(check_compiled_frame(current), "incorrect caller");
512 assert(elem_type->is_klass(), "not a class");
513 jint dims[5];
514 dims[0] = len1;
515 dims[1] = len2;
516 dims[2] = len3;
517 dims[3] = len4;
518 dims[4] = len5;
519 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
520 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(5, dims, THREAD);
521 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
522 current->set_vm_result_oop(obj);
523 JRT_END
524
525 JRT_ENTRY(void, OptoRuntime::multianewarrayN_C(Klass* elem_type, arrayOopDesc* dims, JavaThread* current))
526 assert(check_compiled_frame(current), "incorrect caller");
527 assert(elem_type->is_klass(), "not a class");
528 assert(oop(dims)->is_typeArray(), "not an array");
529
530 ResourceMark rm;
531 jint len = dims->length();
532 assert(len > 0, "Dimensions array should contain data");
533 jint *c_dims = NEW_RESOURCE_ARRAY(jint, len);
534 ArrayAccess<>::arraycopy_to_native<>(dims, typeArrayOopDesc::element_offset<jint>(0),
535 c_dims, len);
536
537 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
538 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(len, c_dims, THREAD);
539 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
540 current->set_vm_result_oop(obj);
541 JRT_END
542
543 JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notify_C(oopDesc* obj, JavaThread* current))
544
545 // Very few notify/notifyAll operations find any threads on the waitset, so
546 // the dominant fast-path is to simply return.
547 // Relatedly, it's critical that notify/notifyAll be fast in order to
548 // reduce lock hold times.
549 if (!SafepointSynchronize::is_synchronizing()) {
550 if (ObjectSynchronizer::quick_notify(obj, current, false)) {
551 return;
552 }
553 }
554
555 // This is the case the fast-path above isn't provisioned to handle.
556 // The fast-path is designed to handle frequently arising cases in an efficient manner.
557 // (The fast-path is just a degenerate variant of the slow-path).
558 // Perform the dreaded state transition and pass control into the slow-path.
559 JRT_BLOCK;
560 Handle h_obj(current, obj);
561 ObjectSynchronizer::notify(h_obj, CHECK);
562 JRT_BLOCK_END;
563 JRT_END
564
565 JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notifyAll_C(oopDesc* obj, JavaThread* current))
566
567 if (!SafepointSynchronize::is_synchronizing() ) {
568 if (ObjectSynchronizer::quick_notify(obj, current, true)) {
569 return;
570 }
571 }
572
573 // This is the case the fast-path above isn't provisioned to handle.
574 // The fast-path is designed to handle frequently arising cases in an efficient manner.
575 // (The fast-path is just a degenerate variant of the slow-path).
576 // Perform the dreaded state transition and pass control into the slow-path.
577 JRT_BLOCK;
578 Handle h_obj(current, obj);
579 ObjectSynchronizer::notifyall(h_obj, CHECK);
580 JRT_BLOCK_END;
581 JRT_END
582
583 static const TypeFunc* make_new_instance_Type() {
584 // create input type (domain)
585 const Type **fields = TypeTuple::fields(1);
1793 assert(reg >= 0 && reg < _last_Mach_Reg, "must be a machine register");
1794 switch (register_save_policy[reg]) {
1795 case 'C': return false; //SOC
1796 case 'E': return true ; //SOE
1797 case 'N': return false; //NS
1798 case 'A': return false; //AS
1799 }
1800 ShouldNotReachHere();
1801 return false;
1802 }
1803
1804 //-----------------------------------------------------------------------
1805 // Exceptions
1806 //
1807
1808 static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg);
1809
1810 // The method is an entry that is always called by a C++ method not
1811 // directly from compiled code. Compiled code will call the C++ method following.
1812 // We can't allow async exception to be installed during exception processing.
1813 JRT_ENTRY_NO_ASYNC(address, OptoRuntime::handle_exception_C_helper(JavaThread* current, nmethod* &nm))
1814 // The frame we rethrow the exception to might not have been processed by the GC yet.
1815 // The stack watermark barrier takes care of detecting that and ensuring the frame
1816 // has updated oops.
1817 StackWatermarkSet::after_unwind(current);
1818
1819 // Do not confuse exception_oop with pending_exception. The exception_oop
1820 // is only used to pass arguments into the method. Not for general
1821 // exception handling. DO NOT CHANGE IT to use pending_exception, since
1822 // the runtime stubs checks this on exit.
1823 assert(current->exception_oop() != nullptr, "exception oop is found");
1824 address handler_address = nullptr;
1825
1826 Handle exception(current, current->exception_oop());
1827 address pc = current->exception_pc();
1828
1829 // Clear out the exception oop and pc since looking up an
1830 // exception handler can cause class loading, which might throw an
1831 // exception and those fields are expected to be clear during
1832 // normal bytecode execution.
1833 current->clear_exception_oop_and_pc();
2066 frame caller_frame = stub_frame.sender(®_map);
2067 return caller_frame.is_deoptimized_frame();
2068 }
2069
2070 static const TypeFunc* make_register_finalizer_Type() {
2071 // create input type (domain)
2072 const Type **fields = TypeTuple::fields(1);
2073 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // oop; Receiver
2074 // // The JavaThread* is passed to each routine as the last argument
2075 // fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // JavaThread *; Executing thread
2076 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1,fields);
2077
2078 // create result type (range)
2079 fields = TypeTuple::fields(0);
2080
2081 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2082
2083 return TypeFunc::make(domain,range);
2084 }
2085
2086 #if INCLUDE_JFR
2087 static const TypeFunc* make_class_id_load_barrier_Type() {
2088 // create input type (domain)
2089 const Type **fields = TypeTuple::fields(1);
2090 fields[TypeFunc::Parms+0] = TypeInstPtr::KLASS;
2091 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms + 1, fields);
2092
2093 // create result type (range)
2094 fields = TypeTuple::fields(0);
2095
2096 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms + 0, fields);
2097
2098 return TypeFunc::make(domain,range);
2099 }
2100 #endif // INCLUDE_JFR
2101
2102 //-----------------------------------------------------------------------------
2103 static const TypeFunc* make_dtrace_method_entry_exit_Type() {
2104 // create input type (domain)
2105 const Type **fields = TypeTuple::fields(2);
2106 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2107 fields[TypeFunc::Parms+1] = TypeMetadataPtr::BOTTOM; // Method*; Method we are entering
2108 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
2109
2110 // create result type (range)
2111 fields = TypeTuple::fields(0);
2112
2113 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2114
2115 return TypeFunc::make(domain,range);
2116 }
2117
2118 static const TypeFunc* make_dtrace_object_alloc_Type() {
2119 // create input type (domain)
2120 const Type **fields = TypeTuple::fields(2);
2121 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2122 fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL; // oop; newly allocated object
2123
2124 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
2125
2126 // create result type (range)
2127 fields = TypeTuple::fields(0);
2128
2129 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2130
2131 return TypeFunc::make(domain,range);
2132 }
2133
2134 JRT_ENTRY_NO_ASYNC(void, OptoRuntime::register_finalizer_C(oopDesc* obj, JavaThread* current))
2135 assert(oopDesc::is_oop(obj), "must be a valid oop");
2136 assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
2137 InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
2138 JRT_END
2139
2140 //-----------------------------------------------------------------------------
2141
2142 NamedCounter * volatile OptoRuntime::_named_counters = nullptr;
2143
2144 //
2145 // dump the collected NamedCounters.
2146 //
2147 void OptoRuntime::print_named_counters() {
2148 int total_lock_count = 0;
2149 int eliminated_lock_count = 0;
2150
2151 NamedCounter* c = _named_counters;
2152 while (c) {
2153 if (c->tag() == NamedCounter::LockCounter || c->tag() == NamedCounter::EliminatedLockCounter) {
2154 int count = c->count();
2155 if (count > 0) {
2156 bool eliminated = c->tag() == NamedCounter::EliminatedLockCounter;
2157 if (Verbose) {
2158 tty->print_cr("%d %s%s", count, c->name(), eliminated ? " (eliminated)" : "");
2159 }
2302 static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg) {
2303 trace_exception_counter++;
2304 stringStream tempst;
2305
2306 tempst.print("%d [Exception (%s): ", trace_exception_counter, msg);
2307 exception_oop->print_value_on(&tempst);
2308 tempst.print(" in ");
2309 CodeBlob* blob = CodeCache::find_blob(exception_pc);
2310 if (blob->is_nmethod()) {
2311 blob->as_nmethod()->method()->print_value_on(&tempst);
2312 } else if (blob->is_runtime_stub()) {
2313 tempst.print("<runtime-stub>");
2314 } else {
2315 tempst.print("<unknown>");
2316 }
2317 tempst.print(" at " INTPTR_FORMAT, p2i(exception_pc));
2318 tempst.print("]");
2319
2320 st->print_raw_cr(tempst.freeze());
2321 }
|
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "classfile/vmClasses.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "code/codeCache.hpp"
28 #include "code/compiledIC.hpp"
29 #include "code/nmethod.hpp"
30 #include "code/pcDesc.hpp"
31 #include "code/scopeDesc.hpp"
32 #include "code/vtableStubs.hpp"
33 #include "compiler/compilationMemoryStatistic.hpp"
34 #include "compiler/compileBroker.hpp"
35 #include "compiler/compilerDefinitions.inline.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/g1/g1HeapRegion.hpp"
38 #include "gc/shared/barrierSet.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gcLocker.hpp"
41 #include "interpreter/bytecode.hpp"
42 #include "interpreter/interpreter.hpp"
43 #include "interpreter/linkResolver.hpp"
44 #include "logging/log.hpp"
45 #include "logging/logStream.hpp"
46 #include "memory/oopFactory.hpp"
47 #include "memory/resourceArea.hpp"
48 #include "oops/objArrayKlass.hpp"
49 #include "oops/klass.inline.hpp"
50 #include "oops/oop.inline.hpp"
51 #include "oops/typeArrayOop.inline.hpp"
52 #include "opto/ad.hpp"
53 #include "opto/addnode.hpp"
54 #include "opto/callnode.hpp"
55 #include "opto/cfgnode.hpp"
56 #include "opto/graphKit.hpp"
57 #include "opto/machnode.hpp"
58 #include "opto/matcher.hpp"
59 #include "opto/memnode.hpp"
60 #include "opto/mulnode.hpp"
61 #include "opto/output.hpp"
62 #include "opto/runtime.hpp"
63 #include "opto/subnode.hpp"
64 #include "prims/jvmtiExport.hpp"
65 #include "runtime/atomic.hpp"
66 #include "runtime/frame.inline.hpp"
67 #include "runtime/handles.inline.hpp"
68 #include "runtime/interfaceSupport.inline.hpp"
69 #include "runtime/java.hpp"
70 #include "runtime/javaCalls.hpp"
71 #include "runtime/perfData.inline.hpp"
72 #include "runtime/sharedRuntime.hpp"
73 #include "runtime/signature.hpp"
74 #include "runtime/stackWatermarkSet.hpp"
75 #include "runtime/synchronizer.hpp"
76 #include "runtime/threadCritical.hpp"
77 #include "runtime/threadWXSetters.inline.hpp"
78 #include "runtime/vframe.hpp"
79 #include "runtime/vframeArray.hpp"
80 #include "runtime/vframe_hp.hpp"
81 #include "services/management.hpp"
82 #include "utilities/copy.hpp"
83 #include "utilities/preserveException.hpp"
84
85
86 // For debugging purposes:
87 // To force FullGCALot inside a runtime function, add the following two lines
88 //
89 // Universe::release_fullgc_alot_dummy();
90 // Universe::heap()->collect();
91 //
92 // At command line specify the parameters: -XX:+FullGCALot -XX:FullGCALotStart=100000000
93
94
95 #define C2_BLOB_FIELD_DEFINE(name, type) \
96 type OptoRuntime:: BLOB_FIELD_NAME(name) = nullptr;
97 #define C2_STUB_FIELD_NAME(name) _ ## name ## _Java
98 #define C2_STUB_FIELD_DEFINE(name, f, t, r) \
99 address OptoRuntime:: C2_STUB_FIELD_NAME(name) = nullptr;
100 #define C2_JVMTI_STUB_FIELD_DEFINE(name) \
101 address OptoRuntime:: STUB_FIELD_NAME(name) = nullptr;
102 C2_STUBS_DO(C2_BLOB_FIELD_DEFINE, C2_STUB_FIELD_DEFINE, C2_JVMTI_STUB_FIELD_DEFINE)
103 #undef C2_BLOB_FIELD_DEFINE
104 #undef C2_STUB_FIELD_DEFINE
105 #undef C2_JVMTI_STUB_FIELD_DEFINE
106
107
108 #define C2_BLOB_NAME_DEFINE(name, type) "C2 Runtime " # name "_blob",
109 #define C2_STUB_NAME_DEFINE(name, f, t, r) "C2 Runtime " # name,
110 #define C2_JVMTI_STUB_NAME_DEFINE(name) "C2 Runtime " # name,
111 const char* OptoRuntime::_stub_names[] = {
112 C2_STUBS_DO(C2_BLOB_NAME_DEFINE, C2_STUB_NAME_DEFINE, C2_JVMTI_STUB_NAME_DEFINE)
113 };
114 #undef C2_BLOB_NAME_DEFINE
115 #undef C2_STUB_NAME_DEFINE
116 #undef C2_JVMTI_STUB_NAME_DEFINE
117
118 address OptoRuntime::_vtable_must_compile_Java = nullptr;
119
120 PerfCounter* _perf_OptoRuntime_class_init_barrier_redundant_count = nullptr;
121
122 // This should be called in an assertion at the start of OptoRuntime routines
123 // which are entered from compiled code (all of them)
124 #ifdef ASSERT
125 static bool check_compiled_frame(JavaThread* thread) {
126 assert(thread->last_frame().is_runtime_frame(), "cannot call runtime directly from compiled code");
127 RegisterMap map(thread,
128 RegisterMap::UpdateMap::skip,
129 RegisterMap::ProcessFrames::include,
130 RegisterMap::WalkContinuation::skip);
131 frame caller = thread->last_frame().sender(&map);
132 assert(caller.is_compiled_frame(), "not being called from compiled like code");
133 return true;
134 }
135 #endif // ASSERT
136
137 /*
138 #define gen(env, var, type_func_gen, c_func, fancy_jump, pass_tls, return_pc) \
139 var = generate_stub(env, type_func_gen, CAST_FROM_FN_PTR(address, c_func), #var, fancy_jump, pass_tls, return_pc); \
140 if (var == nullptr) { return false; }
141 */
168 C2_STUB_NAME(name), \
169 fancy_jump, \
170 pass_tls, \
171 pass_retpc); \
172 if (C2_STUB_FIELD_NAME(name) == nullptr) { return false; } \
173
174 #define C2_JVMTI_STUB_C_FUNC(name) CAST_FROM_FN_PTR(address, SharedRuntime::name)
175
176 #define GEN_C2_JVMTI_STUB(name) \
177 STUB_FIELD_NAME(name) = \
178 generate_stub(env, \
179 notify_jvmti_vthread_Type, \
180 C2_JVMTI_STUB_C_FUNC(name), \
181 C2_STUB_NAME(name), \
182 0, \
183 true, \
184 false); \
185 if (STUB_FIELD_NAME(name) == nullptr) { return false; } \
186
187 bool OptoRuntime::generate(ciEnv* env) {
188 init_counters();
189
190 C2_STUBS_DO(GEN_C2_BLOB, GEN_C2_STUB, GEN_C2_JVMTI_STUB)
191
192 return true;
193 }
194
195 #undef GEN_C2_BLOB
196
197 #undef C2_STUB_FIELD_NAME
198 #undef C2_STUB_TYPEFUNC
199 #undef C2_STUB_C_FUNC
200 #undef C2_STUB_NAME
201 #undef GEN_C2_STUB
202
203 #undef C2_JVMTI_STUB_C_FUNC
204 #undef GEN_C2_JVMTI_STUB
205 // #undef gen
206
207 const TypeFunc* OptoRuntime::_new_instance_Type = nullptr;
208 const TypeFunc* OptoRuntime::_new_array_Type = nullptr;
275 const TypeFunc* OptoRuntime::_updateBytesAdler32_Type = nullptr;
276 const TypeFunc* OptoRuntime::_osr_end_Type = nullptr;
277 const TypeFunc* OptoRuntime::_register_finalizer_Type = nullptr;
278 #if INCLUDE_JFR
279 const TypeFunc* OptoRuntime::_class_id_load_barrier_Type = nullptr;
280 #endif // INCLUDE_JFR
281 #if INCLUDE_JVMTI
282 const TypeFunc* OptoRuntime::_notify_jvmti_vthread_Type = nullptr;
283 #endif // INCLUDE_JVMTI
284 const TypeFunc* OptoRuntime::_dtrace_method_entry_exit_Type = nullptr;
285 const TypeFunc* OptoRuntime::_dtrace_object_alloc_Type = nullptr;
286
287 // Helper method to do generation of RunTimeStub's
288 address OptoRuntime::generate_stub(ciEnv* env,
289 TypeFunc_generator gen, address C_function,
290 const char *name, int is_fancy_jump,
291 bool pass_tls,
292 bool return_pc) {
293
294 // Matching the default directive, we currently have no method to match.
295 DirectiveSet* directive = DirectivesStack::getDefaultDirective(CompilerThread::current()->compiler());
296 CompilationMemoryStatisticMark cmsm(directive);
297 ResourceMark rm;
298 Compile C(env, gen, C_function, name, is_fancy_jump, pass_tls, return_pc, directive);
299 DirectivesStack::release(directive);
300 return C.stub_entry_point();
301 }
302
303 const char* OptoRuntime::stub_name(address entry) {
304 #ifndef PRODUCT
305 CodeBlob* cb = CodeCache::find_blob(entry);
306 RuntimeStub* rs =(RuntimeStub *)cb;
307 assert(rs != nullptr && rs->is_runtime_stub(), "not a runtime stub");
308 return rs->name();
309 #else
310 // Fast implementation for product mode (maybe it should be inlined too)
311 return "runtime stub";
312 #endif
313 }
314
315 // local methods passed as arguments to stub generator that forward
319 oopDesc* dest, jint dest_pos,
320 jint length, JavaThread* thread) {
321 SharedRuntime::slow_arraycopy_C(src, src_pos, dest, dest_pos, length, thread);
322 }
323
324 void OptoRuntime::complete_monitor_locking_C(oopDesc* obj, BasicLock* lock, JavaThread* current) {
325 SharedRuntime::complete_monitor_locking_C(obj, lock, current);
326 }
327
328
329 //=============================================================================
330 // Opto compiler runtime routines
331 //=============================================================================
332
333
334 //=============================allocation======================================
335 // We failed the fast-path allocation. Now we need to do a scavenge or GC
336 // and try allocation again.
337
338 // object allocation
339 JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, new_instance_C, OptoRuntime::new_instance_C(Klass* klass, JavaThread* current))
340 JRT_BLOCK;
341 #ifndef PRODUCT
342 SharedRuntime::_new_instance_ctr++; // new instance requires GC
343 #endif
344 assert(check_compiled_frame(current), "incorrect caller");
345
346 // These checks are cheap to make and support reflective allocation.
347 int lh = klass->layout_helper();
348 if (Klass::layout_helper_needs_slow_path(lh) || !InstanceKlass::cast(klass)->is_initialized()) {
349 Handle holder(current, klass->klass_holder()); // keep the klass alive
350 klass->check_valid_for_instantiation(false, THREAD);
351 if (!HAS_PENDING_EXCEPTION) {
352 InstanceKlass::cast(klass)->initialize(THREAD);
353 }
354 }
355
356 if (!HAS_PENDING_EXCEPTION) {
357 // Scavenge and allocate an instance.
358 Handle holder(current, klass->klass_holder()); // keep the klass alive
359 oop result = InstanceKlass::cast(klass)->allocate_instance(THREAD);
360 current->set_vm_result_oop(result);
361
362 // Pass oops back through thread local storage. Our apparent type to Java
363 // is that we return an oop, but we can block on exit from this routine and
364 // a GC can trash the oop in C's return register. The generated stub will
365 // fetch the oop from TLS after any possible GC.
366 }
367
368 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
369 JRT_BLOCK_END;
370
371 // inform GC that we won't do card marks for initializing writes.
372 SharedRuntime::on_slowpath_allocation_exit(current);
373 JRT_END
374
375
376 // array allocation
377 JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, new_array_C, OptoRuntime::new_array_C(Klass* array_type, int len, JavaThread* current))
378 JRT_BLOCK;
379 #ifndef PRODUCT
380 SharedRuntime::_new_array_ctr++; // new array requires GC
381 #endif
382 assert(check_compiled_frame(current), "incorrect caller");
383
384 // Scavenge and allocate an instance.
385 oop result;
386
387 if (array_type->is_typeArray_klass()) {
388 // The oopFactory likes to work with the element type.
389 // (We could bypass the oopFactory, since it doesn't add much value.)
390 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
391 result = oopFactory::new_typeArray(elem_type, len, THREAD);
392 } else {
393 // Although the oopFactory likes to work with the elem_type,
394 // the compiler prefers the array_type, since it must already have
395 // that latter value in hand for the fast path.
396 Handle holder(current, array_type->klass_holder()); // keep the array klass alive
397 Klass* elem_type = ObjArrayKlass::cast(array_type)->element_klass();
398 result = oopFactory::new_objArray(elem_type, len, THREAD);
399 }
400
401 // Pass oops back through thread local storage. Our apparent type to Java
402 // is that we return an oop, but we can block on exit from this routine and
403 // a GC can trash the oop in C's return register. The generated stub will
404 // fetch the oop from TLS after any possible GC.
405 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
406 current->set_vm_result_oop(result);
407 JRT_BLOCK_END;
408
409 // inform GC that we won't do card marks for initializing writes.
410 SharedRuntime::on_slowpath_allocation_exit(current);
411 JRT_END
412
413 // array allocation without zeroing
414 JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, new_array_nozero_C, OptoRuntime::new_array_nozero_C(Klass* array_type, int len, JavaThread* current))
415 JRT_BLOCK;
416 #ifndef PRODUCT
417 SharedRuntime::_new_array_ctr++; // new array requires GC
418 #endif
419 assert(check_compiled_frame(current), "incorrect caller");
420
421 // Scavenge and allocate an instance.
422 oop result;
423
424 assert(array_type->is_typeArray_klass(), "should be called only for type array");
425 // The oopFactory likes to work with the element type.
426 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
427 result = oopFactory::new_typeArray_nozero(elem_type, len, THREAD);
428
429 // Pass oops back through thread local storage. Our apparent type to Java
430 // is that we return an oop, but we can block on exit from this routine and
431 // a GC can trash the oop in C's return register. The generated stub will
432 // fetch the oop from TLS after any possible GC.
433 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
434 current->set_vm_result_oop(result);
446 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
447 size_t hs_bytes = arrayOopDesc::base_offset_in_bytes(elem_type);
448 assert(is_aligned(hs_bytes, BytesPerInt), "must be 4 byte aligned");
449 HeapWord* obj = cast_from_oop<HeapWord*>(result);
450 if (!is_aligned(hs_bytes, BytesPerLong)) {
451 *reinterpret_cast<jint*>(reinterpret_cast<char*>(obj) + hs_bytes) = 0;
452 hs_bytes += BytesPerInt;
453 }
454
455 // Optimized zeroing.
456 assert(is_aligned(hs_bytes, BytesPerLong), "must be 8-byte aligned");
457 const size_t aligned_hs = hs_bytes / BytesPerLong;
458 Copy::fill_to_aligned_words(obj+aligned_hs, size-aligned_hs);
459 }
460
461 JRT_END
462
463 // Note: multianewarray for one dimension is handled inline by GraphKit::new_array.
464
465 // multianewarray for 2 dimensions
466 JRT_ENTRY_PROF(void, OptoRuntime, multianewarray2_C, OptoRuntime::multianewarray2_C(Klass* elem_type, int len1, int len2, JavaThread* current))
467 #ifndef PRODUCT
468 SharedRuntime::_multi2_ctr++; // multianewarray for 1 dimension
469 #endif
470 assert(check_compiled_frame(current), "incorrect caller");
471 assert(elem_type->is_klass(), "not a class");
472 jint dims[2];
473 dims[0] = len1;
474 dims[1] = len2;
475 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
476 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(2, dims, THREAD);
477 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
478 current->set_vm_result_oop(obj);
479 JRT_END
480
481 // multianewarray for 3 dimensions
482 JRT_ENTRY_PROF(void, OptoRuntime, multianewarray3_C, OptoRuntime::multianewarray3_C(Klass* elem_type, int len1, int len2, int len3, JavaThread* current))
483 #ifndef PRODUCT
484 SharedRuntime::_multi3_ctr++; // multianewarray for 1 dimension
485 #endif
486 assert(check_compiled_frame(current), "incorrect caller");
487 assert(elem_type->is_klass(), "not a class");
488 jint dims[3];
489 dims[0] = len1;
490 dims[1] = len2;
491 dims[2] = len3;
492 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
493 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(3, dims, THREAD);
494 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
495 current->set_vm_result_oop(obj);
496 JRT_END
497
498 // multianewarray for 4 dimensions
499 JRT_ENTRY_PROF(void, OptoRuntime, multianewarray4_C, OptoRuntime::multianewarray4_C(Klass* elem_type, int len1, int len2, int len3, int len4, JavaThread* current))
500 #ifndef PRODUCT
501 SharedRuntime::_multi4_ctr++; // multianewarray for 1 dimension
502 #endif
503 assert(check_compiled_frame(current), "incorrect caller");
504 assert(elem_type->is_klass(), "not a class");
505 jint dims[4];
506 dims[0] = len1;
507 dims[1] = len2;
508 dims[2] = len3;
509 dims[3] = len4;
510 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
511 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(4, dims, THREAD);
512 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
513 current->set_vm_result_oop(obj);
514 JRT_END
515
516 // multianewarray for 5 dimensions
517 JRT_ENTRY(void, OptoRuntime::multianewarray5_C(Klass* elem_type, int len1, int len2, int len3, int len4, int len5, JavaThread* current))
518 #ifndef PRODUCT
519 SharedRuntime::_multi5_ctr++; // multianewarray for 1 dimension
520 #endif
521 assert(check_compiled_frame(current), "incorrect caller");
522 assert(elem_type->is_klass(), "not a class");
523 jint dims[5];
524 dims[0] = len1;
525 dims[1] = len2;
526 dims[2] = len3;
527 dims[3] = len4;
528 dims[4] = len5;
529 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
530 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(5, dims, THREAD);
531 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
532 current->set_vm_result_oop(obj);
533 JRT_END
534
535 JRT_ENTRY_PROF(void, OptoRuntime, multianewarrayN_C, OptoRuntime::multianewarrayN_C(Klass* elem_type, arrayOopDesc* dims, JavaThread* current))
536 assert(check_compiled_frame(current), "incorrect caller");
537 assert(elem_type->is_klass(), "not a class");
538 assert(oop(dims)->is_typeArray(), "not an array");
539
540 ResourceMark rm;
541 jint len = dims->length();
542 assert(len > 0, "Dimensions array should contain data");
543 jint *c_dims = NEW_RESOURCE_ARRAY(jint, len);
544 ArrayAccess<>::arraycopy_to_native<>(dims, typeArrayOopDesc::element_offset<jint>(0),
545 c_dims, len);
546
547 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
548 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(len, c_dims, THREAD);
549 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
550 current->set_vm_result_oop(obj);
551 JRT_END
552
553 JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, monitor_notify_C, OptoRuntime::monitor_notify_C(oopDesc* obj, JavaThread* current))
554
555 // Very few notify/notifyAll operations find any threads on the waitset, so
556 // the dominant fast-path is to simply return.
557 // Relatedly, it's critical that notify/notifyAll be fast in order to
558 // reduce lock hold times.
559 if (!SafepointSynchronize::is_synchronizing()) {
560 if (ObjectSynchronizer::quick_notify(obj, current, false)) {
561 return;
562 }
563 }
564
565 // This is the case the fast-path above isn't provisioned to handle.
566 // The fast-path is designed to handle frequently arising cases in an efficient manner.
567 // (The fast-path is just a degenerate variant of the slow-path).
568 // Perform the dreaded state transition and pass control into the slow-path.
569 JRT_BLOCK;
570 Handle h_obj(current, obj);
571 ObjectSynchronizer::notify(h_obj, CHECK);
572 JRT_BLOCK_END;
573 JRT_END
574
575 JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, monitor_notifyAll_C, OptoRuntime::monitor_notifyAll_C(oopDesc* obj, JavaThread* current))
576
577 if (!SafepointSynchronize::is_synchronizing() ) {
578 if (ObjectSynchronizer::quick_notify(obj, current, true)) {
579 return;
580 }
581 }
582
583 // This is the case the fast-path above isn't provisioned to handle.
584 // The fast-path is designed to handle frequently arising cases in an efficient manner.
585 // (The fast-path is just a degenerate variant of the slow-path).
586 // Perform the dreaded state transition and pass control into the slow-path.
587 JRT_BLOCK;
588 Handle h_obj(current, obj);
589 ObjectSynchronizer::notifyall(h_obj, CHECK);
590 JRT_BLOCK_END;
591 JRT_END
592
593 static const TypeFunc* make_new_instance_Type() {
594 // create input type (domain)
595 const Type **fields = TypeTuple::fields(1);
1803 assert(reg >= 0 && reg < _last_Mach_Reg, "must be a machine register");
1804 switch (register_save_policy[reg]) {
1805 case 'C': return false; //SOC
1806 case 'E': return true ; //SOE
1807 case 'N': return false; //NS
1808 case 'A': return false; //AS
1809 }
1810 ShouldNotReachHere();
1811 return false;
1812 }
1813
1814 //-----------------------------------------------------------------------
1815 // Exceptions
1816 //
1817
1818 static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg);
1819
1820 // The method is an entry that is always called by a C++ method not
1821 // directly from compiled code. Compiled code will call the C++ method following.
1822 // We can't allow async exception to be installed during exception processing.
1823 JRT_ENTRY_NO_ASYNC_PROF(address, OptoRuntime, handle_exception_C_helper, OptoRuntime::handle_exception_C_helper(JavaThread* current, nmethod* &nm))
1824 // The frame we rethrow the exception to might not have been processed by the GC yet.
1825 // The stack watermark barrier takes care of detecting that and ensuring the frame
1826 // has updated oops.
1827 StackWatermarkSet::after_unwind(current);
1828
1829 // Do not confuse exception_oop with pending_exception. The exception_oop
1830 // is only used to pass arguments into the method. Not for general
1831 // exception handling. DO NOT CHANGE IT to use pending_exception, since
1832 // the runtime stubs checks this on exit.
1833 assert(current->exception_oop() != nullptr, "exception oop is found");
1834 address handler_address = nullptr;
1835
1836 Handle exception(current, current->exception_oop());
1837 address pc = current->exception_pc();
1838
1839 // Clear out the exception oop and pc since looking up an
1840 // exception handler can cause class loading, which might throw an
1841 // exception and those fields are expected to be clear during
1842 // normal bytecode execution.
1843 current->clear_exception_oop_and_pc();
2076 frame caller_frame = stub_frame.sender(®_map);
2077 return caller_frame.is_deoptimized_frame();
2078 }
2079
2080 static const TypeFunc* make_register_finalizer_Type() {
2081 // create input type (domain)
2082 const Type **fields = TypeTuple::fields(1);
2083 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // oop; Receiver
2084 // // The JavaThread* is passed to each routine as the last argument
2085 // fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // JavaThread *; Executing thread
2086 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1,fields);
2087
2088 // create result type (range)
2089 fields = TypeTuple::fields(0);
2090
2091 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2092
2093 return TypeFunc::make(domain,range);
2094 }
2095
2096 const TypeFunc *OptoRuntime::class_init_barrier_Type() {
2097 // create input type (domain)
2098 const Type** fields = TypeTuple::fields(1);
2099 fields[TypeFunc::Parms+0] = TypeKlassPtr::NOTNULL;
2100 // // The JavaThread* is passed to each routine as the last argument
2101 // fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // JavaThread *; Executing thread
2102 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+1, fields);
2103
2104 // create result type (range)
2105 fields = TypeTuple::fields(0);
2106 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
2107 return TypeFunc::make(domain,range);
2108 }
2109
2110 #if INCLUDE_JFR
2111 static const TypeFunc* make_class_id_load_barrier_Type() {
2112 // create input type (domain)
2113 const Type **fields = TypeTuple::fields(1);
2114 fields[TypeFunc::Parms+0] = TypeInstPtr::KLASS;
2115 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms + 1, fields);
2116
2117 // create result type (range)
2118 fields = TypeTuple::fields(0);
2119
2120 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms + 0, fields);
2121
2122 return TypeFunc::make(domain,range);
2123 }
2124 #endif // INCLUDE_JFR
2125
2126 //-----------------------------------------------------------------------------
2127 // runtime upcall support
2128 const TypeFunc *OptoRuntime::runtime_up_call_Type() {
2129 // create input type (domain)
2130 const Type **fields = TypeTuple::fields(1);
2131 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2132 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1,fields);
2133
2134 // create result type (range)
2135 fields = TypeTuple::fields(0);
2136
2137 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2138
2139 return TypeFunc::make(domain,range);
2140 }
2141
2142 //-----------------------------------------------------------------------------
2143 static const TypeFunc* make_dtrace_method_entry_exit_Type() {
2144 // create input type (domain)
2145 const Type **fields = TypeTuple::fields(2);
2146 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2147 fields[TypeFunc::Parms+1] = TypeMetadataPtr::BOTTOM; // Method*; Method we are entering
2148 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
2149
2150 // create result type (range)
2151 fields = TypeTuple::fields(0);
2152
2153 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2154
2155 return TypeFunc::make(domain,range);
2156 }
2157
2158 static const TypeFunc* make_dtrace_object_alloc_Type() {
2159 // create input type (domain)
2160 const Type **fields = TypeTuple::fields(2);
2161 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2162 fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL; // oop; newly allocated object
2163
2164 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
2165
2166 // create result type (range)
2167 fields = TypeTuple::fields(0);
2168
2169 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2170
2171 return TypeFunc::make(domain,range);
2172 }
2173
2174 JRT_ENTRY_NO_ASYNC_PROF(void, OptoRuntime, register_finalizer_C, OptoRuntime::register_finalizer_C(oopDesc* obj, JavaThread* current))
2175 assert(oopDesc::is_oop(obj), "must be a valid oop");
2176 assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
2177 InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
2178 JRT_END
2179
2180 JRT_ENTRY_NO_ASYNC_PROF(void, OptoRuntime, class_init_barrier_C, OptoRuntime::class_init_barrier_C(Klass* k, JavaThread* current))
2181 InstanceKlass* ik = InstanceKlass::cast(k);
2182 if (ik->should_be_initialized()) {
2183 ik->initialize(CHECK);
2184 } else if (UsePerfData) {
2185 _perf_OptoRuntime_class_init_barrier_redundant_count->inc();
2186 }
2187 JRT_END
2188
2189 //-----------------------------------------------------------------------------
2190
2191 NamedCounter * volatile OptoRuntime::_named_counters = nullptr;
2192
2193 //
2194 // dump the collected NamedCounters.
2195 //
2196 void OptoRuntime::print_named_counters() {
2197 int total_lock_count = 0;
2198 int eliminated_lock_count = 0;
2199
2200 NamedCounter* c = _named_counters;
2201 while (c) {
2202 if (c->tag() == NamedCounter::LockCounter || c->tag() == NamedCounter::EliminatedLockCounter) {
2203 int count = c->count();
2204 if (count > 0) {
2205 bool eliminated = c->tag() == NamedCounter::EliminatedLockCounter;
2206 if (Verbose) {
2207 tty->print_cr("%d %s%s", count, c->name(), eliminated ? " (eliminated)" : "");
2208 }
2351 static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg) {
2352 trace_exception_counter++;
2353 stringStream tempst;
2354
2355 tempst.print("%d [Exception (%s): ", trace_exception_counter, msg);
2356 exception_oop->print_value_on(&tempst);
2357 tempst.print(" in ");
2358 CodeBlob* blob = CodeCache::find_blob(exception_pc);
2359 if (blob->is_nmethod()) {
2360 blob->as_nmethod()->method()->print_value_on(&tempst);
2361 } else if (blob->is_runtime_stub()) {
2362 tempst.print("<runtime-stub>");
2363 } else {
2364 tempst.print("<unknown>");
2365 }
2366 tempst.print(" at " INTPTR_FORMAT, p2i(exception_pc));
2367 tempst.print("]");
2368
2369 st->print_raw_cr(tempst.freeze());
2370 }
2371
2372 #define DO_COUNTERS2(macro2, macro1) \
2373 macro2(OptoRuntime, new_instance_C) \
2374 macro2(OptoRuntime, new_array_C) \
2375 macro2(OptoRuntime, new_array_nozero_C) \
2376 macro2(OptoRuntime, multianewarray2_C) \
2377 macro2(OptoRuntime, multianewarray3_C) \
2378 macro2(OptoRuntime, multianewarray4_C) \
2379 macro2(OptoRuntime, multianewarrayN_C) \
2380 macro2(OptoRuntime, monitor_notify_C) \
2381 macro2(OptoRuntime, monitor_notifyAll_C) \
2382 macro2(OptoRuntime, handle_exception_C_helper) \
2383 macro2(OptoRuntime, register_finalizer_C) \
2384 macro2(OptoRuntime, class_init_barrier_C) \
2385 macro1(OptoRuntime, class_init_barrier_redundant)
2386
2387 #define INIT_COUNTER_TIME_AND_CNT(sub, name) \
2388 NEWPERFTICKCOUNTERS(_perf_##sub##_##name##_timer, SUN_CI, #sub "::" #name); \
2389 NEWPERFEVENTCOUNTER(_perf_##sub##_##name##_count, SUN_CI, #sub "::" #name "_count");
2390
2391 #define INIT_COUNTER_CNT(sub, name) \
2392 NEWPERFEVENTCOUNTER(_perf_##sub##_##name##_count, SUN_CI, #sub "::" #name "_count");
2393
2394 void OptoRuntime::init_counters() {
2395 assert(CompilerConfig::is_c2_enabled(), "");
2396
2397 if (UsePerfData) {
2398 EXCEPTION_MARK;
2399
2400 DO_COUNTERS2(INIT_COUNTER_TIME_AND_CNT, INIT_COUNTER_CNT)
2401
2402 if (HAS_PENDING_EXCEPTION) {
2403 vm_exit_during_initialization("jvm_perf_init failed unexpectedly");
2404 }
2405 }
2406 }
2407 #undef INIT_COUNTER_TIME_AND_CNT
2408 #undef INIT_COUNTER_CNT
2409
2410 #define PRINT_COUNTER_TIME_AND_CNT(sub, name) { \
2411 jlong count = _perf_##sub##_##name##_count->get_value(); \
2412 if (count > 0) { \
2413 st->print_cr(" %-50s = " JLONG_FORMAT_W(6) "us (elapsed) " JLONG_FORMAT_W(6) "us (thread) (" JLONG_FORMAT_W(5) " events)", #sub "::" #name, \
2414 _perf_##sub##_##name##_timer->elapsed_counter_value_us(), \
2415 _perf_##sub##_##name##_timer->thread_counter_value_us(), \
2416 count); \
2417 }}
2418
2419 #define PRINT_COUNTER_CNT(sub, name) { \
2420 jlong count = _perf_##sub##_##name##_count->get_value(); \
2421 if (count > 0) { \
2422 st->print_cr(" %-30s = " JLONG_FORMAT_W(5) " events", #name, count); \
2423 }}
2424
2425 void OptoRuntime::print_counters_on(outputStream* st) {
2426 if (UsePerfData && ProfileRuntimeCalls && CompilerConfig::is_c2_enabled()) {
2427 DO_COUNTERS2(PRINT_COUNTER_TIME_AND_CNT, PRINT_COUNTER_CNT)
2428 } else {
2429 st->print_cr(" OptoRuntime: no info (%s is disabled)",
2430 (!CompilerConfig::is_c2_enabled() ? "C2" : (UsePerfData ? "ProfileRuntimeCalls" : "UsePerfData")));
2431 }
2432 }
2433
2434 #undef PRINT_COUNTER_TIME_AND_CNT
2435 #undef PRINT_COUNTER_CNT
2436 #undef DO_COUNTERS2
|