15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "classfile/vmClasses.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "code/codeCache.hpp"
28 #include "code/compiledIC.hpp"
29 #include "code/nmethod.hpp"
30 #include "code/pcDesc.hpp"
31 #include "code/scopeDesc.hpp"
32 #include "code/vtableStubs.hpp"
33 #include "compiler/compilationMemoryStatistic.hpp"
34 #include "compiler/compileBroker.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/g1/g1HeapRegion.hpp"
37 #include "gc/shared/barrierSet.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gcLocker.hpp"
40 #include "interpreter/bytecode.hpp"
41 #include "interpreter/interpreter.hpp"
42 #include "interpreter/linkResolver.hpp"
43 #include "logging/log.hpp"
44 #include "logging/logStream.hpp"
45 #include "memory/oopFactory.hpp"
46 #include "memory/resourceArea.hpp"
47 #include "oops/objArrayKlass.hpp"
48 #include "oops/klass.inline.hpp"
49 #include "oops/oop.inline.hpp"
50 #include "oops/typeArrayOop.inline.hpp"
51 #include "opto/ad.hpp"
52 #include "opto/addnode.hpp"
53 #include "opto/callnode.hpp"
54 #include "opto/cfgnode.hpp"
55 #include "opto/graphKit.hpp"
56 #include "opto/machnode.hpp"
57 #include "opto/matcher.hpp"
58 #include "opto/memnode.hpp"
59 #include "opto/mulnode.hpp"
60 #include "opto/output.hpp"
61 #include "opto/runtime.hpp"
62 #include "opto/subnode.hpp"
63 #include "prims/jvmtiExport.hpp"
64 #include "runtime/atomic.hpp"
65 #include "runtime/frame.inline.hpp"
66 #include "runtime/handles.inline.hpp"
67 #include "runtime/interfaceSupport.inline.hpp"
68 #include "runtime/javaCalls.hpp"
69 #include "runtime/sharedRuntime.hpp"
70 #include "runtime/signature.hpp"
71 #include "runtime/stackWatermarkSet.hpp"
72 #include "runtime/synchronizer.hpp"
73 #include "runtime/threadWXSetters.inline.hpp"
74 #include "runtime/vframe.hpp"
75 #include "runtime/vframeArray.hpp"
76 #include "runtime/vframe_hp.hpp"
77 #include "utilities/copy.hpp"
78 #include "utilities/preserveException.hpp"
79
80
81 // For debugging purposes:
82 // To force FullGCALot inside a runtime function, add the following two lines
83 //
84 // Universe::release_fullgc_alot_dummy();
85 // Universe::heap()->collect();
86 //
87 // At command line specify the parameters: -XX:+FullGCALot -XX:FullGCALotStart=100000000
88
89
90 #define C2_BLOB_FIELD_DEFINE(name, type) \
91 type OptoRuntime:: BLOB_FIELD_NAME(name) = nullptr;
92 #define C2_STUB_FIELD_NAME(name) _ ## name ## _Java
93 #define C2_STUB_FIELD_DEFINE(name, f, t, r) \
94 address OptoRuntime:: C2_STUB_FIELD_NAME(name) = nullptr;
95 #define C2_JVMTI_STUB_FIELD_DEFINE(name) \
96 address OptoRuntime:: STUB_FIELD_NAME(name) = nullptr;
97 C2_STUBS_DO(C2_BLOB_FIELD_DEFINE, C2_STUB_FIELD_DEFINE, C2_JVMTI_STUB_FIELD_DEFINE)
98 #undef C2_BLOB_FIELD_DEFINE
99 #undef C2_STUB_FIELD_DEFINE
100 #undef C2_JVMTI_STUB_FIELD_DEFINE
101
102 #define C2_BLOB_NAME_DEFINE(name, type) "C2 Runtime " # name "_blob",
103 #define C2_STUB_NAME_DEFINE(name, f, t, r) "C2 Runtime " # name,
104 #define C2_JVMTI_STUB_NAME_DEFINE(name) "C2 Runtime " # name,
105 const char* OptoRuntime::_stub_names[] = {
106 C2_STUBS_DO(C2_BLOB_NAME_DEFINE, C2_STUB_NAME_DEFINE, C2_JVMTI_STUB_NAME_DEFINE)
107 };
108 #undef C2_BLOB_NAME_DEFINE
109 #undef C2_STUB_NAME_DEFINE
110 #undef C2_JVMTI_STUB_NAME_DEFINE
111
112 // This should be called in an assertion at the start of OptoRuntime routines
113 // which are entered from compiled code (all of them)
114 #ifdef ASSERT
115 static bool check_compiled_frame(JavaThread* thread) {
116 assert(thread->last_frame().is_runtime_frame(), "cannot call runtime directly from compiled code");
117 RegisterMap map(thread,
118 RegisterMap::UpdateMap::skip,
119 RegisterMap::ProcessFrames::include,
120 RegisterMap::WalkContinuation::skip);
121 frame caller = thread->last_frame().sender(&map);
122 assert(caller.is_compiled_frame(), "not being called from compiled like code");
123 return true;
124 }
125 #endif // ASSERT
126
127 /*
128 #define gen(env, var, type_func_gen, c_func, fancy_jump, pass_tls, return_pc) \
129 var = generate_stub(env, type_func_gen, CAST_FROM_FN_PTR(address, c_func), #var, fancy_jump, pass_tls, return_pc); \
130 if (var == nullptr) { return false; }
131 */
161 fancy_jump, \
162 pass_tls, \
163 pass_retpc); \
164 if (C2_STUB_FIELD_NAME(name) == nullptr) { return false; } \
165
166 #define C2_JVMTI_STUB_C_FUNC(name) CAST_FROM_FN_PTR(address, SharedRuntime::name)
167
168 #define GEN_C2_JVMTI_STUB(name) \
169 STUB_FIELD_NAME(name) = \
170 generate_stub(env, \
171 notify_jvmti_vthread_Type, \
172 C2_JVMTI_STUB_C_FUNC(name), \
173 C2_STUB_NAME(name), \
174 (int)C2_STUB_ID(name), \
175 0, \
176 true, \
177 false); \
178 if (STUB_FIELD_NAME(name) == nullptr) { return false; } \
179
180 bool OptoRuntime::generate(ciEnv* env) {
181
182 C2_STUBS_DO(GEN_C2_BLOB, GEN_C2_STUB, GEN_C2_JVMTI_STUB)
183
184 return true;
185 }
186
187 #undef GEN_C2_BLOB
188
189 #undef C2_STUB_FIELD_NAME
190 #undef C2_STUB_TYPEFUNC
191 #undef C2_STUB_C_FUNC
192 #undef C2_STUB_NAME
193 #undef GEN_C2_STUB
194
195 #undef C2_JVMTI_STUB_C_FUNC
196 #undef GEN_C2_JVMTI_STUB
197 // #undef gen
198
199 const TypeFunc* OptoRuntime::_new_instance_Type = nullptr;
200 const TypeFunc* OptoRuntime::_new_array_Type = nullptr;
267 const TypeFunc* OptoRuntime::_updateBytesAdler32_Type = nullptr;
268 const TypeFunc* OptoRuntime::_osr_end_Type = nullptr;
269 const TypeFunc* OptoRuntime::_register_finalizer_Type = nullptr;
270 #if INCLUDE_JFR
271 const TypeFunc* OptoRuntime::_class_id_load_barrier_Type = nullptr;
272 #endif // INCLUDE_JFR
273 #if INCLUDE_JVMTI
274 const TypeFunc* OptoRuntime::_notify_jvmti_vthread_Type = nullptr;
275 #endif // INCLUDE_JVMTI
276 const TypeFunc* OptoRuntime::_dtrace_method_entry_exit_Type = nullptr;
277 const TypeFunc* OptoRuntime::_dtrace_object_alloc_Type = nullptr;
278
279 // Helper method to do generation of RunTimeStub's
280 address OptoRuntime::generate_stub(ciEnv* env,
281 TypeFunc_generator gen, address C_function,
282 const char *name, int stub_id,
283 int is_fancy_jump, bool pass_tls,
284 bool return_pc) {
285
286 // Matching the default directive, we currently have no method to match.
287 DirectiveSet* directive = DirectivesStack::getDefaultDirective(CompileBroker::compiler(CompLevel_full_optimization));
288 CompilationMemoryStatisticMark cmsm(directive);
289 ResourceMark rm;
290 Compile C(env, gen, C_function, name, stub_id, is_fancy_jump, pass_tls, return_pc, directive);
291 DirectivesStack::release(directive);
292 return C.stub_entry_point();
293 }
294
295 const char* OptoRuntime::stub_name(address entry) {
296 #ifndef PRODUCT
297 CodeBlob* cb = CodeCache::find_blob(entry);
298 RuntimeStub* rs =(RuntimeStub *)cb;
299 assert(rs != nullptr && rs->is_runtime_stub(), "not a runtime stub");
300 return rs->name();
301 #else
302 // Fast implementation for product mode (maybe it should be inlined too)
303 return "runtime stub";
304 #endif
305 }
306
307 // local methods passed as arguments to stub generator that forward
311 oopDesc* dest, jint dest_pos,
312 jint length, JavaThread* thread) {
313 SharedRuntime::slow_arraycopy_C(src, src_pos, dest, dest_pos, length, thread);
314 }
315
316 void OptoRuntime::complete_monitor_locking_C(oopDesc* obj, BasicLock* lock, JavaThread* current) {
317 SharedRuntime::complete_monitor_locking_C(obj, lock, current);
318 }
319
320
321 //=============================================================================
322 // Opto compiler runtime routines
323 //=============================================================================
324
325
326 //=============================allocation======================================
327 // We failed the fast-path allocation. Now we need to do a scavenge or GC
328 // and try allocation again.
329
330 // object allocation
331 JRT_BLOCK_ENTRY(void, OptoRuntime::new_instance_C(Klass* klass, JavaThread* current))
332 JRT_BLOCK;
333 #ifndef PRODUCT
334 SharedRuntime::_new_instance_ctr++; // new instance requires GC
335 #endif
336 assert(check_compiled_frame(current), "incorrect caller");
337
338 // These checks are cheap to make and support reflective allocation.
339 int lh = klass->layout_helper();
340 if (Klass::layout_helper_needs_slow_path(lh) || !InstanceKlass::cast(klass)->is_initialized()) {
341 Handle holder(current, klass->klass_holder()); // keep the klass alive
342 klass->check_valid_for_instantiation(false, THREAD);
343 if (!HAS_PENDING_EXCEPTION) {
344 InstanceKlass::cast(klass)->initialize(THREAD);
345 }
346 }
347
348 if (!HAS_PENDING_EXCEPTION) {
349 // Scavenge and allocate an instance.
350 Handle holder(current, klass->klass_holder()); // keep the klass alive
351 oop result = InstanceKlass::cast(klass)->allocate_instance(THREAD);
352 current->set_vm_result_oop(result);
353
354 // Pass oops back through thread local storage. Our apparent type to Java
355 // is that we return an oop, but we can block on exit from this routine and
356 // a GC can trash the oop in C's return register. The generated stub will
357 // fetch the oop from TLS after any possible GC.
358 }
359
360 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
361 JRT_BLOCK_END;
362
363 // inform GC that we won't do card marks for initializing writes.
364 SharedRuntime::on_slowpath_allocation_exit(current);
365 JRT_END
366
367
368 // array allocation
369 JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_C(Klass* array_type, int len, JavaThread* current))
370 JRT_BLOCK;
371 #ifndef PRODUCT
372 SharedRuntime::_new_array_ctr++; // new array requires GC
373 #endif
374 assert(check_compiled_frame(current), "incorrect caller");
375
376 // Scavenge and allocate an instance.
377 oop result;
378
379 if (array_type->is_typeArray_klass()) {
380 // The oopFactory likes to work with the element type.
381 // (We could bypass the oopFactory, since it doesn't add much value.)
382 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
383 result = oopFactory::new_typeArray(elem_type, len, THREAD);
384 } else {
385 // Although the oopFactory likes to work with the elem_type,
386 // the compiler prefers the array_type, since it must already have
387 // that latter value in hand for the fast path.
388 Handle holder(current, array_type->klass_holder()); // keep the array klass alive
389 Klass* elem_type = ObjArrayKlass::cast(array_type)->element_klass();
390 result = oopFactory::new_objArray(elem_type, len, THREAD);
391 }
392
393 // Pass oops back through thread local storage. Our apparent type to Java
394 // is that we return an oop, but we can block on exit from this routine and
395 // a GC can trash the oop in C's return register. The generated stub will
396 // fetch the oop from TLS after any possible GC.
397 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
398 current->set_vm_result_oop(result);
399 JRT_BLOCK_END;
400
401 // inform GC that we won't do card marks for initializing writes.
402 SharedRuntime::on_slowpath_allocation_exit(current);
403 JRT_END
404
405 // array allocation without zeroing
406 JRT_BLOCK_ENTRY(void, OptoRuntime::new_array_nozero_C(Klass* array_type, int len, JavaThread* current))
407 JRT_BLOCK;
408 #ifndef PRODUCT
409 SharedRuntime::_new_array_ctr++; // new array requires GC
410 #endif
411 assert(check_compiled_frame(current), "incorrect caller");
412
413 // Scavenge and allocate an instance.
414 oop result;
415
416 assert(array_type->is_typeArray_klass(), "should be called only for type array");
417 // The oopFactory likes to work with the element type.
418 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
419 result = oopFactory::new_typeArray_nozero(elem_type, len, THREAD);
420
421 // Pass oops back through thread local storage. Our apparent type to Java
422 // is that we return an oop, but we can block on exit from this routine and
423 // a GC can trash the oop in C's return register. The generated stub will
424 // fetch the oop from TLS after any possible GC.
425 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
426 current->set_vm_result_oop(result);
438 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
439 size_t hs_bytes = arrayOopDesc::base_offset_in_bytes(elem_type);
440 assert(is_aligned(hs_bytes, BytesPerInt), "must be 4 byte aligned");
441 HeapWord* obj = cast_from_oop<HeapWord*>(result);
442 if (!is_aligned(hs_bytes, BytesPerLong)) {
443 *reinterpret_cast<jint*>(reinterpret_cast<char*>(obj) + hs_bytes) = 0;
444 hs_bytes += BytesPerInt;
445 }
446
447 // Optimized zeroing.
448 assert(is_aligned(hs_bytes, BytesPerLong), "must be 8-byte aligned");
449 const size_t aligned_hs = hs_bytes / BytesPerLong;
450 Copy::fill_to_aligned_words(obj+aligned_hs, size-aligned_hs);
451 }
452
453 JRT_END
454
455 // Note: multianewarray for one dimension is handled inline by GraphKit::new_array.
456
457 // multianewarray for 2 dimensions
458 JRT_ENTRY(void, OptoRuntime::multianewarray2_C(Klass* elem_type, int len1, int len2, JavaThread* current))
459 #ifndef PRODUCT
460 SharedRuntime::_multi2_ctr++; // multianewarray for 1 dimension
461 #endif
462 assert(check_compiled_frame(current), "incorrect caller");
463 assert(elem_type->is_klass(), "not a class");
464 jint dims[2];
465 dims[0] = len1;
466 dims[1] = len2;
467 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
468 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(2, dims, THREAD);
469 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
470 current->set_vm_result_oop(obj);
471 JRT_END
472
473 // multianewarray for 3 dimensions
474 JRT_ENTRY(void, OptoRuntime::multianewarray3_C(Klass* elem_type, int len1, int len2, int len3, JavaThread* current))
475 #ifndef PRODUCT
476 SharedRuntime::_multi3_ctr++; // multianewarray for 1 dimension
477 #endif
478 assert(check_compiled_frame(current), "incorrect caller");
479 assert(elem_type->is_klass(), "not a class");
480 jint dims[3];
481 dims[0] = len1;
482 dims[1] = len2;
483 dims[2] = len3;
484 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
485 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(3, dims, THREAD);
486 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
487 current->set_vm_result_oop(obj);
488 JRT_END
489
490 // multianewarray for 4 dimensions
491 JRT_ENTRY(void, OptoRuntime::multianewarray4_C(Klass* elem_type, int len1, int len2, int len3, int len4, JavaThread* current))
492 #ifndef PRODUCT
493 SharedRuntime::_multi4_ctr++; // multianewarray for 1 dimension
494 #endif
495 assert(check_compiled_frame(current), "incorrect caller");
496 assert(elem_type->is_klass(), "not a class");
497 jint dims[4];
498 dims[0] = len1;
499 dims[1] = len2;
500 dims[2] = len3;
501 dims[3] = len4;
502 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
503 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(4, dims, THREAD);
504 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
505 current->set_vm_result_oop(obj);
506 JRT_END
507
508 // multianewarray for 5 dimensions
509 JRT_ENTRY(void, OptoRuntime::multianewarray5_C(Klass* elem_type, int len1, int len2, int len3, int len4, int len5, JavaThread* current))
510 #ifndef PRODUCT
511 SharedRuntime::_multi5_ctr++; // multianewarray for 1 dimension
512 #endif
513 assert(check_compiled_frame(current), "incorrect caller");
514 assert(elem_type->is_klass(), "not a class");
515 jint dims[5];
516 dims[0] = len1;
517 dims[1] = len2;
518 dims[2] = len3;
519 dims[3] = len4;
520 dims[4] = len5;
521 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
522 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(5, dims, THREAD);
523 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
524 current->set_vm_result_oop(obj);
525 JRT_END
526
527 JRT_ENTRY(void, OptoRuntime::multianewarrayN_C(Klass* elem_type, arrayOopDesc* dims, JavaThread* current))
528 assert(check_compiled_frame(current), "incorrect caller");
529 assert(elem_type->is_klass(), "not a class");
530 assert(oop(dims)->is_typeArray(), "not an array");
531
532 ResourceMark rm;
533 jint len = dims->length();
534 assert(len > 0, "Dimensions array should contain data");
535 jint *c_dims = NEW_RESOURCE_ARRAY(jint, len);
536 ArrayAccess<>::arraycopy_to_native<>(dims, typeArrayOopDesc::element_offset<jint>(0),
537 c_dims, len);
538
539 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
540 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(len, c_dims, THREAD);
541 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
542 current->set_vm_result_oop(obj);
543 JRT_END
544
545 JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notify_C(oopDesc* obj, JavaThread* current))
546
547 // Very few notify/notifyAll operations find any threads on the waitset, so
548 // the dominant fast-path is to simply return.
549 // Relatedly, it's critical that notify/notifyAll be fast in order to
550 // reduce lock hold times.
551 if (!SafepointSynchronize::is_synchronizing()) {
552 if (ObjectSynchronizer::quick_notify(obj, current, false)) {
553 return;
554 }
555 }
556
557 // This is the case the fast-path above isn't provisioned to handle.
558 // The fast-path is designed to handle frequently arising cases in an efficient manner.
559 // (The fast-path is just a degenerate variant of the slow-path).
560 // Perform the dreaded state transition and pass control into the slow-path.
561 JRT_BLOCK;
562 Handle h_obj(current, obj);
563 ObjectSynchronizer::notify(h_obj, CHECK);
564 JRT_BLOCK_END;
565 JRT_END
566
567 JRT_BLOCK_ENTRY(void, OptoRuntime::monitor_notifyAll_C(oopDesc* obj, JavaThread* current))
568
569 if (!SafepointSynchronize::is_synchronizing() ) {
570 if (ObjectSynchronizer::quick_notify(obj, current, true)) {
571 return;
572 }
573 }
574
575 // This is the case the fast-path above isn't provisioned to handle.
576 // The fast-path is designed to handle frequently arising cases in an efficient manner.
577 // (The fast-path is just a degenerate variant of the slow-path).
578 // Perform the dreaded state transition and pass control into the slow-path.
579 JRT_BLOCK;
580 Handle h_obj(current, obj);
581 ObjectSynchronizer::notifyall(h_obj, CHECK);
582 JRT_BLOCK_END;
583 JRT_END
584
585 static const TypeFunc* make_new_instance_Type() {
586 // create input type (domain)
587 const Type **fields = TypeTuple::fields(1);
1797 assert(reg >= 0 && reg < _last_Mach_Reg, "must be a machine register");
1798 switch (register_save_policy[reg]) {
1799 case 'C': return false; //SOC
1800 case 'E': return true ; //SOE
1801 case 'N': return false; //NS
1802 case 'A': return false; //AS
1803 }
1804 ShouldNotReachHere();
1805 return false;
1806 }
1807
1808 //-----------------------------------------------------------------------
1809 // Exceptions
1810 //
1811
1812 static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg);
1813
1814 // The method is an entry that is always called by a C++ method not
1815 // directly from compiled code. Compiled code will call the C++ method following.
1816 // We can't allow async exception to be installed during exception processing.
1817 JRT_ENTRY_NO_ASYNC(address, OptoRuntime::handle_exception_C_helper(JavaThread* current, nmethod* &nm))
1818 // The frame we rethrow the exception to might not have been processed by the GC yet.
1819 // The stack watermark barrier takes care of detecting that and ensuring the frame
1820 // has updated oops.
1821 StackWatermarkSet::after_unwind(current);
1822
1823 // Do not confuse exception_oop with pending_exception. The exception_oop
1824 // is only used to pass arguments into the method. Not for general
1825 // exception handling. DO NOT CHANGE IT to use pending_exception, since
1826 // the runtime stubs checks this on exit.
1827 assert(current->exception_oop() != nullptr, "exception oop is found");
1828 address handler_address = nullptr;
1829
1830 Handle exception(current, current->exception_oop());
1831 address pc = current->exception_pc();
1832
1833 // Clear out the exception oop and pc since looking up an
1834 // exception handler can cause class loading, which might throw an
1835 // exception and those fields are expected to be clear during
1836 // normal bytecode execution.
1837 current->clear_exception_oop_and_pc();
2070 frame caller_frame = stub_frame.sender(®_map);
2071 return caller_frame.is_deoptimized_frame();
2072 }
2073
2074 static const TypeFunc* make_register_finalizer_Type() {
2075 // create input type (domain)
2076 const Type **fields = TypeTuple::fields(1);
2077 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // oop; Receiver
2078 // // The JavaThread* is passed to each routine as the last argument
2079 // fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // JavaThread *; Executing thread
2080 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1,fields);
2081
2082 // create result type (range)
2083 fields = TypeTuple::fields(0);
2084
2085 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2086
2087 return TypeFunc::make(domain,range);
2088 }
2089
2090 #if INCLUDE_JFR
2091 static const TypeFunc* make_class_id_load_barrier_Type() {
2092 // create input type (domain)
2093 const Type **fields = TypeTuple::fields(1);
2094 fields[TypeFunc::Parms+0] = TypeInstPtr::KLASS;
2095 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms + 1, fields);
2096
2097 // create result type (range)
2098 fields = TypeTuple::fields(0);
2099
2100 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms + 0, fields);
2101
2102 return TypeFunc::make(domain,range);
2103 }
2104 #endif // INCLUDE_JFR
2105
2106 //-----------------------------------------------------------------------------
2107 static const TypeFunc* make_dtrace_method_entry_exit_Type() {
2108 // create input type (domain)
2109 const Type **fields = TypeTuple::fields(2);
2110 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2111 fields[TypeFunc::Parms+1] = TypeMetadataPtr::BOTTOM; // Method*; Method we are entering
2112 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
2113
2114 // create result type (range)
2115 fields = TypeTuple::fields(0);
2116
2117 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2118
2119 return TypeFunc::make(domain,range);
2120 }
2121
2122 static const TypeFunc* make_dtrace_object_alloc_Type() {
2123 // create input type (domain)
2124 const Type **fields = TypeTuple::fields(2);
2125 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2126 fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL; // oop; newly allocated object
2127
2128 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
2129
2130 // create result type (range)
2131 fields = TypeTuple::fields(0);
2132
2133 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2134
2135 return TypeFunc::make(domain,range);
2136 }
2137
2138 JRT_ENTRY_NO_ASYNC(void, OptoRuntime::register_finalizer_C(oopDesc* obj, JavaThread* current))
2139 assert(oopDesc::is_oop(obj), "must be a valid oop");
2140 assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
2141 InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
2142 JRT_END
2143
2144 //-----------------------------------------------------------------------------
2145
2146 NamedCounter * volatile OptoRuntime::_named_counters = nullptr;
2147
2148 //
2149 // dump the collected NamedCounters.
2150 //
2151 void OptoRuntime::print_named_counters() {
2152 int total_lock_count = 0;
2153 int eliminated_lock_count = 0;
2154
2155 NamedCounter* c = _named_counters;
2156 while (c) {
2157 if (c->tag() == NamedCounter::LockCounter || c->tag() == NamedCounter::EliminatedLockCounter) {
2158 int count = c->count();
2159 if (count > 0) {
2160 bool eliminated = c->tag() == NamedCounter::EliminatedLockCounter;
2161 if (Verbose) {
2162 tty->print_cr("%d %s%s", count, c->name(), eliminated ? " (eliminated)" : "");
2163 }
2306 static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg) {
2307 trace_exception_counter++;
2308 stringStream tempst;
2309
2310 tempst.print("%d [Exception (%s): ", trace_exception_counter, msg);
2311 exception_oop->print_value_on(&tempst);
2312 tempst.print(" in ");
2313 CodeBlob* blob = CodeCache::find_blob(exception_pc);
2314 if (blob->is_nmethod()) {
2315 blob->as_nmethod()->method()->print_value_on(&tempst);
2316 } else if (blob->is_runtime_stub()) {
2317 tempst.print("<runtime-stub>");
2318 } else {
2319 tempst.print("<unknown>");
2320 }
2321 tempst.print(" at " INTPTR_FORMAT, p2i(exception_pc));
2322 tempst.print("]");
2323
2324 st->print_raw_cr(tempst.freeze());
2325 }
|
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "classfile/vmClasses.hpp"
26 #include "classfile/vmSymbols.hpp"
27 #include "code/codeCache.hpp"
28 #include "code/compiledIC.hpp"
29 #include "code/nmethod.hpp"
30 #include "code/pcDesc.hpp"
31 #include "code/scopeDesc.hpp"
32 #include "code/vtableStubs.hpp"
33 #include "compiler/compilationMemoryStatistic.hpp"
34 #include "compiler/compileBroker.hpp"
35 #include "compiler/compilerDefinitions.inline.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/g1/g1HeapRegion.hpp"
38 #include "gc/shared/barrierSet.hpp"
39 #include "gc/shared/collectedHeap.hpp"
40 #include "gc/shared/gcLocker.hpp"
41 #include "interpreter/bytecode.hpp"
42 #include "interpreter/interpreter.hpp"
43 #include "interpreter/linkResolver.hpp"
44 #include "logging/log.hpp"
45 #include "logging/logStream.hpp"
46 #include "memory/oopFactory.hpp"
47 #include "memory/resourceArea.hpp"
48 #include "oops/objArrayKlass.hpp"
49 #include "oops/klass.inline.hpp"
50 #include "oops/oop.inline.hpp"
51 #include "oops/typeArrayOop.inline.hpp"
52 #include "opto/ad.hpp"
53 #include "opto/addnode.hpp"
54 #include "opto/callnode.hpp"
55 #include "opto/cfgnode.hpp"
56 #include "opto/graphKit.hpp"
57 #include "opto/machnode.hpp"
58 #include "opto/matcher.hpp"
59 #include "opto/memnode.hpp"
60 #include "opto/mulnode.hpp"
61 #include "opto/output.hpp"
62 #include "opto/runtime.hpp"
63 #include "opto/subnode.hpp"
64 #include "prims/jvmtiExport.hpp"
65 #include "runtime/atomic.hpp"
66 #include "runtime/frame.inline.hpp"
67 #include "runtime/handles.inline.hpp"
68 #include "runtime/interfaceSupport.inline.hpp"
69 #include "runtime/java.hpp"
70 #include "runtime/javaCalls.hpp"
71 #include "runtime/perfData.inline.hpp"
72 #include "runtime/sharedRuntime.hpp"
73 #include "runtime/signature.hpp"
74 #include "runtime/stackWatermarkSet.hpp"
75 #include "runtime/synchronizer.hpp"
76 #include "runtime/threadWXSetters.inline.hpp"
77 #include "runtime/vframe.hpp"
78 #include "runtime/vframeArray.hpp"
79 #include "runtime/vframe_hp.hpp"
80 #include "services/management.hpp"
81 #include "utilities/copy.hpp"
82 #include "utilities/preserveException.hpp"
83
84
85 // For debugging purposes:
86 // To force FullGCALot inside a runtime function, add the following two lines
87 //
88 // Universe::release_fullgc_alot_dummy();
89 // Universe::heap()->collect();
90 //
91 // At command line specify the parameters: -XX:+FullGCALot -XX:FullGCALotStart=100000000
92
93
94 #define C2_BLOB_FIELD_DEFINE(name, type) \
95 type OptoRuntime:: BLOB_FIELD_NAME(name) = nullptr;
96 #define C2_STUB_FIELD_NAME(name) _ ## name ## _Java
97 #define C2_STUB_FIELD_DEFINE(name, f, t, r) \
98 address OptoRuntime:: C2_STUB_FIELD_NAME(name) = nullptr;
99 #define C2_JVMTI_STUB_FIELD_DEFINE(name) \
100 address OptoRuntime:: STUB_FIELD_NAME(name) = nullptr;
101 C2_STUBS_DO(C2_BLOB_FIELD_DEFINE, C2_STUB_FIELD_DEFINE, C2_JVMTI_STUB_FIELD_DEFINE)
102 #undef C2_BLOB_FIELD_DEFINE
103 #undef C2_STUB_FIELD_DEFINE
104 #undef C2_JVMTI_STUB_FIELD_DEFINE
105
106 #define C2_BLOB_NAME_DEFINE(name, type) "C2 Runtime " # name "_blob",
107 #define C2_STUB_NAME_DEFINE(name, f, t, r) "C2 Runtime " # name,
108 #define C2_JVMTI_STUB_NAME_DEFINE(name) "C2 Runtime " # name,
109 const char* OptoRuntime::_stub_names[] = {
110 C2_STUBS_DO(C2_BLOB_NAME_DEFINE, C2_STUB_NAME_DEFINE, C2_JVMTI_STUB_NAME_DEFINE)
111 };
112 #undef C2_BLOB_NAME_DEFINE
113 #undef C2_STUB_NAME_DEFINE
114 #undef C2_JVMTI_STUB_NAME_DEFINE
115
116 address OptoRuntime::_vtable_must_compile_Java = nullptr;
117
118 PerfCounter* _perf_OptoRuntime_class_init_barrier_redundant_count = nullptr;
119
120 // This should be called in an assertion at the start of OptoRuntime routines
121 // which are entered from compiled code (all of them)
122 #ifdef ASSERT
123 static bool check_compiled_frame(JavaThread* thread) {
124 assert(thread->last_frame().is_runtime_frame(), "cannot call runtime directly from compiled code");
125 RegisterMap map(thread,
126 RegisterMap::UpdateMap::skip,
127 RegisterMap::ProcessFrames::include,
128 RegisterMap::WalkContinuation::skip);
129 frame caller = thread->last_frame().sender(&map);
130 assert(caller.is_compiled_frame(), "not being called from compiled like code");
131 return true;
132 }
133 #endif // ASSERT
134
135 /*
136 #define gen(env, var, type_func_gen, c_func, fancy_jump, pass_tls, return_pc) \
137 var = generate_stub(env, type_func_gen, CAST_FROM_FN_PTR(address, c_func), #var, fancy_jump, pass_tls, return_pc); \
138 if (var == nullptr) { return false; }
139 */
169 fancy_jump, \
170 pass_tls, \
171 pass_retpc); \
172 if (C2_STUB_FIELD_NAME(name) == nullptr) { return false; } \
173
174 #define C2_JVMTI_STUB_C_FUNC(name) CAST_FROM_FN_PTR(address, SharedRuntime::name)
175
176 #define GEN_C2_JVMTI_STUB(name) \
177 STUB_FIELD_NAME(name) = \
178 generate_stub(env, \
179 notify_jvmti_vthread_Type, \
180 C2_JVMTI_STUB_C_FUNC(name), \
181 C2_STUB_NAME(name), \
182 (int)C2_STUB_ID(name), \
183 0, \
184 true, \
185 false); \
186 if (STUB_FIELD_NAME(name) == nullptr) { return false; } \
187
188 bool OptoRuntime::generate(ciEnv* env) {
189 init_counters();
190
191 C2_STUBS_DO(GEN_C2_BLOB, GEN_C2_STUB, GEN_C2_JVMTI_STUB)
192
193 return true;
194 }
195
196 #undef GEN_C2_BLOB
197
198 #undef C2_STUB_FIELD_NAME
199 #undef C2_STUB_TYPEFUNC
200 #undef C2_STUB_C_FUNC
201 #undef C2_STUB_NAME
202 #undef GEN_C2_STUB
203
204 #undef C2_JVMTI_STUB_C_FUNC
205 #undef GEN_C2_JVMTI_STUB
206 // #undef gen
207
208 const TypeFunc* OptoRuntime::_new_instance_Type = nullptr;
209 const TypeFunc* OptoRuntime::_new_array_Type = nullptr;
276 const TypeFunc* OptoRuntime::_updateBytesAdler32_Type = nullptr;
277 const TypeFunc* OptoRuntime::_osr_end_Type = nullptr;
278 const TypeFunc* OptoRuntime::_register_finalizer_Type = nullptr;
279 #if INCLUDE_JFR
280 const TypeFunc* OptoRuntime::_class_id_load_barrier_Type = nullptr;
281 #endif // INCLUDE_JFR
282 #if INCLUDE_JVMTI
283 const TypeFunc* OptoRuntime::_notify_jvmti_vthread_Type = nullptr;
284 #endif // INCLUDE_JVMTI
285 const TypeFunc* OptoRuntime::_dtrace_method_entry_exit_Type = nullptr;
286 const TypeFunc* OptoRuntime::_dtrace_object_alloc_Type = nullptr;
287
288 // Helper method to do generation of RunTimeStub's
289 address OptoRuntime::generate_stub(ciEnv* env,
290 TypeFunc_generator gen, address C_function,
291 const char *name, int stub_id,
292 int is_fancy_jump, bool pass_tls,
293 bool return_pc) {
294
295 // Matching the default directive, we currently have no method to match.
296 DirectiveSet* directive = DirectivesStack::getDefaultDirective(CompilerThread::current()->compiler());
297 CompilationMemoryStatisticMark cmsm(directive);
298 ResourceMark rm;
299 Compile C(env, gen, C_function, name, stub_id, is_fancy_jump, pass_tls, return_pc, directive);
300 DirectivesStack::release(directive);
301 return C.stub_entry_point();
302 }
303
304 const char* OptoRuntime::stub_name(address entry) {
305 #ifndef PRODUCT
306 CodeBlob* cb = CodeCache::find_blob(entry);
307 RuntimeStub* rs =(RuntimeStub *)cb;
308 assert(rs != nullptr && rs->is_runtime_stub(), "not a runtime stub");
309 return rs->name();
310 #else
311 // Fast implementation for product mode (maybe it should be inlined too)
312 return "runtime stub";
313 #endif
314 }
315
316 // local methods passed as arguments to stub generator that forward
320 oopDesc* dest, jint dest_pos,
321 jint length, JavaThread* thread) {
322 SharedRuntime::slow_arraycopy_C(src, src_pos, dest, dest_pos, length, thread);
323 }
324
325 void OptoRuntime::complete_monitor_locking_C(oopDesc* obj, BasicLock* lock, JavaThread* current) {
326 SharedRuntime::complete_monitor_locking_C(obj, lock, current);
327 }
328
329
330 //=============================================================================
331 // Opto compiler runtime routines
332 //=============================================================================
333
334
335 //=============================allocation======================================
336 // We failed the fast-path allocation. Now we need to do a scavenge or GC
337 // and try allocation again.
338
339 // object allocation
340 JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, new_instance_C, OptoRuntime::new_instance_C(Klass* klass, JavaThread* current))
341 JRT_BLOCK;
342 #ifndef PRODUCT
343 SharedRuntime::_new_instance_ctr++; // new instance requires GC
344 #endif
345 assert(check_compiled_frame(current), "incorrect caller");
346
347 // These checks are cheap to make and support reflective allocation.
348 int lh = klass->layout_helper();
349 if (Klass::layout_helper_needs_slow_path(lh) || !InstanceKlass::cast(klass)->is_initialized()) {
350 Handle holder(current, klass->klass_holder()); // keep the klass alive
351 klass->check_valid_for_instantiation(false, THREAD);
352 if (!HAS_PENDING_EXCEPTION) {
353 InstanceKlass::cast(klass)->initialize(THREAD);
354 }
355 }
356
357 if (!HAS_PENDING_EXCEPTION) {
358 // Scavenge and allocate an instance.
359 Handle holder(current, klass->klass_holder()); // keep the klass alive
360 oop result = InstanceKlass::cast(klass)->allocate_instance(THREAD);
361 current->set_vm_result_oop(result);
362
363 // Pass oops back through thread local storage. Our apparent type to Java
364 // is that we return an oop, but we can block on exit from this routine and
365 // a GC can trash the oop in C's return register. The generated stub will
366 // fetch the oop from TLS after any possible GC.
367 }
368
369 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
370 JRT_BLOCK_END;
371
372 // inform GC that we won't do card marks for initializing writes.
373 SharedRuntime::on_slowpath_allocation_exit(current);
374 JRT_END
375
376
377 // array allocation
378 JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, new_array_C, OptoRuntime::new_array_C(Klass* array_type, int len, JavaThread* current))
379 JRT_BLOCK;
380 #ifndef PRODUCT
381 SharedRuntime::_new_array_ctr++; // new array requires GC
382 #endif
383 assert(check_compiled_frame(current), "incorrect caller");
384
385 // Scavenge and allocate an instance.
386 oop result;
387
388 if (array_type->is_typeArray_klass()) {
389 // The oopFactory likes to work with the element type.
390 // (We could bypass the oopFactory, since it doesn't add much value.)
391 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
392 result = oopFactory::new_typeArray(elem_type, len, THREAD);
393 } else {
394 // Although the oopFactory likes to work with the elem_type,
395 // the compiler prefers the array_type, since it must already have
396 // that latter value in hand for the fast path.
397 Handle holder(current, array_type->klass_holder()); // keep the array klass alive
398 Klass* elem_type = ObjArrayKlass::cast(array_type)->element_klass();
399 result = oopFactory::new_objArray(elem_type, len, THREAD);
400 }
401
402 // Pass oops back through thread local storage. Our apparent type to Java
403 // is that we return an oop, but we can block on exit from this routine and
404 // a GC can trash the oop in C's return register. The generated stub will
405 // fetch the oop from TLS after any possible GC.
406 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
407 current->set_vm_result_oop(result);
408 JRT_BLOCK_END;
409
410 // inform GC that we won't do card marks for initializing writes.
411 SharedRuntime::on_slowpath_allocation_exit(current);
412 JRT_END
413
414 // array allocation without zeroing
415 JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, new_array_nozero_C, OptoRuntime::new_array_nozero_C(Klass* array_type, int len, JavaThread* current))
416 JRT_BLOCK;
417 #ifndef PRODUCT
418 SharedRuntime::_new_array_ctr++; // new array requires GC
419 #endif
420 assert(check_compiled_frame(current), "incorrect caller");
421
422 // Scavenge and allocate an instance.
423 oop result;
424
425 assert(array_type->is_typeArray_klass(), "should be called only for type array");
426 // The oopFactory likes to work with the element type.
427 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
428 result = oopFactory::new_typeArray_nozero(elem_type, len, THREAD);
429
430 // Pass oops back through thread local storage. Our apparent type to Java
431 // is that we return an oop, but we can block on exit from this routine and
432 // a GC can trash the oop in C's return register. The generated stub will
433 // fetch the oop from TLS after any possible GC.
434 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
435 current->set_vm_result_oop(result);
447 BasicType elem_type = TypeArrayKlass::cast(array_type)->element_type();
448 size_t hs_bytes = arrayOopDesc::base_offset_in_bytes(elem_type);
449 assert(is_aligned(hs_bytes, BytesPerInt), "must be 4 byte aligned");
450 HeapWord* obj = cast_from_oop<HeapWord*>(result);
451 if (!is_aligned(hs_bytes, BytesPerLong)) {
452 *reinterpret_cast<jint*>(reinterpret_cast<char*>(obj) + hs_bytes) = 0;
453 hs_bytes += BytesPerInt;
454 }
455
456 // Optimized zeroing.
457 assert(is_aligned(hs_bytes, BytesPerLong), "must be 8-byte aligned");
458 const size_t aligned_hs = hs_bytes / BytesPerLong;
459 Copy::fill_to_aligned_words(obj+aligned_hs, size-aligned_hs);
460 }
461
462 JRT_END
463
464 // Note: multianewarray for one dimension is handled inline by GraphKit::new_array.
465
466 // multianewarray for 2 dimensions
467 JRT_ENTRY_PROF(void, OptoRuntime, multianewarray2_C, OptoRuntime::multianewarray2_C(Klass* elem_type, int len1, int len2, JavaThread* current))
468 #ifndef PRODUCT
469 SharedRuntime::_multi2_ctr++; // multianewarray for 1 dimension
470 #endif
471 assert(check_compiled_frame(current), "incorrect caller");
472 assert(elem_type->is_klass(), "not a class");
473 jint dims[2];
474 dims[0] = len1;
475 dims[1] = len2;
476 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
477 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(2, dims, THREAD);
478 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
479 current->set_vm_result_oop(obj);
480 JRT_END
481
482 // multianewarray for 3 dimensions
483 JRT_ENTRY_PROF(void, OptoRuntime, multianewarray3_C, OptoRuntime::multianewarray3_C(Klass* elem_type, int len1, int len2, int len3, JavaThread* current))
484 #ifndef PRODUCT
485 SharedRuntime::_multi3_ctr++; // multianewarray for 1 dimension
486 #endif
487 assert(check_compiled_frame(current), "incorrect caller");
488 assert(elem_type->is_klass(), "not a class");
489 jint dims[3];
490 dims[0] = len1;
491 dims[1] = len2;
492 dims[2] = len3;
493 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
494 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(3, dims, THREAD);
495 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
496 current->set_vm_result_oop(obj);
497 JRT_END
498
499 // multianewarray for 4 dimensions
500 JRT_ENTRY_PROF(void, OptoRuntime, multianewarray4_C, OptoRuntime::multianewarray4_C(Klass* elem_type, int len1, int len2, int len3, int len4, JavaThread* current))
501 #ifndef PRODUCT
502 SharedRuntime::_multi4_ctr++; // multianewarray for 1 dimension
503 #endif
504 assert(check_compiled_frame(current), "incorrect caller");
505 assert(elem_type->is_klass(), "not a class");
506 jint dims[4];
507 dims[0] = len1;
508 dims[1] = len2;
509 dims[2] = len3;
510 dims[3] = len4;
511 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
512 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(4, dims, THREAD);
513 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
514 current->set_vm_result_oop(obj);
515 JRT_END
516
517 // multianewarray for 5 dimensions
518 JRT_ENTRY(void, OptoRuntime::multianewarray5_C(Klass* elem_type, int len1, int len2, int len3, int len4, int len5, JavaThread* current))
519 #ifndef PRODUCT
520 SharedRuntime::_multi5_ctr++; // multianewarray for 1 dimension
521 #endif
522 assert(check_compiled_frame(current), "incorrect caller");
523 assert(elem_type->is_klass(), "not a class");
524 jint dims[5];
525 dims[0] = len1;
526 dims[1] = len2;
527 dims[2] = len3;
528 dims[3] = len4;
529 dims[4] = len5;
530 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
531 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(5, dims, THREAD);
532 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
533 current->set_vm_result_oop(obj);
534 JRT_END
535
536 JRT_ENTRY_PROF(void, OptoRuntime, multianewarrayN_C, OptoRuntime::multianewarrayN_C(Klass* elem_type, arrayOopDesc* dims, JavaThread* current))
537 assert(check_compiled_frame(current), "incorrect caller");
538 assert(elem_type->is_klass(), "not a class");
539 assert(oop(dims)->is_typeArray(), "not an array");
540
541 ResourceMark rm;
542 jint len = dims->length();
543 assert(len > 0, "Dimensions array should contain data");
544 jint *c_dims = NEW_RESOURCE_ARRAY(jint, len);
545 ArrayAccess<>::arraycopy_to_native<>(dims, typeArrayOopDesc::element_offset<jint>(0),
546 c_dims, len);
547
548 Handle holder(current, elem_type->klass_holder()); // keep the klass alive
549 oop obj = ArrayKlass::cast(elem_type)->multi_allocate(len, c_dims, THREAD);
550 deoptimize_caller_frame(current, HAS_PENDING_EXCEPTION);
551 current->set_vm_result_oop(obj);
552 JRT_END
553
554 JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, monitor_notify_C, OptoRuntime::monitor_notify_C(oopDesc* obj, JavaThread* current))
555
556 // Very few notify/notifyAll operations find any threads on the waitset, so
557 // the dominant fast-path is to simply return.
558 // Relatedly, it's critical that notify/notifyAll be fast in order to
559 // reduce lock hold times.
560 if (!SafepointSynchronize::is_synchronizing()) {
561 if (ObjectSynchronizer::quick_notify(obj, current, false)) {
562 return;
563 }
564 }
565
566 // This is the case the fast-path above isn't provisioned to handle.
567 // The fast-path is designed to handle frequently arising cases in an efficient manner.
568 // (The fast-path is just a degenerate variant of the slow-path).
569 // Perform the dreaded state transition and pass control into the slow-path.
570 JRT_BLOCK;
571 Handle h_obj(current, obj);
572 ObjectSynchronizer::notify(h_obj, CHECK);
573 JRT_BLOCK_END;
574 JRT_END
575
576 JRT_BLOCK_ENTRY_PROF(void, OptoRuntime, monitor_notifyAll_C, OptoRuntime::monitor_notifyAll_C(oopDesc* obj, JavaThread* current))
577
578 if (!SafepointSynchronize::is_synchronizing() ) {
579 if (ObjectSynchronizer::quick_notify(obj, current, true)) {
580 return;
581 }
582 }
583
584 // This is the case the fast-path above isn't provisioned to handle.
585 // The fast-path is designed to handle frequently arising cases in an efficient manner.
586 // (The fast-path is just a degenerate variant of the slow-path).
587 // Perform the dreaded state transition and pass control into the slow-path.
588 JRT_BLOCK;
589 Handle h_obj(current, obj);
590 ObjectSynchronizer::notifyall(h_obj, CHECK);
591 JRT_BLOCK_END;
592 JRT_END
593
594 static const TypeFunc* make_new_instance_Type() {
595 // create input type (domain)
596 const Type **fields = TypeTuple::fields(1);
1806 assert(reg >= 0 && reg < _last_Mach_Reg, "must be a machine register");
1807 switch (register_save_policy[reg]) {
1808 case 'C': return false; //SOC
1809 case 'E': return true ; //SOE
1810 case 'N': return false; //NS
1811 case 'A': return false; //AS
1812 }
1813 ShouldNotReachHere();
1814 return false;
1815 }
1816
1817 //-----------------------------------------------------------------------
1818 // Exceptions
1819 //
1820
1821 static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg);
1822
1823 // The method is an entry that is always called by a C++ method not
1824 // directly from compiled code. Compiled code will call the C++ method following.
1825 // We can't allow async exception to be installed during exception processing.
1826 JRT_ENTRY_NO_ASYNC_PROF(address, OptoRuntime, handle_exception_C_helper, OptoRuntime::handle_exception_C_helper(JavaThread* current, nmethod* &nm))
1827 // The frame we rethrow the exception to might not have been processed by the GC yet.
1828 // The stack watermark barrier takes care of detecting that and ensuring the frame
1829 // has updated oops.
1830 StackWatermarkSet::after_unwind(current);
1831
1832 // Do not confuse exception_oop with pending_exception. The exception_oop
1833 // is only used to pass arguments into the method. Not for general
1834 // exception handling. DO NOT CHANGE IT to use pending_exception, since
1835 // the runtime stubs checks this on exit.
1836 assert(current->exception_oop() != nullptr, "exception oop is found");
1837 address handler_address = nullptr;
1838
1839 Handle exception(current, current->exception_oop());
1840 address pc = current->exception_pc();
1841
1842 // Clear out the exception oop and pc since looking up an
1843 // exception handler can cause class loading, which might throw an
1844 // exception and those fields are expected to be clear during
1845 // normal bytecode execution.
1846 current->clear_exception_oop_and_pc();
2079 frame caller_frame = stub_frame.sender(®_map);
2080 return caller_frame.is_deoptimized_frame();
2081 }
2082
2083 static const TypeFunc* make_register_finalizer_Type() {
2084 // create input type (domain)
2085 const Type **fields = TypeTuple::fields(1);
2086 fields[TypeFunc::Parms+0] = TypeInstPtr::NOTNULL; // oop; Receiver
2087 // // The JavaThread* is passed to each routine as the last argument
2088 // fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // JavaThread *; Executing thread
2089 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1,fields);
2090
2091 // create result type (range)
2092 fields = TypeTuple::fields(0);
2093
2094 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2095
2096 return TypeFunc::make(domain,range);
2097 }
2098
2099 const TypeFunc *OptoRuntime::class_init_barrier_Type() {
2100 // create input type (domain)
2101 const Type** fields = TypeTuple::fields(1);
2102 fields[TypeFunc::Parms+0] = TypeKlassPtr::NOTNULL;
2103 // // The JavaThread* is passed to each routine as the last argument
2104 // fields[TypeFunc::Parms+1] = TypeRawPtr::NOTNULL; // JavaThread *; Executing thread
2105 const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms+1, fields);
2106
2107 // create result type (range)
2108 fields = TypeTuple::fields(0);
2109 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0, fields);
2110 return TypeFunc::make(domain,range);
2111 }
2112
2113 #if INCLUDE_JFR
2114 static const TypeFunc* make_class_id_load_barrier_Type() {
2115 // create input type (domain)
2116 const Type **fields = TypeTuple::fields(1);
2117 fields[TypeFunc::Parms+0] = TypeInstPtr::KLASS;
2118 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms + 1, fields);
2119
2120 // create result type (range)
2121 fields = TypeTuple::fields(0);
2122
2123 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms + 0, fields);
2124
2125 return TypeFunc::make(domain,range);
2126 }
2127 #endif // INCLUDE_JFR
2128
2129 //-----------------------------------------------------------------------------
2130 // runtime upcall support
2131 const TypeFunc *OptoRuntime::runtime_up_call_Type() {
2132 // create input type (domain)
2133 const Type **fields = TypeTuple::fields(1);
2134 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2135 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+1,fields);
2136
2137 // create result type (range)
2138 fields = TypeTuple::fields(0);
2139
2140 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2141
2142 return TypeFunc::make(domain,range);
2143 }
2144
2145 //-----------------------------------------------------------------------------
2146 static const TypeFunc* make_dtrace_method_entry_exit_Type() {
2147 // create input type (domain)
2148 const Type **fields = TypeTuple::fields(2);
2149 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2150 fields[TypeFunc::Parms+1] = TypeMetadataPtr::BOTTOM; // Method*; Method we are entering
2151 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
2152
2153 // create result type (range)
2154 fields = TypeTuple::fields(0);
2155
2156 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2157
2158 return TypeFunc::make(domain,range);
2159 }
2160
2161 static const TypeFunc* make_dtrace_object_alloc_Type() {
2162 // create input type (domain)
2163 const Type **fields = TypeTuple::fields(2);
2164 fields[TypeFunc::Parms+0] = TypeRawPtr::BOTTOM; // Thread-local storage
2165 fields[TypeFunc::Parms+1] = TypeInstPtr::NOTNULL; // oop; newly allocated object
2166
2167 const TypeTuple *domain = TypeTuple::make(TypeFunc::Parms+2,fields);
2168
2169 // create result type (range)
2170 fields = TypeTuple::fields(0);
2171
2172 const TypeTuple *range = TypeTuple::make(TypeFunc::Parms+0,fields);
2173
2174 return TypeFunc::make(domain,range);
2175 }
2176
2177 JRT_ENTRY_NO_ASYNC_PROF(void, OptoRuntime, register_finalizer_C, OptoRuntime::register_finalizer_C(oopDesc* obj, JavaThread* current))
2178 assert(oopDesc::is_oop(obj), "must be a valid oop");
2179 assert(obj->klass()->has_finalizer(), "shouldn't be here otherwise");
2180 InstanceKlass::register_finalizer(instanceOop(obj), CHECK);
2181 JRT_END
2182
2183 JRT_ENTRY_NO_ASYNC_PROF(void, OptoRuntime, class_init_barrier_C, OptoRuntime::class_init_barrier_C(Klass* k, JavaThread* current))
2184 InstanceKlass* ik = InstanceKlass::cast(k);
2185 if (ik->should_be_initialized()) {
2186 ik->initialize(CHECK);
2187 } else if (UsePerfData) {
2188 _perf_OptoRuntime_class_init_barrier_redundant_count->inc();
2189 }
2190 JRT_END
2191
2192 //-----------------------------------------------------------------------------
2193
2194 NamedCounter * volatile OptoRuntime::_named_counters = nullptr;
2195
2196 //
2197 // dump the collected NamedCounters.
2198 //
2199 void OptoRuntime::print_named_counters() {
2200 int total_lock_count = 0;
2201 int eliminated_lock_count = 0;
2202
2203 NamedCounter* c = _named_counters;
2204 while (c) {
2205 if (c->tag() == NamedCounter::LockCounter || c->tag() == NamedCounter::EliminatedLockCounter) {
2206 int count = c->count();
2207 if (count > 0) {
2208 bool eliminated = c->tag() == NamedCounter::EliminatedLockCounter;
2209 if (Verbose) {
2210 tty->print_cr("%d %s%s", count, c->name(), eliminated ? " (eliminated)" : "");
2211 }
2354 static void trace_exception(outputStream* st, oop exception_oop, address exception_pc, const char* msg) {
2355 trace_exception_counter++;
2356 stringStream tempst;
2357
2358 tempst.print("%d [Exception (%s): ", trace_exception_counter, msg);
2359 exception_oop->print_value_on(&tempst);
2360 tempst.print(" in ");
2361 CodeBlob* blob = CodeCache::find_blob(exception_pc);
2362 if (blob->is_nmethod()) {
2363 blob->as_nmethod()->method()->print_value_on(&tempst);
2364 } else if (blob->is_runtime_stub()) {
2365 tempst.print("<runtime-stub>");
2366 } else {
2367 tempst.print("<unknown>");
2368 }
2369 tempst.print(" at " INTPTR_FORMAT, p2i(exception_pc));
2370 tempst.print("]");
2371
2372 st->print_raw_cr(tempst.freeze());
2373 }
2374
2375 #define DO_COUNTERS2(macro2, macro1) \
2376 macro2(OptoRuntime, new_instance_C) \
2377 macro2(OptoRuntime, new_array_C) \
2378 macro2(OptoRuntime, new_array_nozero_C) \
2379 macro2(OptoRuntime, multianewarray2_C) \
2380 macro2(OptoRuntime, multianewarray3_C) \
2381 macro2(OptoRuntime, multianewarray4_C) \
2382 macro2(OptoRuntime, multianewarrayN_C) \
2383 macro2(OptoRuntime, monitor_notify_C) \
2384 macro2(OptoRuntime, monitor_notifyAll_C) \
2385 macro2(OptoRuntime, handle_exception_C_helper) \
2386 macro2(OptoRuntime, register_finalizer_C) \
2387 macro2(OptoRuntime, class_init_barrier_C) \
2388 macro1(OptoRuntime, class_init_barrier_redundant)
2389
2390 #define INIT_COUNTER_TIME_AND_CNT(sub, name) \
2391 NEWPERFTICKCOUNTERS(_perf_##sub##_##name##_timer, SUN_CI, #sub "::" #name); \
2392 NEWPERFEVENTCOUNTER(_perf_##sub##_##name##_count, SUN_CI, #sub "::" #name "_count");
2393
2394 #define INIT_COUNTER_CNT(sub, name) \
2395 NEWPERFEVENTCOUNTER(_perf_##sub##_##name##_count, SUN_CI, #sub "::" #name "_count");
2396
2397 void OptoRuntime::init_counters() {
2398 assert(CompilerConfig::is_c2_enabled(), "");
2399
2400 if (UsePerfData) {
2401 EXCEPTION_MARK;
2402
2403 DO_COUNTERS2(INIT_COUNTER_TIME_AND_CNT, INIT_COUNTER_CNT)
2404
2405 if (HAS_PENDING_EXCEPTION) {
2406 vm_exit_during_initialization("jvm_perf_init failed unexpectedly");
2407 }
2408 }
2409 }
2410 #undef INIT_COUNTER_TIME_AND_CNT
2411 #undef INIT_COUNTER_CNT
2412
2413 #define PRINT_COUNTER_TIME_AND_CNT(sub, name) { \
2414 jlong count = _perf_##sub##_##name##_count->get_value(); \
2415 if (count > 0) { \
2416 st->print_cr(" %-50s = " JLONG_FORMAT_W(6) "us (elapsed) " JLONG_FORMAT_W(6) "us (thread) (" JLONG_FORMAT_W(5) " events)", #sub "::" #name, \
2417 _perf_##sub##_##name##_timer->elapsed_counter_value_us(), \
2418 _perf_##sub##_##name##_timer->thread_counter_value_us(), \
2419 count); \
2420 }}
2421
2422 #define PRINT_COUNTER_CNT(sub, name) { \
2423 jlong count = _perf_##sub##_##name##_count->get_value(); \
2424 if (count > 0) { \
2425 st->print_cr(" %-30s = " JLONG_FORMAT_W(5) " events", #name, count); \
2426 }}
2427
2428 void OptoRuntime::print_counters_on(outputStream* st) {
2429 if (UsePerfData && ProfileRuntimeCalls && CompilerConfig::is_c2_enabled()) {
2430 DO_COUNTERS2(PRINT_COUNTER_TIME_AND_CNT, PRINT_COUNTER_CNT)
2431 } else {
2432 st->print_cr(" OptoRuntime: no info (%s is disabled)",
2433 (!CompilerConfig::is_c2_enabled() ? "C2" : (UsePerfData ? "ProfileRuntimeCalls" : "UsePerfData")));
2434 }
2435 }
2436
2437 #undef PRINT_COUNTER_TIME_AND_CNT
2438 #undef PRINT_COUNTER_CNT
2439 #undef DO_COUNTERS2
|