455 case vmIntrinsics::_getAndAddInt: return inline_unsafe_load_store(T_INT, LS_get_add, Volatile);
456 case vmIntrinsics::_getAndAddLong: return inline_unsafe_load_store(T_LONG, LS_get_add, Volatile);
457
458 case vmIntrinsics::_getAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_get_set, Volatile);
459 case vmIntrinsics::_getAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_get_set, Volatile);
460 case vmIntrinsics::_getAndSetInt: return inline_unsafe_load_store(T_INT, LS_get_set, Volatile);
461 case vmIntrinsics::_getAndSetLong: return inline_unsafe_load_store(T_LONG, LS_get_set, Volatile);
462 case vmIntrinsics::_getAndSetReference: return inline_unsafe_load_store(T_OBJECT, LS_get_set, Volatile);
463
464 case vmIntrinsics::_loadFence:
465 case vmIntrinsics::_storeFence:
466 case vmIntrinsics::_storeStoreFence:
467 case vmIntrinsics::_fullFence: return inline_unsafe_fence(intrinsic_id());
468
469 case vmIntrinsics::_onSpinWait: return inline_onspinwait();
470
471 case vmIntrinsics::_currentCarrierThread: return inline_native_currentCarrierThread();
472 case vmIntrinsics::_currentThread: return inline_native_currentThread();
473 case vmIntrinsics::_setCurrentThread: return inline_native_setCurrentThread();
474
475 case vmIntrinsics::_extentLocalCache: return inline_native_extentLocalCache();
476 case vmIntrinsics::_setExtentLocalCache: return inline_native_setExtentLocalCache();
477
478 #ifdef JFR_HAVE_INTRINSICS
479 case vmIntrinsics::_counterTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, JfrTime::time_function()), "counterTime");
480 case vmIntrinsics::_getEventWriter: return inline_native_getEventWriter();
481 #endif
482 case vmIntrinsics::_currentTimeMillis: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis");
483 case vmIntrinsics::_nanoTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime");
484 case vmIntrinsics::_writeback0: return inline_unsafe_writeback0();
485 case vmIntrinsics::_writebackPreSync0: return inline_unsafe_writebackSync0(true);
486 case vmIntrinsics::_writebackPostSync0: return inline_unsafe_writebackSync0(false);
487 case vmIntrinsics::_allocateInstance: return inline_unsafe_allocate();
488 case vmIntrinsics::_copyMemory: return inline_unsafe_copyMemory();
489 case vmIntrinsics::_getLength: return inline_native_getLength();
490 case vmIntrinsics::_copyOf: return inline_array_copyOf(false);
491 case vmIntrinsics::_copyOfRange: return inline_array_copyOf(true);
492 case vmIntrinsics::_equalsB: return inline_array_equals(StrIntrinsicNode::LL);
493 case vmIntrinsics::_equalsC: return inline_array_equals(StrIntrinsicNode::UU);
494 case vmIntrinsics::_Preconditions_checkIndex: return inline_preconditions_checkIndex(T_INT);
495 case vmIntrinsics::_Preconditions_checkLongIndex: return inline_preconditions_checkIndex(T_LONG);
496 case vmIntrinsics::_clone: return inline_native_clone(intrinsic()->is_virtual());
3338 }
3339
3340 //------------------------inline_native_setVthread------------------
3341 bool LibraryCallKit::inline_native_setCurrentThread() {
3342 assert(C->method()->changes_current_thread(),
3343 "method changes current Thread but is not annotated ChangesCurrentThread");
3344 Node* arr = argument(1);
3345 Node* thread = _gvn.transform(new ThreadLocalNode());
3346 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::vthread_offset()));
3347 Node* thread_obj_handle
3348 = make_load(NULL, p, p->bottom_type()->is_ptr(), T_OBJECT, MemNode::unordered);
3349 thread_obj_handle = _gvn.transform(thread_obj_handle);
3350 const TypePtr *adr_type = _gvn.type(thread_obj_handle)->isa_ptr();
3351 // Stores of oops to native memory not supported yet by BarrierSetC2::store_at_resolved
3352 // access_store_at(NULL, thread_obj_handle, adr_type, arr, _gvn.type(arr), T_OBJECT, IN_NATIVE | MO_UNORDERED);
3353 store_to_memory(control(), thread_obj_handle, arr, T_OBJECT, adr_type, MemNode::unordered);
3354 JFR_ONLY(extend_setCurrentThread(thread, arr);)
3355 return true;
3356 }
3357
3358 Node* LibraryCallKit::extentLocalCache_helper() {
3359 ciKlass *objects_klass = ciObjArrayKlass::make(env()->Object_klass());
3360 const TypeOopPtr *etype = TypeOopPtr::make_from_klass(env()->Object_klass());
3361
3362 bool xk = etype->klass_is_exact();
3363
3364 Node* thread = _gvn.transform(new ThreadLocalNode());
3365 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::extentLocalCache_offset()));
3366 return _gvn.transform(LoadNode::make(_gvn, NULL, immutable_memory(), p, p->bottom_type()->is_ptr(),
3367 TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered));
3368 }
3369
3370 //------------------------inline_native_extentLocalCache------------------
3371 bool LibraryCallKit::inline_native_extentLocalCache() {
3372 ciKlass *objects_klass = ciObjArrayKlass::make(env()->Object_klass());
3373 const TypeOopPtr *etype = TypeOopPtr::make_from_klass(env()->Object_klass());
3374 const TypeAry* arr0 = TypeAry::make(etype, TypeInt::POS);
3375
3376 // Because we create the extentLocal cache lazily we have to make the
3377 // type of the result BotPTR.
3378 bool xk = etype->klass_is_exact();
3379 const Type* objects_type = TypeAryPtr::make(TypePtr::BotPTR, arr0, objects_klass, xk, 0);
3380 Node* cache_obj_handle = extentLocalCache_helper();
3381 set_result(access_load(cache_obj_handle, objects_type, T_OBJECT, IN_NATIVE));
3382
3383 return true;
3384 }
3385
3386 //------------------------inline_native_setExtentLocalCache------------------
3387 bool LibraryCallKit::inline_native_setExtentLocalCache() {
3388 Node* arr = argument(0);
3389 Node* cache_obj_handle = extentLocalCache_helper();
3390
3391 const TypePtr *adr_type = _gvn.type(cache_obj_handle)->isa_ptr();
3392 store_to_memory(control(), cache_obj_handle, arr, T_OBJECT, adr_type,
3393 MemNode::unordered);
3394
3395 return true;
3396 }
3397
3398 //---------------------------load_mirror_from_klass----------------------------
3399 // Given a klass oop, load its java mirror (a java.lang.Class oop).
3400 Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
3401 Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
3402 Node* load = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3403 // mirror = ((OopHandle)mirror)->resolve();
3404 return access_load(load, TypeInstPtr::MIRROR, T_OBJECT, IN_NATIVE);
3405 }
3406
3407 //-----------------------load_klass_from_mirror_common-------------------------
3408 // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop.
3409 // Test the klass oop for null (signifying a primitive Class like Integer.TYPE),
|
455 case vmIntrinsics::_getAndAddInt: return inline_unsafe_load_store(T_INT, LS_get_add, Volatile);
456 case vmIntrinsics::_getAndAddLong: return inline_unsafe_load_store(T_LONG, LS_get_add, Volatile);
457
458 case vmIntrinsics::_getAndSetByte: return inline_unsafe_load_store(T_BYTE, LS_get_set, Volatile);
459 case vmIntrinsics::_getAndSetShort: return inline_unsafe_load_store(T_SHORT, LS_get_set, Volatile);
460 case vmIntrinsics::_getAndSetInt: return inline_unsafe_load_store(T_INT, LS_get_set, Volatile);
461 case vmIntrinsics::_getAndSetLong: return inline_unsafe_load_store(T_LONG, LS_get_set, Volatile);
462 case vmIntrinsics::_getAndSetReference: return inline_unsafe_load_store(T_OBJECT, LS_get_set, Volatile);
463
464 case vmIntrinsics::_loadFence:
465 case vmIntrinsics::_storeFence:
466 case vmIntrinsics::_storeStoreFence:
467 case vmIntrinsics::_fullFence: return inline_unsafe_fence(intrinsic_id());
468
469 case vmIntrinsics::_onSpinWait: return inline_onspinwait();
470
471 case vmIntrinsics::_currentCarrierThread: return inline_native_currentCarrierThread();
472 case vmIntrinsics::_currentThread: return inline_native_currentThread();
473 case vmIntrinsics::_setCurrentThread: return inline_native_setCurrentThread();
474
475 case vmIntrinsics::_scopedValueCache: return inline_native_scopedValueCache();
476 case vmIntrinsics::_setScopedValueCache: return inline_native_setScopedValueCache();
477
478 #ifdef JFR_HAVE_INTRINSICS
479 case vmIntrinsics::_counterTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, JfrTime::time_function()), "counterTime");
480 case vmIntrinsics::_getEventWriter: return inline_native_getEventWriter();
481 #endif
482 case vmIntrinsics::_currentTimeMillis: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeMillis), "currentTimeMillis");
483 case vmIntrinsics::_nanoTime: return inline_native_time_funcs(CAST_FROM_FN_PTR(address, os::javaTimeNanos), "nanoTime");
484 case vmIntrinsics::_writeback0: return inline_unsafe_writeback0();
485 case vmIntrinsics::_writebackPreSync0: return inline_unsafe_writebackSync0(true);
486 case vmIntrinsics::_writebackPostSync0: return inline_unsafe_writebackSync0(false);
487 case vmIntrinsics::_allocateInstance: return inline_unsafe_allocate();
488 case vmIntrinsics::_copyMemory: return inline_unsafe_copyMemory();
489 case vmIntrinsics::_getLength: return inline_native_getLength();
490 case vmIntrinsics::_copyOf: return inline_array_copyOf(false);
491 case vmIntrinsics::_copyOfRange: return inline_array_copyOf(true);
492 case vmIntrinsics::_equalsB: return inline_array_equals(StrIntrinsicNode::LL);
493 case vmIntrinsics::_equalsC: return inline_array_equals(StrIntrinsicNode::UU);
494 case vmIntrinsics::_Preconditions_checkIndex: return inline_preconditions_checkIndex(T_INT);
495 case vmIntrinsics::_Preconditions_checkLongIndex: return inline_preconditions_checkIndex(T_LONG);
496 case vmIntrinsics::_clone: return inline_native_clone(intrinsic()->is_virtual());
3338 }
3339
3340 //------------------------inline_native_setVthread------------------
3341 bool LibraryCallKit::inline_native_setCurrentThread() {
3342 assert(C->method()->changes_current_thread(),
3343 "method changes current Thread but is not annotated ChangesCurrentThread");
3344 Node* arr = argument(1);
3345 Node* thread = _gvn.transform(new ThreadLocalNode());
3346 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::vthread_offset()));
3347 Node* thread_obj_handle
3348 = make_load(NULL, p, p->bottom_type()->is_ptr(), T_OBJECT, MemNode::unordered);
3349 thread_obj_handle = _gvn.transform(thread_obj_handle);
3350 const TypePtr *adr_type = _gvn.type(thread_obj_handle)->isa_ptr();
3351 // Stores of oops to native memory not supported yet by BarrierSetC2::store_at_resolved
3352 // access_store_at(NULL, thread_obj_handle, adr_type, arr, _gvn.type(arr), T_OBJECT, IN_NATIVE | MO_UNORDERED);
3353 store_to_memory(control(), thread_obj_handle, arr, T_OBJECT, adr_type, MemNode::unordered);
3354 JFR_ONLY(extend_setCurrentThread(thread, arr);)
3355 return true;
3356 }
3357
3358 Node* LibraryCallKit::scopedValueCache_helper() {
3359 ciKlass *objects_klass = ciObjArrayKlass::make(env()->Object_klass());
3360 const TypeOopPtr *etype = TypeOopPtr::make_from_klass(env()->Object_klass());
3361
3362 bool xk = etype->klass_is_exact();
3363
3364 Node* thread = _gvn.transform(new ThreadLocalNode());
3365 Node* p = basic_plus_adr(top()/*!oop*/, thread, in_bytes(JavaThread::scopedValueCache_offset()));
3366 // We cannot use immutable_memory() because we might flip onto a
3367 // different carrier thread, at which point we'll need to use that
3368 // carrier thread's cache.
3369 // return _gvn.transform(LoadNode::make(_gvn, NULL, immutable_memory(), p, p->bottom_type()->is_ptr(),
3370 // TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered));
3371 return make_load(NULL, p, p->bottom_type()->is_ptr(), T_ADDRESS, MemNode::unordered);
3372 }
3373
3374 //------------------------inline_native_scopedValueCache------------------
3375 bool LibraryCallKit::inline_native_scopedValueCache() {
3376 ciKlass *objects_klass = ciObjArrayKlass::make(env()->Object_klass());
3377 const TypeOopPtr *etype = TypeOopPtr::make_from_klass(env()->Object_klass());
3378 const TypeAry* arr0 = TypeAry::make(etype, TypeInt::POS);
3379
3380 // Because we create the scopedValue cache lazily we have to make the
3381 // type of the result BotPTR.
3382 bool xk = etype->klass_is_exact();
3383 const Type* objects_type = TypeAryPtr::make(TypePtr::BotPTR, arr0, objects_klass, xk, 0);
3384 Node* cache_obj_handle = scopedValueCache_helper();
3385 set_result(access_load(cache_obj_handle, objects_type, T_OBJECT, IN_NATIVE));
3386
3387 return true;
3388 }
3389
3390 //------------------------inline_native_setScopedValueCache------------------
3391 bool LibraryCallKit::inline_native_setScopedValueCache() {
3392 Node* arr = argument(0);
3393 Node* cache_obj_handle = scopedValueCache_helper();
3394
3395 const TypePtr *adr_type = _gvn.type(cache_obj_handle)->isa_ptr();
3396 store_to_memory(control(), cache_obj_handle, arr, T_OBJECT, adr_type,
3397 MemNode::unordered);
3398
3399 return true;
3400 }
3401
3402 //---------------------------load_mirror_from_klass----------------------------
3403 // Given a klass oop, load its java mirror (a java.lang.Class oop).
3404 Node* LibraryCallKit::load_mirror_from_klass(Node* klass) {
3405 Node* p = basic_plus_adr(klass, in_bytes(Klass::java_mirror_offset()));
3406 Node* load = make_load(NULL, p, TypeRawPtr::NOTNULL, T_ADDRESS, MemNode::unordered);
3407 // mirror = ((OopHandle)mirror)->resolve();
3408 return access_load(load, TypeInstPtr::MIRROR, T_OBJECT, IN_NATIVE);
3409 }
3410
3411 //-----------------------load_klass_from_mirror_common-------------------------
3412 // Given a java mirror (a java.lang.Class oop), load its corresponding klass oop.
3413 // Test the klass oop for null (signifying a primitive Class like Integer.TYPE),
|