< prev index next >

src/share/vm/opto/library_call.cpp

Print this page

        

*** 35,49 **** --- 35,51 ---- #include "opto/idealKit.hpp" #include "opto/mathexactnode.hpp" #include "opto/mulnode.hpp" #include "opto/parse.hpp" #include "opto/runtime.hpp" + #include "opto/shenandoahSupport.hpp" #include "opto/subnode.hpp" #include "prims/nativeLookup.hpp" #include "runtime/sharedRuntime.hpp" #include "trace/traceMacros.hpp" + class LibraryIntrinsic : public InlineCallGenerator { // Extend the set of intrinsics known to the runtime: public: private: bool _is_virtual;
*** 223,233 **** Node* finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName); bool inline_min_max(vmIntrinsics::ID id); Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y); // This returns Type::AnyPtr, RawPtr, or OopPtr. int classify_unsafe_addr(Node* &base, Node* &offset); ! Node* make_unsafe_address(Node* base, Node* offset); // Helper for inline_unsafe_access. // Generates the guards that check whether the result of // Unsafe.getObject should be recorded in an SATB log buffer. void insert_pre_barrier(Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar); bool inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile, bool is_unaligned); --- 225,235 ---- Node* finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName); bool inline_min_max(vmIntrinsics::ID id); Node* generate_min_max(vmIntrinsics::ID id, Node* x, Node* y); // This returns Type::AnyPtr, RawPtr, or OopPtr. int classify_unsafe_addr(Node* &base, Node* &offset); ! Node* make_unsafe_address(Node* base, Node* offset, bool is_store); // Helper for inline_unsafe_access. // Generates the guards that check whether the result of // Unsafe.getObject should be recorded in an SATB log buffer. void insert_pre_barrier(Node* base_oop, Node* offset, Node* pre_val, bool need_mem_bar); bool inline_unsafe_access(bool is_native_ptr, bool is_store, BasicType type, bool is_volatile, bool is_unaligned);
*** 328,337 **** --- 330,343 ---- bool inline_mulAdd(); bool inline_montgomeryMultiply(); bool inline_montgomerySquare(); bool inline_profileBoolean(); + + Node* shenandoah_cast_not_null(Node* n) { + return UseShenandoahGC ? cast_not_null(n, false) : n; + } }; //---------------------------make_vm_intrinsic---------------------------- CallGenerator* Compile::make_vm_intrinsic(ciMethod* m, bool is_virtual) {
*** 1160,1176 **** --- 1166,1186 ---- Node* LibraryCallKit::make_string_method_node(int opcode, Node* str1, Node* str2) { Node* no_ctrl = NULL; // Get start addr of string Node* str1_value = load_String_value(no_ctrl, str1); + str1_value = shenandoah_cast_not_null(str1_value); + str1_value = shenandoah_read_barrier(str1_value); Node* str1_offset = load_String_offset(no_ctrl, str1); Node* str1_start = array_element_address(str1_value, str1_offset, T_CHAR); // Get length of string 1 Node* str1_len = load_String_length(no_ctrl, str1); Node* str2_value = load_String_value(no_ctrl, str2); + str2_value = shenandoah_cast_not_null(str2_value); + str2_value = shenandoah_read_barrier(str2_value); Node* str2_offset = load_String_offset(no_ctrl, str2); Node* str2_start = array_element_address(str2_value, str2_offset, T_CHAR); Node* str2_len = NULL; Node* result = NULL;
*** 1260,1269 **** --- 1270,1281 ---- // paths (plus control) merge RegionNode* region = new (C) RegionNode(5); Node* phi = new (C) PhiNode(region, TypeInt::BOOL); // does source == target string? + receiver = shenandoah_write_barrier(receiver); + argument = shenandoah_write_barrier(argument); Node* cmp = _gvn.transform(new (C) CmpPNode(receiver, argument)); Node* bol = _gvn.transform(new (C) BoolNode(cmp, BoolTest::eq)); Node* if_eq = generate_slow_guard(bol, NULL); if (if_eq != NULL) {
*** 1299,1316 **** --- 1311,1332 ---- Node* no_ctrl = NULL; // Get start addr of receiver Node* receiver_val = load_String_value(no_ctrl, receiver); + receiver_val = shenandoah_cast_not_null(receiver_val); + receiver_val = shenandoah_read_barrier(receiver_val); Node* receiver_offset = load_String_offset(no_ctrl, receiver); Node* receiver_start = array_element_address(receiver_val, receiver_offset, T_CHAR); // Get length of receiver Node* receiver_cnt = load_String_length(no_ctrl, receiver); // Get start addr of argument Node* argument_val = load_String_value(no_ctrl, argument); + argument_val = shenandoah_cast_not_null(argument_val); + argument_val = shenandoah_read_barrier(argument_val); Node* argument_offset = load_String_offset(no_ctrl, argument); Node* argument_start = array_element_address(argument_val, argument_offset, T_CHAR); // Get length of argument Node* argument_cnt = load_String_length(no_ctrl, argument);
*** 1343,1352 **** --- 1359,1372 ---- //------------------------------inline_array_equals---------------------------- bool LibraryCallKit::inline_array_equals() { Node* arg1 = argument(0); Node* arg2 = argument(1); + + arg1 = shenandoah_read_barrier(arg1); + arg2 = shenandoah_read_barrier(arg2); + set_result(_gvn.transform(new (C) AryEqNode(control(), memory(TypeAryPtr::CHARS), arg1, arg2))); return true; } // Java version of String.indexOf(constant string)
*** 1418,1427 **** --- 1438,1449 ---- float unlikely = PROB_UNLIKELY(0.9); const int nargs = 0; // no arguments to push back for uncommon trap in predicate Node* source = load_String_value(no_ctrl, string_object); + source = shenandoah_cast_not_null(source); + source = shenandoah_read_barrier(source); Node* sourceOffset = load_String_offset(no_ctrl, string_object); Node* sourceCount = load_String_length(no_ctrl, string_object); Node* target = _gvn.transform( makecon(TypeOopPtr::make_from_constant(target_array, true))); jint target_length = target_array->length();
*** 1431,1440 **** --- 1453,1464 ---- // String.value field is known to be @Stable. if (UseImplicitStableValues) { target = cast_array_to_stable(target, target_type); } + target = shenandoah_read_barrier(target); + IdealKit kit(this, false, true); #define __ kit. Node* zero = __ ConI(0); Node* one = __ ConI(1); Node* cache = __ ConI(cache_i);
*** 1518,1535 **** --- 1542,1563 ---- Node* result_phi = new (C) PhiNode(result_rgn, TypeInt::INT); Node* no_ctrl = NULL; // Get start addr of source string Node* source = load_String_value(no_ctrl, receiver); + source = shenandoah_cast_not_null(source); + source = shenandoah_read_barrier(source); Node* source_offset = load_String_offset(no_ctrl, receiver); Node* source_start = array_element_address(source, source_offset, T_CHAR); // Get length of source string Node* source_cnt = load_String_length(no_ctrl, receiver); // Get start addr of substring Node* substr = load_String_value(no_ctrl, arg); + substr = shenandoah_cast_not_null(substr); + substr = shenandoah_read_barrier(substr); Node* substr_offset = load_String_offset(no_ctrl, arg); Node* substr_start = array_element_address(substr, substr_offset, T_CHAR); // Get length of source string Node* substr_cnt = load_String_length(no_ctrl, arg);
*** 1598,1607 **** --- 1626,1636 ---- if (o != 0 || c != pat->length()) { return false; } receiver = null_check(receiver, T_OBJECT); + receiver = shenandoah_read_barrier(receiver); // NOTE: No null check on the argument is needed since it's a constant String oop. if (stopped()) { return true; }
*** 2363,2377 **** // No information: return Type::AnyPtr; } } ! inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset) { int kind = classify_unsafe_addr(base, offset); if (kind == Type::RawPtr) { return basic_plus_adr(top(), base, offset); } else { return basic_plus_adr(base, offset); } } //--------------------------inline_number_methods----------------------------- --- 2392,2437 ---- // No information: return Type::AnyPtr; } } ! inline Node* LibraryCallKit::make_unsafe_address(Node* base, Node* offset, bool is_store) { int kind = classify_unsafe_addr(base, offset); if (kind == Type::RawPtr) { return basic_plus_adr(top(), base, offset); } else { + if (UseShenandoahGC) { + if (kind == Type::OopPtr) { + // A cast without a null check should be sufficient here (we + // know base is an oop with a low offset so it can't be null) + // but if there's a dominating null check with both branches + // taken and the cast is pushed in both branches, the cast + // will become top in the null branch but the control flow + // won't go away. Use a null check instead. Worst case, the + // null check becomes an implicit null check with the follow + // barrier and is essentially free. + Node* ctrl = top(); + base = null_check_oop(base, &ctrl, true); + if (is_store) { + base = shenandoah_write_barrier(base); + } else { + base = shenandoah_read_barrier(base); + } + } else if (kind == Type::AnyPtr) { + if (UseShenandoahGC && + _gvn.type(base)->isa_aryptr()) { + Node* ctrl = top(); + base = null_check_oop(base, &ctrl, true); + } + + if (is_store) { + base = shenandoah_write_barrier(base); + } else { + base = shenandoah_read_barrier(base); + } + } + } return basic_plus_adr(base, offset); } } //--------------------------inline_number_methods-----------------------------
*** 2419,2429 **** // is enabled, we need to log the value in the referent field in an SATB buffer. // This routine performs some compile time filters and generates suitable // runtime filters that guard the pre-barrier code. // Also add memory barrier for non volatile load from the referent field // to prevent commoning of loads across safepoint. ! if (!UseG1GC && !need_mem_bar) return; // Some compile time checks. // If offset is a constant, is it java_lang_ref_Reference::_reference_offset? --- 2479,2489 ---- // is enabled, we need to log the value in the referent field in an SATB buffer. // This routine performs some compile time filters and generates suitable // runtime filters that guard the pre-barrier code. // Also add memory barrier for non volatile load from the referent field // to prevent commoning of loads across safepoint. ! if (!(UseG1GC || UseShenandoahGC) && !need_mem_bar) return; // Some compile time checks. // If offset is a constant, is it java_lang_ref_Reference::_reference_offset?
*** 2618,2634 **** // by oopDesc::field_base. assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); // 32-bit machines ignore the high half! offset = ConvL2X(offset); ! adr = make_unsafe_address(base, offset); heap_base_oop = base; val = is_store ? argument(4) : NULL; } else { Node* ptr = argument(1); // type: long ptr = ConvL2X(ptr); // adjust Java long to machine word ! adr = make_unsafe_address(NULL, ptr); val = is_store ? argument(3) : NULL; } // Can base be NULL? Otherwise, always on-heap access. bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop)); --- 2678,2694 ---- // by oopDesc::field_base. assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); // 32-bit machines ignore the high half! offset = ConvL2X(offset); ! adr = make_unsafe_address(base, offset, is_store); heap_base_oop = base; val = is_store ? argument(4) : NULL; } else { Node* ptr = argument(1); // type: long ptr = ConvL2X(ptr); // adjust Java long to machine word ! adr = make_unsafe_address(NULL, ptr, is_store); val = is_store ? argument(3) : NULL; } // Can base be NULL? Otherwise, always on-heap access. bool can_access_non_heap = TypePtr::NULL_PTR->higher_equal(_gvn.type(heap_base_oop));
*** 2780,2789 **** --- 2840,2850 ---- (void) store_to_memory(control(), adr, val, type, adr_type, mo, is_volatile, unaligned, mismatched); } else { // Possibly an oop being stored to Java heap or native memory if (!can_access_non_heap) { // oop to Java heap. + val = shenandoah_read_barrier_storeval(val); (void) store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type, mo, mismatched); } else { // We can't tell at compile time if we are storing in the Java heap or outside // of it. So we need to emit code to conditionally do the proper type of // store.
*** 2792,2802 **** #define __ ideal. // QQQ who knows what probability is here?? __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); { // Sync IdealKit and graphKit. sync_kit(ideal); ! Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, val, type, mo, mismatched); // Update IdealKit memory. __ sync_kit(this); } __ else_(); { __ store(__ ctrl(), adr, val, type, alias_type->index(), mo, is_volatile, mismatched); } __ end_if(); --- 2853,2864 ---- #define __ ideal. // QQQ who knows what probability is here?? __ if_then(heap_base_oop, BoolTest::ne, null(), PROB_UNLIKELY(0.999)); { // Sync IdealKit and graphKit. sync_kit(ideal); ! Node* rb = shenandoah_read_barrier_storeval(val); ! Node* st = store_oop_to_unknown(control(), heap_base_oop, adr, adr_type, rb, type, mo, mismatched); // Update IdealKit memory. __ sync_kit(this); } __ else_(); { __ store(__ ctrl(), adr, val, type, alias_type->index(), mo, is_volatile, mismatched); } __ end_if();
*** 2867,2881 **** // by oopDesc::field_base. assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); // 32-bit machines ignore the high half! offset = ConvL2X(offset); ! adr = make_unsafe_address(base, offset); } else { Node* ptr = argument(idx + 0); // type: long ptr = ConvL2X(ptr); // adjust Java long to machine word ! adr = make_unsafe_address(NULL, ptr); } // Generate the read or write prefetch Node *prefetch; if (is_store) { --- 2929,2943 ---- // by oopDesc::field_base. assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); // 32-bit machines ignore the high half! offset = ConvL2X(offset); ! adr = make_unsafe_address(base, offset, false); } else { Node* ptr = argument(idx + 0); // type: long ptr = ConvL2X(ptr); // adjust Java long to machine word ! adr = make_unsafe_address(NULL, ptr, false); } // Generate the read or write prefetch Node *prefetch; if (is_store) {
*** 2974,2984 **** // to be plain byte offsets, which are also the same as those accepted // by oopDesc::field_base. assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); // 32-bit machines ignore the high half of long offsets offset = ConvL2X(offset); ! Node* adr = make_unsafe_address(base, offset); const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); Compile::AliasType* alias_type = C->alias_type(adr_type); BasicType bt = alias_type->basic_type(); if (bt != T_ILLEGAL && --- 3036,3046 ---- // to be plain byte offsets, which are also the same as those accepted // by oopDesc::field_base. assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); // 32-bit machines ignore the high half of long offsets offset = ConvL2X(offset); ! Node* adr = make_unsafe_address(base, offset, true); const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); Compile::AliasType* alias_type = C->alias_type(adr_type); BasicType bt = alias_type->basic_type(); if (bt != T_ILLEGAL &&
*** 3051,3060 **** --- 3113,3124 ---- // could be delayed during Parse (for example, in adjust_map_after_if()). // Execute transformation here to avoid barrier generation in such case. if (_gvn.type(newval) == TypePtr::NULL_PTR) newval = _gvn.makecon(TypePtr::NULL_PTR); + newval = shenandoah_read_barrier_storeval(newval); + // Reference stores need a store barrier. if (kind == LS_xchg) { // If pre-barrier must execute before the oop store, old value will require do_load here. if (!can_move_pre_barrier()) { pre_barrier(true /* do_load*/,
*** 3179,3201 **** // Build field offset expression. assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); // 32-bit machines ignore the high half of long offsets offset = ConvL2X(offset); ! Node* adr = make_unsafe_address(base, offset); const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); const Type *value_type = Type::get_const_basic_type(type); Compile::AliasType* alias_type = C->alias_type(adr_type); insert_mem_bar(Op_MemBarRelease); insert_mem_bar(Op_MemBarCPUOrder); // Ensure that the store is atomic for longs: const bool require_atomic_access = true; Node* store; ! if (type == T_OBJECT) // reference stores need a store barrier. store = store_oop_to_unknown(control(), base, adr, adr_type, val, type, MemNode::release); ! else { store = store_to_memory(control(), adr, val, type, adr_type, MemNode::release, require_atomic_access); } insert_mem_bar(Op_MemBarCPUOrder); return true; } --- 3243,3266 ---- // Build field offset expression. assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); // 32-bit machines ignore the high half of long offsets offset = ConvL2X(offset); ! Node* adr = make_unsafe_address(base, offset, true); const TypePtr *adr_type = _gvn.type(adr)->isa_ptr(); const Type *value_type = Type::get_const_basic_type(type); Compile::AliasType* alias_type = C->alias_type(adr_type); insert_mem_bar(Op_MemBarRelease); insert_mem_bar(Op_MemBarCPUOrder); // Ensure that the store is atomic for longs: const bool require_atomic_access = true; Node* store; ! if (type == T_OBJECT) { // reference stores need a store barrier. ! val = shenandoah_read_barrier_storeval(val); store = store_oop_to_unknown(control(), base, adr, adr_type, val, type, MemNode::release); ! } else { store = store_to_memory(control(), adr, val, type, adr_type, MemNode::release, require_atomic_access); } insert_mem_bar(Op_MemBarCPUOrder); return true; }
*** 3369,3378 **** --- 3434,3445 ---- // (a) Receiving thread must be the current thread. Node* rec_thr = argument(0); Node* tls_ptr = NULL; Node* cur_thr = generate_current_thread(tls_ptr); + cur_thr = shenandoah_write_barrier(cur_thr); + rec_thr = shenandoah_write_barrier(rec_thr); Node* cmp_thr = _gvn.transform(new (C) CmpPNode(cur_thr, rec_thr)); Node* bol_thr = _gvn.transform(new (C) BoolNode(cmp_thr, BoolTest::ne)); generate_slow_guard(bol_thr, slow_region);
*** 3714,3724 **** --- 3781,3793 ---- PATH_LIMIT }; RegionNode* region = new (C) RegionNode(PATH_LIMIT); Node* phi = new (C) PhiNode(region, TypeInt::BOOL); + Node* mem_phi= new (C) PhiNode(region, Type::MEMORY, TypePtr::BOTTOM); record_for_igvn(region); + Node* init_mem = map()->memory(); const TypePtr* adr_type = TypeRawPtr::BOTTOM; // memory type of loads const TypeKlassPtr* kls_type = TypeKlassPtr::OBJECT_OR_NULL; int class_klass_offset = java_lang_Class::klass_offset_in_bytes();
*** 3733,3742 **** --- 3802,3814 ---- Node* p = basic_plus_adr(arg, class_klass_offset); Node* kls = LoadKlassNode::make(_gvn, NULL, immutable_memory(), p, adr_type, kls_type); klasses[which_arg] = _gvn.transform(kls); } + args[0] = shenandoah_write_barrier(args[0]); + args[1] = shenandoah_write_barrier(args[1]); + // Having loaded both klasses, test each for null. bool never_see_null = !too_many_traps(Deoptimization::Reason_null_check); for (which_arg = 0; which_arg <= 1; which_arg++) { Node* kls = klasses[which_arg]; Node* null_ctl = top();
*** 3777,3798 **** phi->set_req(_prim_same_path, intcon(1)); phi->set_req(_ref_subtype_path, intcon(1)); // pull together the cases: assert(region->req() == PATH_LIMIT, "sane region"); for (uint i = 1; i < region->req(); i++) { Node* ctl = region->in(i); if (ctl == NULL || ctl == top()) { region->set_req(i, top()); phi ->set_req(i, top()); ! } else if (phi->in(i) == NULL) { phi->set_req(i, intcon(0)); // all other paths produce 'false' } } set_control(_gvn.transform(region)); set_result(_gvn.transform(phi)); return true; } //---------------------generate_array_guard_common------------------------ Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region, --- 3849,3876 ---- phi->set_req(_prim_same_path, intcon(1)); phi->set_req(_ref_subtype_path, intcon(1)); // pull together the cases: assert(region->req() == PATH_LIMIT, "sane region"); + Node* cur_mem = reset_memory(); for (uint i = 1; i < region->req(); i++) { Node* ctl = region->in(i); if (ctl == NULL || ctl == top()) { region->set_req(i, top()); phi ->set_req(i, top()); ! mem_phi->set_req(i, top()); ! } else { ! if (phi->in(i) == NULL) { phi->set_req(i, intcon(0)); // all other paths produce 'false' } + mem_phi->set_req(i, (i == _prim_0_path || i == _prim_same_path) ? cur_mem : init_mem); + } } set_control(_gvn.transform(region)); set_result(_gvn.transform(phi)); + set_all_memory(_gvn.transform(mem_phi)); return true; } //---------------------generate_array_guard_common------------------------ Node* LibraryCallKit::generate_array_guard_common(Node* kls, RegionNode* region,
*** 4003,4012 **** --- 4081,4092 ---- Node* orig_tail = _gvn.transform(new (C) SubINode(orig_length, start)); Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length); newcopy = new_array(klass_node, length, 0); // no argments to push + original = shenandoah_read_barrier(original); + // Generate a direct call to the right arraycopy function(s). // We know the copy is disjoint but we might not know if the // oop stores need checking. // Extreme case: Arrays.copyOf((Integer[])x, 10, String[].class). // This will fail a store-check if x contains any non-nulls.
*** 4450,4461 **** Node* size = ConvL2X(argument(7)); // type: long assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); ! Node* src = make_unsafe_address(src_ptr, src_off); ! Node* dst = make_unsafe_address(dst_ptr, dst_off); // Conservatively insert a memory barrier on all memory slices. // Do not let writes of the copy source or destination float below the copy. insert_mem_bar(Op_MemBarCPUOrder); --- 4530,4541 ---- Node* size = ConvL2X(argument(7)); // type: long assert(Unsafe_field_offset_to_byte_offset(11) == 11, "fieldOffset must be byte-scaled"); ! Node* src = make_unsafe_address(src_ptr, src_off, false); ! Node* dst = make_unsafe_address(dst_ptr, dst_off, true); // Conservatively insert a memory barrier on all memory slices. // Do not let writes of the copy source or destination float below the copy. insert_mem_bar(Op_MemBarCPUOrder);
*** 4478,4487 **** --- 4558,4569 ---- void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) { assert(obj_size != NULL, ""); Node* raw_obj = alloc_obj->in(1); assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); + obj = shenandoah_read_barrier(obj); + AllocateNode* alloc = NULL; if (ReduceBulkZeroing) { // We will be completely responsible for initializing this object - // mark Initialize node as complete. alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn);
*** 4530,4539 **** --- 4612,4630 ---- bool disjoint_bases = true; generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, src, NULL, dest, NULL, countx, /*dest_uninitialized*/true); + if (UseShenandoahGC && ShenandoahCloneBarrier) { + // Make sure that references in the cloned object are updated for Shenandoah. + make_runtime_call(RC_LEAF|RC_NO_FP, + OptoRuntime::shenandoah_clone_barrier_Type(), + CAST_FROM_FN_PTR(address, SharedRuntime::shenandoah_clone_barrier), + "shenandoah_clone_barrier", TypePtr::BOTTOM, + alloc_obj); + } + // If necessary, emit some card marks afterwards. (Non-arrays only.) if (card_mark) { assert(!is_array, ""); // Put in store barrier for any and all oops we are sticking // into this object. (We could avoid this if we could prove
*** 4638,4647 **** --- 4729,4741 ---- // because card marking is required on each card of the array. Node* is_obja = generate_objArray_guard(obj_klass, (RegionNode*)NULL); if (is_obja != NULL) { PreserveJVMState pjvms2(this); set_control(is_obja); + + obj = shenandoah_read_barrier(obj); + // Generate a direct call to the right arraycopy function(s). bool disjoint_bases = true; bool length_never_negative = true; generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, obj, intcon(0), alloc_obj, intcon(0),
*** 4855,4864 **** --- 4949,4961 ---- if (!has_src || !has_dest) { // Conservatively insert a memory barrier on all memory slices. // Do not let writes into the source float below the arraycopy. insert_mem_bar(Op_MemBarCPUOrder); + src = shenandoah_read_barrier(src); + dest = shenandoah_write_barrier(dest); + // Call StubRoutines::generic_arraycopy stub. generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT, src, src_offset, dest, dest_offset, length); // Do not let reads from the destination float above the arraycopy.
*** 4879,4888 **** --- 4976,4989 ---- if (dest_elem == T_ARRAY) dest_elem = T_OBJECT; if (src_elem != dest_elem || dest_elem == T_VOID) { // The component types are not the same or are not recognized. Punt. // (But, avoid the native method wrapper to JVM_ArrayCopy.) + + src = shenandoah_read_barrier(src); + dest = shenandoah_write_barrier(dest); + generate_slow_arraycopy(TypePtr::BOTTOM, src, src_offset, dest, dest_offset, length, /*dest_uninitialized*/false); return true; }
*** 4944,4953 **** --- 5045,5057 ---- // This should be corrected, given time. We do our null check with the // stack pointer restored. src = null_check(src, T_ARRAY); dest = null_check(dest, T_ARRAY); + src = shenandoah_read_barrier(src); + dest = shenandoah_write_barrier(dest); + // (4) src_offset must not be negative. generate_negative_guard(src_offset, slow_region); // (5) dest_offset must not be negative. generate_negative_guard(dest_offset, slow_region);
*** 5260,5270 **** checked_value = cv; } // At this point we know we do not need type checks on oop stores. // Let's see if we need card marks: ! if (alloc != NULL && use_ReduceInitialCardMarks()) { // If we do not need card marks, copy using the jint or jlong stub. copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT); assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type), "sizes agree"); } --- 5364,5374 ---- checked_value = cv; } // At this point we know we do not need type checks on oop stores. // Let's see if we need card marks: ! if (alloc != NULL && use_ReduceInitialCardMarks() && ! UseShenandoahGC) { // If we do not need card marks, copy using the jint or jlong stub. copy_type = LP64_ONLY(UseCompressedOops ? T_INT : T_LONG) NOT_LP64(T_INT); assert(type2aelembytes(basic_elem_type) == type2aelembytes(copy_type), "sizes agree"); }
*** 5412,5421 **** --- 5516,5527 ---- LibraryCallKit::tightly_coupled_allocation(Node* ptr, RegionNode* slow_region) { if (stopped()) return NULL; // no fast path if (C->AliasLevel() == 0) return NULL; // no MergeMems around + ptr = ShenandoahBarrierNode::skip_through_barrier(ptr); + AllocateArrayNode* alloc = AllocateArrayNode::Ideal_array_allocation(ptr, &_gvn); if (alloc == NULL) return NULL; Node* rawmem = memory(Compile::AliasIdxRaw); // Is the allocation's memory state untouched?
*** 5790,5799 **** --- 5896,5911 ---- Node *src_offset = argument(1); Node *dst = argument(2); Node *dst_offset = argument(3); Node *length = argument(4); + src = shenandoah_cast_not_null(src); + dst = shenandoah_cast_not_null(dst); + + src = shenandoah_read_barrier(src); + dst = shenandoah_write_barrier(dst); + const Type* src_type = src->Value(&_gvn); const Type* dst_type = dst->Value(&_gvn); const TypeAryPtr* top_src = src_type->isa_aryptr(); const TypeAryPtr* top_dest = dst_type->isa_aryptr(); if (top_src == NULL || top_src->klass() == NULL ||
*** 5839,5848 **** --- 5951,5966 ---- Node* xlen = argument(1); Node* y = argument(2); Node* ylen = argument(3); Node* z = argument(4); + x = shenandoah_cast_not_null(x); + x = shenandoah_read_barrier(x); + y = shenandoah_cast_not_null(y); + y = shenandoah_read_barrier(y); + z = shenandoah_write_barrier(z); + const Type* x_type = x->Value(&_gvn); const Type* y_type = y->Value(&_gvn); const TypeAryPtr* top_x = x_type->isa_aryptr(); const TypeAryPtr* top_y = y_type->isa_aryptr(); if (top_x == NULL || top_x->klass() == NULL ||
*** 5884,5894 **** __ if_then(z, BoolTest::eq, null()); { __ increment (need_alloc, one); } __ else_(); { // Update graphKit memory and control from IdealKit. sync_kit(ideal); ! Node* zlen_arg = load_array_length(z); // Update IdealKit memory and control from graphKit. __ sync_kit(this); __ if_then(zlen_arg, BoolTest::lt, zlen); { __ increment (need_alloc, one); } __ end_if(); --- 6002,6021 ---- __ if_then(z, BoolTest::eq, null()); { __ increment (need_alloc, one); } __ else_(); { // Update graphKit memory and control from IdealKit. sync_kit(ideal); ! Node* zlen_arg = NULL; ! if (UseShenandoahGC) { ! Node *cast = new (C) CastPPNode(z, TypePtr::NOTNULL); ! cast->init_req(0, control()); ! _gvn.set_type(cast, cast->bottom_type()); ! C->record_for_igvn(cast); ! zlen_arg = load_array_length(cast); ! } else { ! zlen_arg = load_array_length(z); ! } // Update IdealKit memory and control from graphKit. __ sync_kit(this); __ if_then(zlen_arg, BoolTest::lt, zlen); { __ increment (need_alloc, one); } __ end_if();
*** 5939,5948 **** --- 6066,6080 ---- Node* x = argument(0); Node* len = argument(1); Node* z = argument(2); Node* zlen = argument(3); + x = shenandoah_cast_not_null(x); + x = shenandoah_read_barrier(x); + z = shenandoah_cast_not_null(z); + z = shenandoah_write_barrier(z); + const Type* x_type = x->Value(&_gvn); const Type* z_type = z->Value(&_gvn); const TypeAryPtr* top_x = x_type->isa_aryptr(); const TypeAryPtr* top_z = z_type->isa_aryptr(); if (top_x == NULL || top_x->klass() == NULL ||
*** 5986,5995 **** --- 6118,6131 ---- Node* in = argument(1); Node* offset = argument(2); Node* len = argument(3); Node* k = argument(4); + in = shenandoah_read_barrier(in); + out = shenandoah_cast_not_null(out); + out = shenandoah_write_barrier(out); + const Type* out_type = out->Value(&_gvn); const Type* in_type = in->Value(&_gvn); const TypeAryPtr* top_out = out_type->isa_aryptr(); const TypeAryPtr* top_in = in_type->isa_aryptr(); if (top_out == NULL || top_out->klass() == NULL ||
*** 6035,6044 **** --- 6171,6185 ---- Node* n = argument(2); Node* len = argument(3); Node* inv = argument(4); Node* m = argument(6); + a = shenandoah_read_barrier(a); + b = shenandoah_read_barrier(b); + n = shenandoah_read_barrier(n); + m = shenandoah_write_barrier(m); + const Type* a_type = a->Value(&_gvn); const TypeAryPtr* top_a = a_type->isa_aryptr(); const Type* b_type = b->Value(&_gvn); const TypeAryPtr* top_b = b_type->isa_aryptr(); const Type* n_type = a->Value(&_gvn);
*** 6104,6113 **** --- 6245,6258 ---- Node* n = argument(1); Node* len = argument(2); Node* inv = argument(3); Node* m = argument(5); + a = shenandoah_read_barrier(a); + n = shenandoah_read_barrier(n); + m = shenandoah_write_barrier(m); + const Type* a_type = a->Value(&_gvn); const TypeAryPtr* top_a = a_type->isa_aryptr(); const Type* n_type = a->Value(&_gvn); const TypeAryPtr* top_n = n_type->isa_aryptr(); const Type* m_type = a->Value(&_gvn);
*** 6215,6224 **** --- 6360,6372 ---- if (src_elem != T_BYTE) { return false; } // 'src_start' points to src array + scaled offset + src = shenandoah_cast_not_null(src); + src = shenandoah_read_barrier(src); + src = shenandoah_read_barrier(src); Node* src_start = array_element_address(src, offset, src_elem); // We assume that range check is done by caller. // TODO: generate range check (offset+length < src.length) in debug VM.
*** 6312,6321 **** --- 6460,6477 ---- ciSymbol::make(fieldTypeString), is_static); if (field == NULL) return (Node *) NULL; assert (field != NULL, "undefined field"); + if ((ShenandoahOptimizeStaticFinals && field->is_static() && field->is_final()) || + (ShenandoahOptimizeInstanceFinals && !field->is_static() && field->is_final()) || + (ShenandoahOptimizeStableFinals && field->is_stable())) { + // Skip the barrier for special fields + } else { + fromObj = shenandoah_read_barrier(fromObj); + } + // Next code copied from Parse::do_get_xxx(): // Compute address and memory type. int offset = field->offset_in_bytes(); bool is_vol = field->is_volatile();
*** 6372,6381 **** --- 6528,6543 ---- Node* src = argument(1); Node* src_offset = argument(2); Node* dest = argument(3); Node* dest_offset = argument(4); + // Resolve src and dest arrays for ShenandoahGC. + src = shenandoah_cast_not_null(src); + src = shenandoah_read_barrier(src); + dest = shenandoah_cast_not_null(dest); + dest = shenandoah_write_barrier(dest); + // (1) src and dest are arrays. const Type* src_type = src->Value(&_gvn); const Type* dest_type = dest->Value(&_gvn); const TypeAryPtr* top_src = src_type->isa_aryptr(); const TypeAryPtr* top_dest = dest_type->isa_aryptr();
*** 6440,6449 **** --- 6602,6618 ---- Node* src_offset = argument(2); Node* len = argument(3); Node* dest = argument(4); Node* dest_offset = argument(5); + // inline_cipherBlockChaining_AESCrypt_predicate() has its own + // barrier. This one should optimize away. + src = shenandoah_cast_not_null(src); + dest = shenandoah_cast_not_null(dest); + src = shenandoah_read_barrier(src); + dest = shenandoah_write_barrier(dest); + // (1) src and dest are arrays. const Type* src_type = src->Value(&_gvn); const Type* dest_type = dest->Value(&_gvn); const TypeAryPtr* top_src = src_type->isa_aryptr(); const TypeAryPtr* top_dest = dest_type->isa_aryptr();
*** 6484,6493 **** --- 6653,6665 ---- Node* k_start = get_key_start_from_aescrypt_object(aescrypt_object); if (k_start == NULL) return false; // similarly, get the start address of the r vector Node* objRvec = load_field_from_object(cipherBlockChaining_object, "r", "[B", /*is_exact*/ false); + + objRvec = shenandoah_write_barrier(objRvec); + if (objRvec == NULL) return false; Node* r_start = array_element_address(objRvec, intcon(0), T_BYTE); Node* cbcCrypt; if (Matcher::pass_original_key_for_aes()) {
*** 6532,6541 **** --- 6704,6715 ---- Node* objAESCryptKey = load_field_from_object(aescrypt_object, "K", "[I", /*is_exact*/ false); #endif // PPC64 assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt"); if (objAESCryptKey == NULL) return (Node *) NULL; + objAESCryptKey = shenandoah_read_barrier(objAESCryptKey); + // now have the array, need to get the start address of the K array Node* k_start = array_element_address(objAESCryptKey, intcon(0), T_INT); return k_start; }
*** 6543,6552 **** --- 6717,6728 ---- Node * LibraryCallKit::get_original_key_start_from_aescrypt_object(Node *aescrypt_object) { Node* objAESCryptKey = load_field_from_object(aescrypt_object, "lastKey", "[B", /*is_exact*/ false); assert (objAESCryptKey != NULL, "wrong version of com.sun.crypto.provider.AESCrypt"); if (objAESCryptKey == NULL) return (Node *) NULL; + objAESCryptKey = shenandoah_read_barrier(objAESCryptKey); + // now have the array, need to get the start address of the lastKey array Node* original_k_start = array_element_address(objAESCryptKey, intcon(0), T_BYTE); return original_k_start; }
*** 6561,6570 **** --- 6737,6749 ---- // Node* LibraryCallKit::inline_cipherBlockChaining_AESCrypt_predicate(bool decrypting) { // The receiver was checked for NULL already. Node* objCBC = argument(0); + Node* src = argument(1); + Node* dest = argument(4); + // Load embeddedCipher field of CipherBlockChaining object. Node* embeddedCipherObj = load_field_from_object(objCBC, "embeddedCipher", "Lcom/sun/crypto/provider/SymmetricCipher;", /*is_exact*/ false); // get AESCrypt klass for instanceOf check // AESCrypt might not be loaded yet if some other SymmetricCipher got us to this compile point
*** 6579,6588 **** --- 6758,6776 ---- // if AESCrypt is not even loaded, we never take the intrinsic fast path Node* ctrl = control(); set_control(top()); // no regular fast path return ctrl; } + + // Resolve src and dest arrays for ShenandoahGC. Here because new + // memory state is not handled by predicate logic in + // inline_cipherBlockChaining_AESCrypt itself + src = shenandoah_cast_not_null(src); + dest = shenandoah_cast_not_null(dest); + src = shenandoah_write_barrier(src); + dest = shenandoah_write_barrier(dest); + ciInstanceKlass* instklass_AESCrypt = klass_AESCrypt->as_instance_klass(); Node* instof = gen_instanceof(embeddedCipherObj, makecon(TypeKlassPtr::make(instklass_AESCrypt))); Node* cmp_instof = _gvn.transform(new (C) CmpINode(instof, intcon(1))); Node* bool_instof = _gvn.transform(new (C) BoolNode(cmp_instof, BoolTest::ne));
*** 6596,6607 **** // for decryption, we need to add a further check to avoid // taking the intrinsic path when cipher and plain are the same // see the original java code for why. RegionNode* region = new(C) RegionNode(3); region->init_req(1, instof_false); ! Node* src = argument(1); ! Node* dest = argument(4); Node* cmp_src_dest = _gvn.transform(new (C) CmpPNode(src, dest)); Node* bool_src_dest = _gvn.transform(new (C) BoolNode(cmp_src_dest, BoolTest::eq)); Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN); region->init_req(2, src_dest_conjoint); --- 6784,6794 ---- // for decryption, we need to add a further check to avoid // taking the intrinsic path when cipher and plain are the same // see the original java code for why. RegionNode* region = new(C) RegionNode(3); region->init_req(1, instof_false); ! Node* cmp_src_dest = _gvn.transform(new (C) CmpPNode(src, dest)); Node* bool_src_dest = _gvn.transform(new (C) BoolNode(cmp_src_dest, BoolTest::eq)); Node* src_dest_conjoint = generate_guard(bool_src_dest, NULL, PROB_MIN); region->init_req(2, src_dest_conjoint);
*** 6637,6646 **** --- 6824,6835 ---- BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); if (src_elem != T_BYTE) { return false; } // 'src_start' points to src array + offset + src = cast_not_null(src, false); + src = shenandoah_read_barrier(src); Node* src_start = array_element_address(src, ofs, src_elem); Node* state = NULL; address stubAddr; const char *stubName;
*** 6703,6712 **** --- 6892,6903 ---- BasicType src_elem = src_type->isa_aryptr()->klass()->as_array_klass()->element_type()->basic_type(); if (src_elem != T_BYTE) { return false; } // 'src_start' points to src array + offset + src = shenandoah_cast_not_null(src); + src = shenandoah_read_barrier(src); Node* src_start = array_element_address(src, ofs, src_elem); const char* klass_SHA_name = NULL; const char* stub_name = NULL; address stub_addr = NULL;
*** 6784,6793 **** --- 6975,6986 ---- Node * LibraryCallKit::get_state_from_sha_object(Node *sha_object) { Node* sha_state = load_field_from_object(sha_object, "state", "[I", /*is_exact*/ false); assert (sha_state != NULL, "wrong version of sun.security.provider.SHA/SHA2"); if (sha_state == NULL) return (Node *) NULL; + sha_state = shenandoah_write_barrier(sha_state); + // now have the array, need to get the start address of the state array Node* state = array_element_address(sha_state, intcon(0), T_INT); return state; }
*** 6795,6804 **** --- 6988,6999 ---- Node * LibraryCallKit::get_state_from_sha5_object(Node *sha_object) { Node* sha_state = load_field_from_object(sha_object, "state", "[J", /*is_exact*/ false); assert (sha_state != NULL, "wrong version of sun.security.provider.SHA5"); if (sha_state == NULL) return (Node *) NULL; + sha_state = shenandoah_write_barrier(sha_state); + // now have the array, need to get the start address of the state array Node* state = array_element_address(sha_state, intcon(0), T_LONG); return state; }
< prev index next >