3031 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
3032 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3033 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3034 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3035 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
3036 }
3037
3038
3039 // This code replaces a call to arraycopy; no exception may
3040 // be thrown in this code, they must be thrown in the System.arraycopy
3041 // activation frame; we could save some checks if this would not be the case
3042 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3043 ciArrayKlass* default_type = op->expected_type();
3044 Register src = op->src()->as_register();
3045 Register dst = op->dst()->as_register();
3046 Register src_pos = op->src_pos()->as_register();
3047 Register dst_pos = op->dst_pos()->as_register();
3048 Register length = op->length()->as_register();
3049 Register tmp = op->tmp()->as_register();
3050 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3051
3052 CodeStub* stub = op->stub();
3053 int flags = op->flags();
3054 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
3055 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
3056
3057 // if we don't know anything, just go through the generic arraycopy
3058 if (default_type == nullptr) {
3059 // save outgoing arguments on stack in case call to System.arraycopy is needed
3060 // HACK ALERT. This code used to push the parameters in a hardwired fashion
3061 // for interpreter calling conventions. Now we have to do it in new style conventions.
3062 // For the moment until C1 gets the new register allocator I just force all the
3063 // args to the right place (except the register args) and then on the back side
3064 // reload the register args properly if we go slow path. Yuck
3065
3066 // These are proper for the calling convention
3067 store_parameter(length, 2);
3068 store_parameter(dst_pos, 1);
3069 store_parameter(dst, 0);
3070
3155 switch (elem_size) {
3156 case 1 :
3157 scale = Address::times_1;
3158 break;
3159 case 2 :
3160 scale = Address::times_2;
3161 break;
3162 case 4 :
3163 scale = Address::times_4;
3164 break;
3165 case 8 :
3166 scale = Address::times_8;
3167 break;
3168 default:
3169 scale = Address::no_scale;
3170 ShouldNotReachHere();
3171 }
3172
3173 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
3174 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
3175 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
3176 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
3177
3178 // length and pos's are all sign extended at this point on 64bit
3179
3180 // test for null
3181 if (flags & LIR_OpArrayCopy::src_null_check) {
3182 __ testptr(src, src);
3183 __ jcc(Assembler::zero, *stub->entry());
3184 }
3185 if (flags & LIR_OpArrayCopy::dst_null_check) {
3186 __ testptr(dst, dst);
3187 __ jcc(Assembler::zero, *stub->entry());
3188 }
3189
3190 // If the compiler was not able to prove that exact type of the source or the destination
3191 // of the arraycopy is an array type, check at runtime if the source or the destination is
3192 // an instance type.
3193 if (flags & LIR_OpArrayCopy::type_check) {
3194 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3195 __ load_klass(tmp, dst, tmp_load_klass);
3196 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
3222 if (flags & LIR_OpArrayCopy::dst_range_check) {
3223 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
3224 __ cmpl(tmp, dst_length_addr);
3225 __ jcc(Assembler::above, *stub->entry());
3226 }
3227
3228 if (flags & LIR_OpArrayCopy::length_positive_check) {
3229 __ testl(length, length);
3230 __ jcc(Assembler::less, *stub->entry());
3231 }
3232
3233 #ifdef _LP64
3234 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
3235 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
3236 #endif
3237
3238 if (flags & LIR_OpArrayCopy::type_check) {
3239 // We don't know the array types are compatible
3240 if (basic_type != T_OBJECT) {
3241 // Simple test for basic type arrays
3242 if (UseCompressedClassPointers) {
3243 __ movl(tmp, src_klass_addr);
3244 __ cmpl(tmp, dst_klass_addr);
3245 } else {
3246 __ movptr(tmp, src_klass_addr);
3247 __ cmpptr(tmp, dst_klass_addr);
3248 }
3249 __ jcc(Assembler::notEqual, *stub->entry());
3250 } else {
3251 // For object arrays, if src is a sub class of dst then we can
3252 // safely do the copy.
3253 Label cont, slow;
3254
3255 __ push(src);
3256 __ push(dst);
3257
3258 __ load_klass(src, src, tmp_load_klass);
3259 __ load_klass(dst, dst, tmp_load_klass);
3260
3261 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
3262
3263 __ push(src);
3264 __ push(dst);
3265 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
3266 __ pop(dst);
3267 __ pop(src);
3268
3287 __ load_klass(tmp, src, tmp_load_klass);
3288 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3289 __ load_klass(tmp, dst, tmp_load_klass);
3290 }
3291 int lh_offset = in_bytes(Klass::layout_helper_offset());
3292 Address klass_lh_addr(tmp, lh_offset);
3293 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
3294 __ cmpl(klass_lh_addr, objArray_lh);
3295 __ jcc(Assembler::notEqual, *stub->entry());
3296 }
3297
3298 // Spill because stubs can use any register they like and it's
3299 // easier to restore just those that we care about.
3300 store_parameter(dst, 0);
3301 store_parameter(dst_pos, 1);
3302 store_parameter(length, 2);
3303 store_parameter(src_pos, 3);
3304 store_parameter(src, 4);
3305
3306 #ifndef _LP64
3307 __ movptr(tmp, dst_klass_addr);
3308 __ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset()));
3309 __ push(tmp);
3310 __ movl(tmp, Address(tmp, Klass::super_check_offset_offset()));
3311 __ push(tmp);
3312 __ push(length);
3313 __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3314 __ push(tmp);
3315 __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3316 __ push(tmp);
3317
3318 __ call_VM_leaf(copyfunc_addr, 5);
3319 #else
3320 __ movl2ptr(length, length); //higher 32bits must be null
3321
3322 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3323 assert_different_registers(c_rarg0, dst, dst_pos, length);
3324 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3325 assert_different_registers(c_rarg1, dst, length);
3326
3390 }
3391
3392 #ifdef ASSERT
3393 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
3394 // Sanity check the known type with the incoming class. For the
3395 // primitive case the types must match exactly with src.klass and
3396 // dst.klass each exactly matching the default type. For the
3397 // object array case, if no type check is needed then either the
3398 // dst type is exactly the expected type and the src type is a
3399 // subtype which we can't check or src is the same array as dst
3400 // but not necessarily exactly of type default_type.
3401 Label known_ok, halt;
3402 __ mov_metadata(tmp, default_type->constant_encoding());
3403 #ifdef _LP64
3404 if (UseCompressedClassPointers) {
3405 __ encode_klass_not_null(tmp, rscratch1);
3406 }
3407 #endif
3408
3409 if (basic_type != T_OBJECT) {
3410
3411 if (UseCompressedClassPointers) __ cmpl(tmp, dst_klass_addr);
3412 else __ cmpptr(tmp, dst_klass_addr);
3413 __ jcc(Assembler::notEqual, halt);
3414 if (UseCompressedClassPointers) __ cmpl(tmp, src_klass_addr);
3415 else __ cmpptr(tmp, src_klass_addr);
3416 __ jcc(Assembler::equal, known_ok);
3417 } else {
3418 if (UseCompressedClassPointers) __ cmpl(tmp, dst_klass_addr);
3419 else __ cmpptr(tmp, dst_klass_addr);
3420 __ jcc(Assembler::equal, known_ok);
3421 __ cmpptr(src, dst);
3422 __ jcc(Assembler::equal, known_ok);
3423 }
3424 __ bind(halt);
3425 __ stop("incorrect type information in arraycopy");
3426 __ bind(known_ok);
3427 }
3428 #endif
3429
3430 #ifndef PRODUCT
3431 if (PrintC1Statistics) {
3432 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
3433 }
3434 #endif
3435
3436 #ifdef _LP64
3437 assert_different_registers(c_rarg0, dst, dst_pos, length);
3438 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3439 assert_different_registers(c_rarg1, length);
3497 // done
3498 } else if (op->code() == lir_unlock) {
3499 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3500 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3501 } else {
3502 Unimplemented();
3503 }
3504 __ bind(*op->stub()->continuation());
3505 }
3506
3507 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
3508 Register obj = op->obj()->as_pointer_register();
3509 Register result = op->result_opr()->as_pointer_register();
3510
3511 CodeEmitInfo* info = op->info();
3512 if (info != nullptr) {
3513 add_debug_info_for_null_check_here(info);
3514 }
3515
3516 #ifdef _LP64
3517 if (UseCompressedClassPointers) {
3518 __ movl(result, Address(obj, oopDesc::klass_offset_in_bytes()));
3519 __ decode_klass_not_null(result, rscratch1);
3520 } else
3521 #endif
3522 __ movptr(result, Address(obj, oopDesc::klass_offset_in_bytes()));
3523 }
3524
3525 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3526 ciMethod* method = op->profiled_method();
3527 int bci = op->profiled_bci();
3528 ciMethod* callee = op->profiled_callee();
3529 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3530
3531 // Update counter for all call types
3532 ciMethodData* md = method->method_data_or_null();
3533 assert(md != nullptr, "Sanity");
3534 ciProfileData* data = md->bci_to_data(bci);
3535 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
3536 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
3537 Register mdo = op->mdo()->as_register();
3538 __ mov_metadata(mdo, md->constant_encoding());
3539 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
3540 // Perform additional virtual call profiling for invokevirtual and
3541 // invokeinterface bytecodes
3542 if (op->should_profile_receiver_type()) {
|
3031 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
3032 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3033 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3034 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3035 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
3036 }
3037
3038
3039 // This code replaces a call to arraycopy; no exception may
3040 // be thrown in this code, they must be thrown in the System.arraycopy
3041 // activation frame; we could save some checks if this would not be the case
3042 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3043 ciArrayKlass* default_type = op->expected_type();
3044 Register src = op->src()->as_register();
3045 Register dst = op->dst()->as_register();
3046 Register src_pos = op->src_pos()->as_register();
3047 Register dst_pos = op->dst_pos()->as_register();
3048 Register length = op->length()->as_register();
3049 Register tmp = op->tmp()->as_register();
3050 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3051 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
3052
3053 CodeStub* stub = op->stub();
3054 int flags = op->flags();
3055 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
3056 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
3057
3058 // if we don't know anything, just go through the generic arraycopy
3059 if (default_type == nullptr) {
3060 // save outgoing arguments on stack in case call to System.arraycopy is needed
3061 // HACK ALERT. This code used to push the parameters in a hardwired fashion
3062 // for interpreter calling conventions. Now we have to do it in new style conventions.
3063 // For the moment until C1 gets the new register allocator I just force all the
3064 // args to the right place (except the register args) and then on the back side
3065 // reload the register args properly if we go slow path. Yuck
3066
3067 // These are proper for the calling convention
3068 store_parameter(length, 2);
3069 store_parameter(dst_pos, 1);
3070 store_parameter(dst, 0);
3071
3156 switch (elem_size) {
3157 case 1 :
3158 scale = Address::times_1;
3159 break;
3160 case 2 :
3161 scale = Address::times_2;
3162 break;
3163 case 4 :
3164 scale = Address::times_4;
3165 break;
3166 case 8 :
3167 scale = Address::times_8;
3168 break;
3169 default:
3170 scale = Address::no_scale;
3171 ShouldNotReachHere();
3172 }
3173
3174 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
3175 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
3176
3177 // length and pos's are all sign extended at this point on 64bit
3178
3179 // test for null
3180 if (flags & LIR_OpArrayCopy::src_null_check) {
3181 __ testptr(src, src);
3182 __ jcc(Assembler::zero, *stub->entry());
3183 }
3184 if (flags & LIR_OpArrayCopy::dst_null_check) {
3185 __ testptr(dst, dst);
3186 __ jcc(Assembler::zero, *stub->entry());
3187 }
3188
3189 // If the compiler was not able to prove that exact type of the source or the destination
3190 // of the arraycopy is an array type, check at runtime if the source or the destination is
3191 // an instance type.
3192 if (flags & LIR_OpArrayCopy::type_check) {
3193 if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3194 __ load_klass(tmp, dst, tmp_load_klass);
3195 __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
3221 if (flags & LIR_OpArrayCopy::dst_range_check) {
3222 __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
3223 __ cmpl(tmp, dst_length_addr);
3224 __ jcc(Assembler::above, *stub->entry());
3225 }
3226
3227 if (flags & LIR_OpArrayCopy::length_positive_check) {
3228 __ testl(length, length);
3229 __ jcc(Assembler::less, *stub->entry());
3230 }
3231
3232 #ifdef _LP64
3233 __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
3234 __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
3235 #endif
3236
3237 if (flags & LIR_OpArrayCopy::type_check) {
3238 // We don't know the array types are compatible
3239 if (basic_type != T_OBJECT) {
3240 // Simple test for basic type arrays
3241 __ cmp_klass(src, dst, tmp, tmp2);
3242 __ jcc(Assembler::notEqual, *stub->entry());
3243 } else {
3244 // For object arrays, if src is a sub class of dst then we can
3245 // safely do the copy.
3246 Label cont, slow;
3247
3248 __ push(src);
3249 __ push(dst);
3250
3251 __ load_klass(src, src, tmp_load_klass);
3252 __ load_klass(dst, dst, tmp_load_klass);
3253
3254 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
3255
3256 __ push(src);
3257 __ push(dst);
3258 __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
3259 __ pop(dst);
3260 __ pop(src);
3261
3280 __ load_klass(tmp, src, tmp_load_klass);
3281 } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3282 __ load_klass(tmp, dst, tmp_load_klass);
3283 }
3284 int lh_offset = in_bytes(Klass::layout_helper_offset());
3285 Address klass_lh_addr(tmp, lh_offset);
3286 jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
3287 __ cmpl(klass_lh_addr, objArray_lh);
3288 __ jcc(Assembler::notEqual, *stub->entry());
3289 }
3290
3291 // Spill because stubs can use any register they like and it's
3292 // easier to restore just those that we care about.
3293 store_parameter(dst, 0);
3294 store_parameter(dst_pos, 1);
3295 store_parameter(length, 2);
3296 store_parameter(src_pos, 3);
3297 store_parameter(src, 4);
3298
3299 #ifndef _LP64
3300 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
3301 __ movptr(tmp, dst_klass_addr);
3302 __ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset()));
3303 __ push(tmp);
3304 __ movl(tmp, Address(tmp, Klass::super_check_offset_offset()));
3305 __ push(tmp);
3306 __ push(length);
3307 __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3308 __ push(tmp);
3309 __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3310 __ push(tmp);
3311
3312 __ call_VM_leaf(copyfunc_addr, 5);
3313 #else
3314 __ movl2ptr(length, length); //higher 32bits must be null
3315
3316 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3317 assert_different_registers(c_rarg0, dst, dst_pos, length);
3318 __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3319 assert_different_registers(c_rarg1, dst, length);
3320
3384 }
3385
3386 #ifdef ASSERT
3387 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
3388 // Sanity check the known type with the incoming class. For the
3389 // primitive case the types must match exactly with src.klass and
3390 // dst.klass each exactly matching the default type. For the
3391 // object array case, if no type check is needed then either the
3392 // dst type is exactly the expected type and the src type is a
3393 // subtype which we can't check or src is the same array as dst
3394 // but not necessarily exactly of type default_type.
3395 Label known_ok, halt;
3396 __ mov_metadata(tmp, default_type->constant_encoding());
3397 #ifdef _LP64
3398 if (UseCompressedClassPointers) {
3399 __ encode_klass_not_null(tmp, rscratch1);
3400 }
3401 #endif
3402
3403 if (basic_type != T_OBJECT) {
3404 __ cmp_klass(tmp, dst, tmp2);
3405 __ jcc(Assembler::notEqual, halt);
3406 __ cmp_klass(tmp, src, tmp2);
3407 __ jcc(Assembler::equal, known_ok);
3408 } else {
3409 __ cmp_klass(tmp, dst, tmp2);
3410 __ jcc(Assembler::equal, known_ok);
3411 __ cmpptr(src, dst);
3412 __ jcc(Assembler::equal, known_ok);
3413 }
3414 __ bind(halt);
3415 __ stop("incorrect type information in arraycopy");
3416 __ bind(known_ok);
3417 }
3418 #endif
3419
3420 #ifndef PRODUCT
3421 if (PrintC1Statistics) {
3422 __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
3423 }
3424 #endif
3425
3426 #ifdef _LP64
3427 assert_different_registers(c_rarg0, dst, dst_pos, length);
3428 __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3429 assert_different_registers(c_rarg1, length);
3487 // done
3488 } else if (op->code() == lir_unlock) {
3489 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3490 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3491 } else {
3492 Unimplemented();
3493 }
3494 __ bind(*op->stub()->continuation());
3495 }
3496
3497 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
3498 Register obj = op->obj()->as_pointer_register();
3499 Register result = op->result_opr()->as_pointer_register();
3500
3501 CodeEmitInfo* info = op->info();
3502 if (info != nullptr) {
3503 add_debug_info_for_null_check_here(info);
3504 }
3505
3506 #ifdef _LP64
3507 if (UseCompactObjectHeaders) {
3508 Register tmp = rscratch1;
3509 assert_different_registers(tmp, obj);
3510 assert_different_registers(tmp, result);
3511
3512 __ movq(result, Address(obj, oopDesc::mark_offset_in_bytes()));
3513 __ shrq(result, markWord::klass_shift);
3514 __ decode_klass_not_null(result, tmp);
3515 } else if (UseCompressedClassPointers) {
3516 __ movl(result, Address(obj, oopDesc::klass_offset_in_bytes()));
3517 __ decode_klass_not_null(result, rscratch1);
3518 } else
3519 #endif
3520 {
3521 __ movptr(result, Address(obj, oopDesc::klass_offset_in_bytes()));
3522 }
3523 }
3524
3525 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3526 ciMethod* method = op->profiled_method();
3527 int bci = op->profiled_bci();
3528 ciMethod* callee = op->profiled_callee();
3529 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3530
3531 // Update counter for all call types
3532 ciMethodData* md = method->method_data_or_null();
3533 assert(md != nullptr, "Sanity");
3534 ciProfileData* data = md->bci_to_data(bci);
3535 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
3536 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
3537 Register mdo = op->mdo()->as_register();
3538 __ mov_metadata(mdo, md->constant_encoding());
3539 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
3540 // Perform additional virtual call profiling for invokevirtual and
3541 // invokeinterface bytecodes
3542 if (op->should_profile_receiver_type()) {
|