2229
2230 // r0 is -1^K where K == partial copied count
2231 __ eonw(rscratch1, r0, zr);
2232 // adjust length down and src/end pos up by partial copied count
2233 __ subw(length, length, rscratch1);
2234 __ addw(src_pos, src_pos, rscratch1);
2235 __ addw(dst_pos, dst_pos, rscratch1);
2236 __ b(*stub->entry());
2237
2238 __ bind(*stub->continuation());
2239 return;
2240 }
2241
2242 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2243
2244 int elem_size = type2aelembytes(basic_type);
2245 int scale = exact_log2(elem_size);
2246
2247 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2248 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2249 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2250 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2251
2252 // test for null
2253 if (flags & LIR_OpArrayCopy::src_null_check) {
2254 __ cbz(src, *stub->entry());
2255 }
2256 if (flags & LIR_OpArrayCopy::dst_null_check) {
2257 __ cbz(dst, *stub->entry());
2258 }
2259
2260 // If the compiler was not able to prove that exact type of the source or the destination
2261 // of the arraycopy is an array type, check at runtime if the source or the destination is
2262 // an instance type.
2263 if (flags & LIR_OpArrayCopy::type_check) {
2264 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2265 __ load_klass(tmp, dst);
2266 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2267 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2268 __ br(Assembler::GE, *stub->entry());
2269 }
2270
2291 __ br(Assembler::LT, *stub->entry());
2292 }
2293
2294 if (flags & LIR_OpArrayCopy::src_range_check) {
2295 __ addw(tmp, src_pos, length);
2296 __ ldrw(rscratch1, src_length_addr);
2297 __ cmpw(tmp, rscratch1);
2298 __ br(Assembler::HI, *stub->entry());
2299 }
2300 if (flags & LIR_OpArrayCopy::dst_range_check) {
2301 __ addw(tmp, dst_pos, length);
2302 __ ldrw(rscratch1, dst_length_addr);
2303 __ cmpw(tmp, rscratch1);
2304 __ br(Assembler::HI, *stub->entry());
2305 }
2306
2307 if (flags & LIR_OpArrayCopy::type_check) {
2308 // We don't know the array types are compatible
2309 if (basic_type != T_OBJECT) {
2310 // Simple test for basic type arrays
2311 if (UseCompressedClassPointers) {
2312 __ ldrw(tmp, src_klass_addr);
2313 __ ldrw(rscratch1, dst_klass_addr);
2314 __ cmpw(tmp, rscratch1);
2315 } else {
2316 __ ldr(tmp, src_klass_addr);
2317 __ ldr(rscratch1, dst_klass_addr);
2318 __ cmp(tmp, rscratch1);
2319 }
2320 __ br(Assembler::NE, *stub->entry());
2321 } else {
2322 // For object arrays, if src is a sub class of dst then we can
2323 // safely do the copy.
2324 Label cont, slow;
2325
2326 #define PUSH(r1, r2) \
2327 stp(r1, r2, __ pre(sp, -2 * wordSize));
2328
2329 #define POP(r1, r2) \
2330 ldp(r1, r2, __ post(sp, 2 * wordSize));
2331
2332 __ PUSH(src, dst);
2333
2334 __ load_klass(src, src);
2335 __ load_klass(dst, dst);
2336
2337 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2338
2339 __ PUSH(src, dst);
2421 }
2422
2423 __ b(*stub->entry());
2424
2425 __ bind(cont);
2426 __ POP(src, dst);
2427 }
2428 }
2429
2430 #ifdef ASSERT
2431 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2432 // Sanity check the known type with the incoming class. For the
2433 // primitive case the types must match exactly with src.klass and
2434 // dst.klass each exactly matching the default type. For the
2435 // object array case, if no type check is needed then either the
2436 // dst type is exactly the expected type and the src type is a
2437 // subtype which we can't check or src is the same array as dst
2438 // but not necessarily exactly of type default_type.
2439 Label known_ok, halt;
2440 __ mov_metadata(tmp, default_type->constant_encoding());
2441 if (UseCompressedClassPointers) {
2442 __ encode_klass_not_null(tmp);
2443 }
2444
2445 if (basic_type != T_OBJECT) {
2446
2447 if (UseCompressedClassPointers) {
2448 __ ldrw(rscratch1, dst_klass_addr);
2449 __ cmpw(tmp, rscratch1);
2450 } else {
2451 __ ldr(rscratch1, dst_klass_addr);
2452 __ cmp(tmp, rscratch1);
2453 }
2454 __ br(Assembler::NE, halt);
2455 if (UseCompressedClassPointers) {
2456 __ ldrw(rscratch1, src_klass_addr);
2457 __ cmpw(tmp, rscratch1);
2458 } else {
2459 __ ldr(rscratch1, src_klass_addr);
2460 __ cmp(tmp, rscratch1);
2461 }
2462 __ br(Assembler::EQ, known_ok);
2463 } else {
2464 if (UseCompressedClassPointers) {
2465 __ ldrw(rscratch1, dst_klass_addr);
2466 __ cmpw(tmp, rscratch1);
2467 } else {
2468 __ ldr(rscratch1, dst_klass_addr);
2469 __ cmp(tmp, rscratch1);
2470 }
2471 __ br(Assembler::EQ, known_ok);
2472 __ cmp(src, dst);
2473 __ br(Assembler::EQ, known_ok);
2474 }
2475 __ bind(halt);
2476 __ stop("incorrect type information in arraycopy");
2477 __ bind(known_ok);
2478 }
2479 #endif
2480
2481 #ifndef PRODUCT
2482 if (PrintC1Statistics) {
2483 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2484 }
2485 #endif
2486
2487 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2488 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2489 assert_different_registers(c_rarg0, dst, dst_pos, length);
2490 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2534 // done
2535 } else if (op->code() == lir_unlock) {
2536 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2537 __ unlock_object(hdr, obj, lock, temp, *op->stub()->entry());
2538 } else {
2539 Unimplemented();
2540 }
2541 __ bind(*op->stub()->continuation());
2542 }
2543
2544 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2545 Register obj = op->obj()->as_pointer_register();
2546 Register result = op->result_opr()->as_pointer_register();
2547
2548 CodeEmitInfo* info = op->info();
2549 if (info != nullptr) {
2550 add_debug_info_for_null_check_here(info);
2551 }
2552
2553 if (UseCompressedClassPointers) {
2554 __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2555 __ decode_klass_not_null(result);
2556 } else {
2557 __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2558 }
2559 }
2560
2561 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2562 ciMethod* method = op->profiled_method();
2563 int bci = op->profiled_bci();
2564 ciMethod* callee = op->profiled_callee();
2565
2566 // Update counter for all call types
2567 ciMethodData* md = method->method_data_or_null();
2568 assert(md != nullptr, "Sanity");
2569 ciProfileData* data = md->bci_to_data(bci);
2570 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2571 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2572 Register mdo = op->mdo()->as_register();
2573 __ mov_metadata(mdo, md->constant_encoding());
2574 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
|
2229
2230 // r0 is -1^K where K == partial copied count
2231 __ eonw(rscratch1, r0, zr);
2232 // adjust length down and src/end pos up by partial copied count
2233 __ subw(length, length, rscratch1);
2234 __ addw(src_pos, src_pos, rscratch1);
2235 __ addw(dst_pos, dst_pos, rscratch1);
2236 __ b(*stub->entry());
2237
2238 __ bind(*stub->continuation());
2239 return;
2240 }
2241
2242 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2243
2244 int elem_size = type2aelembytes(basic_type);
2245 int scale = exact_log2(elem_size);
2246
2247 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2248 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2249
2250 // test for null
2251 if (flags & LIR_OpArrayCopy::src_null_check) {
2252 __ cbz(src, *stub->entry());
2253 }
2254 if (flags & LIR_OpArrayCopy::dst_null_check) {
2255 __ cbz(dst, *stub->entry());
2256 }
2257
2258 // If the compiler was not able to prove that exact type of the source or the destination
2259 // of the arraycopy is an array type, check at runtime if the source or the destination is
2260 // an instance type.
2261 if (flags & LIR_OpArrayCopy::type_check) {
2262 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2263 __ load_klass(tmp, dst);
2264 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2265 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2266 __ br(Assembler::GE, *stub->entry());
2267 }
2268
2289 __ br(Assembler::LT, *stub->entry());
2290 }
2291
2292 if (flags & LIR_OpArrayCopy::src_range_check) {
2293 __ addw(tmp, src_pos, length);
2294 __ ldrw(rscratch1, src_length_addr);
2295 __ cmpw(tmp, rscratch1);
2296 __ br(Assembler::HI, *stub->entry());
2297 }
2298 if (flags & LIR_OpArrayCopy::dst_range_check) {
2299 __ addw(tmp, dst_pos, length);
2300 __ ldrw(rscratch1, dst_length_addr);
2301 __ cmpw(tmp, rscratch1);
2302 __ br(Assembler::HI, *stub->entry());
2303 }
2304
2305 if (flags & LIR_OpArrayCopy::type_check) {
2306 // We don't know the array types are compatible
2307 if (basic_type != T_OBJECT) {
2308 // Simple test for basic type arrays
2309 __ cmp_klass(src, dst, tmp, rscratch1);
2310 __ br(Assembler::NE, *stub->entry());
2311 } else {
2312 // For object arrays, if src is a sub class of dst then we can
2313 // safely do the copy.
2314 Label cont, slow;
2315
2316 #define PUSH(r1, r2) \
2317 stp(r1, r2, __ pre(sp, -2 * wordSize));
2318
2319 #define POP(r1, r2) \
2320 ldp(r1, r2, __ post(sp, 2 * wordSize));
2321
2322 __ PUSH(src, dst);
2323
2324 __ load_klass(src, src);
2325 __ load_klass(dst, dst);
2326
2327 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2328
2329 __ PUSH(src, dst);
2411 }
2412
2413 __ b(*stub->entry());
2414
2415 __ bind(cont);
2416 __ POP(src, dst);
2417 }
2418 }
2419
2420 #ifdef ASSERT
2421 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2422 // Sanity check the known type with the incoming class. For the
2423 // primitive case the types must match exactly with src.klass and
2424 // dst.klass each exactly matching the default type. For the
2425 // object array case, if no type check is needed then either the
2426 // dst type is exactly the expected type and the src type is a
2427 // subtype which we can't check or src is the same array as dst
2428 // but not necessarily exactly of type default_type.
2429 Label known_ok, halt;
2430 __ mov_metadata(tmp, default_type->constant_encoding());
2431
2432 if (basic_type != T_OBJECT) {
2433 __ cmp_klass(dst, tmp, rscratch1);
2434 __ br(Assembler::NE, halt);
2435 __ cmp_klass(src, tmp, rscratch1);
2436 __ br(Assembler::EQ, known_ok);
2437 } else {
2438 __ cmp_klass(dst, tmp, rscratch1);
2439 __ br(Assembler::EQ, known_ok);
2440 __ cmp(src, dst);
2441 __ br(Assembler::EQ, known_ok);
2442 }
2443 __ bind(halt);
2444 __ stop("incorrect type information in arraycopy");
2445 __ bind(known_ok);
2446 }
2447 #endif
2448
2449 #ifndef PRODUCT
2450 if (PrintC1Statistics) {
2451 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2452 }
2453 #endif
2454
2455 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2456 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2457 assert_different_registers(c_rarg0, dst, dst_pos, length);
2458 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2502 // done
2503 } else if (op->code() == lir_unlock) {
2504 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2505 __ unlock_object(hdr, obj, lock, temp, *op->stub()->entry());
2506 } else {
2507 Unimplemented();
2508 }
2509 __ bind(*op->stub()->continuation());
2510 }
2511
2512 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2513 Register obj = op->obj()->as_pointer_register();
2514 Register result = op->result_opr()->as_pointer_register();
2515
2516 CodeEmitInfo* info = op->info();
2517 if (info != nullptr) {
2518 add_debug_info_for_null_check_here(info);
2519 }
2520
2521 if (UseCompressedClassPointers) {
2522 if (UseCompactObjectHeaders) {
2523 __ ldr(result, Address(obj, oopDesc::mark_offset_in_bytes()));
2524 __ lsr(result, result, markWord::klass_shift);
2525 } else {
2526 __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2527 }
2528 __ decode_klass_not_null(result);
2529 } else {
2530 __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2531 }
2532 }
2533
2534 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2535 ciMethod* method = op->profiled_method();
2536 int bci = op->profiled_bci();
2537 ciMethod* callee = op->profiled_callee();
2538
2539 // Update counter for all call types
2540 ciMethodData* md = method->method_data_or_null();
2541 assert(md != nullptr, "Sanity");
2542 ciProfileData* data = md->bci_to_data(bci);
2543 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2544 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2545 Register mdo = op->mdo()->as_register();
2546 __ mov_metadata(mdo, md->constant_encoding());
2547 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
|