1212 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1213 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1214 __ b(*op->stub()->entry());
1215 } else {
1216 Register tmp1 = op->tmp1()->as_register();
1217 Register tmp2 = op->tmp2()->as_register();
1218 Register tmp3 = op->tmp3()->as_register();
1219 if (len == tmp1) {
1220 tmp1 = tmp3;
1221 } else if (len == tmp2) {
1222 tmp2 = tmp3;
1223 } else if (len == tmp3) {
1224 // everything is ok
1225 } else {
1226 __ mov(tmp3, len);
1227 }
1228 __ allocate_array(op->obj()->as_register(),
1229 len,
1230 tmp1,
1231 tmp2,
1232 arrayOopDesc::header_size(op->type()),
1233 array_element_size(op->type()),
1234 op->klass()->as_register(),
1235 *op->stub()->entry());
1236 }
1237 __ bind(*op->stub()->continuation());
1238 }
1239
1240 void LIR_Assembler::type_profile_helper(Register mdo,
1241 ciMethodData *md, ciProfileData *data,
1242 Register recv, Label* update_done) {
1243 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1244 Label next_test;
1245 // See if the receiver is receiver[n].
1246 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1247 __ ldr(rscratch1, Address(rscratch2));
1248 __ cmp(recv, rscratch1);
1249 __ br(Assembler::NE, next_test);
1250 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1251 __ addptr(data_addr, DataLayout::counter_increment);
1252 __ b(*update_done);
2272
2273 // r0 is -1^K where K == partial copied count
2274 __ eonw(rscratch1, r0, zr);
2275 // adjust length down and src/end pos up by partial copied count
2276 __ subw(length, length, rscratch1);
2277 __ addw(src_pos, src_pos, rscratch1);
2278 __ addw(dst_pos, dst_pos, rscratch1);
2279 __ b(*stub->entry());
2280
2281 __ bind(*stub->continuation());
2282 return;
2283 }
2284
2285 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2286
2287 int elem_size = type2aelembytes(basic_type);
2288 int scale = exact_log2(elem_size);
2289
2290 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2291 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2292 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2293 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2294
2295 // test for null
2296 if (flags & LIR_OpArrayCopy::src_null_check) {
2297 __ cbz(src, *stub->entry());
2298 }
2299 if (flags & LIR_OpArrayCopy::dst_null_check) {
2300 __ cbz(dst, *stub->entry());
2301 }
2302
2303 // If the compiler was not able to prove that exact type of the source or the destination
2304 // of the arraycopy is an array type, check at runtime if the source or the destination is
2305 // an instance type.
2306 if (flags & LIR_OpArrayCopy::type_check) {
2307 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2308 __ load_klass(tmp, dst);
2309 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2310 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2311 __ br(Assembler::GE, *stub->entry());
2312 }
2313
2334 __ br(Assembler::LT, *stub->entry());
2335 }
2336
2337 if (flags & LIR_OpArrayCopy::src_range_check) {
2338 __ addw(tmp, src_pos, length);
2339 __ ldrw(rscratch1, src_length_addr);
2340 __ cmpw(tmp, rscratch1);
2341 __ br(Assembler::HI, *stub->entry());
2342 }
2343 if (flags & LIR_OpArrayCopy::dst_range_check) {
2344 __ addw(tmp, dst_pos, length);
2345 __ ldrw(rscratch1, dst_length_addr);
2346 __ cmpw(tmp, rscratch1);
2347 __ br(Assembler::HI, *stub->entry());
2348 }
2349
2350 if (flags & LIR_OpArrayCopy::type_check) {
2351 // We don't know the array types are compatible
2352 if (basic_type != T_OBJECT) {
2353 // Simple test for basic type arrays
2354 if (UseCompressedClassPointers) {
2355 __ ldrw(tmp, src_klass_addr);
2356 __ ldrw(rscratch1, dst_klass_addr);
2357 __ cmpw(tmp, rscratch1);
2358 } else {
2359 __ ldr(tmp, src_klass_addr);
2360 __ ldr(rscratch1, dst_klass_addr);
2361 __ cmp(tmp, rscratch1);
2362 }
2363 __ br(Assembler::NE, *stub->entry());
2364 } else {
2365 // For object arrays, if src is a sub class of dst then we can
2366 // safely do the copy.
2367 Label cont, slow;
2368
2369 #define PUSH(r1, r2) \
2370 stp(r1, r2, __ pre(sp, -2 * wordSize));
2371
2372 #define POP(r1, r2) \
2373 ldp(r1, r2, __ post(sp, 2 * wordSize));
2374
2375 __ PUSH(src, dst);
2376
2377 __ load_klass(src, src);
2378 __ load_klass(dst, dst);
2379
2380 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2381
2382 __ PUSH(src, dst);
2464 }
2465
2466 __ b(*stub->entry());
2467
2468 __ bind(cont);
2469 __ POP(src, dst);
2470 }
2471 }
2472
2473 #ifdef ASSERT
2474 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2475 // Sanity check the known type with the incoming class. For the
2476 // primitive case the types must match exactly with src.klass and
2477 // dst.klass each exactly matching the default type. For the
2478 // object array case, if no type check is needed then either the
2479 // dst type is exactly the expected type and the src type is a
2480 // subtype which we can't check or src is the same array as dst
2481 // but not necessarily exactly of type default_type.
2482 Label known_ok, halt;
2483 __ mov_metadata(tmp, default_type->constant_encoding());
2484 if (UseCompressedClassPointers) {
2485 __ encode_klass_not_null(tmp);
2486 }
2487
2488 if (basic_type != T_OBJECT) {
2489
2490 if (UseCompressedClassPointers) {
2491 __ ldrw(rscratch1, dst_klass_addr);
2492 __ cmpw(tmp, rscratch1);
2493 } else {
2494 __ ldr(rscratch1, dst_klass_addr);
2495 __ cmp(tmp, rscratch1);
2496 }
2497 __ br(Assembler::NE, halt);
2498 if (UseCompressedClassPointers) {
2499 __ ldrw(rscratch1, src_klass_addr);
2500 __ cmpw(tmp, rscratch1);
2501 } else {
2502 __ ldr(rscratch1, src_klass_addr);
2503 __ cmp(tmp, rscratch1);
2504 }
2505 __ br(Assembler::EQ, known_ok);
2506 } else {
2507 if (UseCompressedClassPointers) {
2508 __ ldrw(rscratch1, dst_klass_addr);
2509 __ cmpw(tmp, rscratch1);
2510 } else {
2511 __ ldr(rscratch1, dst_klass_addr);
2512 __ cmp(tmp, rscratch1);
2513 }
2514 __ br(Assembler::EQ, known_ok);
2515 __ cmp(src, dst);
2516 __ br(Assembler::EQ, known_ok);
2517 }
2518 __ bind(halt);
2519 __ stop("incorrect type information in arraycopy");
2520 __ bind(known_ok);
2521 }
2522 #endif
2523
2524 #ifndef PRODUCT
2525 if (PrintC1Statistics) {
2526 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2527 }
2528 #endif
2529
2530 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2531 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2532 assert_different_registers(c_rarg0, dst, dst_pos, length);
2533 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2574 // done
2575 } else if (op->code() == lir_unlock) {
2576 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2577 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2578 } else {
2579 Unimplemented();
2580 }
2581 __ bind(*op->stub()->continuation());
2582 }
2583
2584 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2585 Register obj = op->obj()->as_pointer_register();
2586 Register result = op->result_opr()->as_pointer_register();
2587
2588 CodeEmitInfo* info = op->info();
2589 if (info != nullptr) {
2590 add_debug_info_for_null_check_here(info);
2591 }
2592
2593 if (UseCompressedClassPointers) {
2594 __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2595 __ decode_klass_not_null(result);
2596 } else {
2597 __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2598 }
2599 }
2600
2601 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2602 ciMethod* method = op->profiled_method();
2603 int bci = op->profiled_bci();
2604 ciMethod* callee = op->profiled_callee();
2605
2606 // Update counter for all call types
2607 ciMethodData* md = method->method_data_or_null();
2608 assert(md != nullptr, "Sanity");
2609 ciProfileData* data = md->bci_to_data(bci);
2610 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2611 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2612 Register mdo = op->mdo()->as_register();
2613 __ mov_metadata(mdo, md->constant_encoding());
2614 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
|
1212 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1213 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1214 __ b(*op->stub()->entry());
1215 } else {
1216 Register tmp1 = op->tmp1()->as_register();
1217 Register tmp2 = op->tmp2()->as_register();
1218 Register tmp3 = op->tmp3()->as_register();
1219 if (len == tmp1) {
1220 tmp1 = tmp3;
1221 } else if (len == tmp2) {
1222 tmp2 = tmp3;
1223 } else if (len == tmp3) {
1224 // everything is ok
1225 } else {
1226 __ mov(tmp3, len);
1227 }
1228 __ allocate_array(op->obj()->as_register(),
1229 len,
1230 tmp1,
1231 tmp2,
1232 arrayOopDesc::base_offset_in_bytes(op->type()),
1233 array_element_size(op->type()),
1234 op->klass()->as_register(),
1235 *op->stub()->entry());
1236 }
1237 __ bind(*op->stub()->continuation());
1238 }
1239
1240 void LIR_Assembler::type_profile_helper(Register mdo,
1241 ciMethodData *md, ciProfileData *data,
1242 Register recv, Label* update_done) {
1243 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1244 Label next_test;
1245 // See if the receiver is receiver[n].
1246 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1247 __ ldr(rscratch1, Address(rscratch2));
1248 __ cmp(recv, rscratch1);
1249 __ br(Assembler::NE, next_test);
1250 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1251 __ addptr(data_addr, DataLayout::counter_increment);
1252 __ b(*update_done);
2272
2273 // r0 is -1^K where K == partial copied count
2274 __ eonw(rscratch1, r0, zr);
2275 // adjust length down and src/end pos up by partial copied count
2276 __ subw(length, length, rscratch1);
2277 __ addw(src_pos, src_pos, rscratch1);
2278 __ addw(dst_pos, dst_pos, rscratch1);
2279 __ b(*stub->entry());
2280
2281 __ bind(*stub->continuation());
2282 return;
2283 }
2284
2285 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2286
2287 int elem_size = type2aelembytes(basic_type);
2288 int scale = exact_log2(elem_size);
2289
2290 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2291 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2292
2293 // test for null
2294 if (flags & LIR_OpArrayCopy::src_null_check) {
2295 __ cbz(src, *stub->entry());
2296 }
2297 if (flags & LIR_OpArrayCopy::dst_null_check) {
2298 __ cbz(dst, *stub->entry());
2299 }
2300
2301 // If the compiler was not able to prove that exact type of the source or the destination
2302 // of the arraycopy is an array type, check at runtime if the source or the destination is
2303 // an instance type.
2304 if (flags & LIR_OpArrayCopy::type_check) {
2305 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2306 __ load_klass(tmp, dst);
2307 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2308 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2309 __ br(Assembler::GE, *stub->entry());
2310 }
2311
2332 __ br(Assembler::LT, *stub->entry());
2333 }
2334
2335 if (flags & LIR_OpArrayCopy::src_range_check) {
2336 __ addw(tmp, src_pos, length);
2337 __ ldrw(rscratch1, src_length_addr);
2338 __ cmpw(tmp, rscratch1);
2339 __ br(Assembler::HI, *stub->entry());
2340 }
2341 if (flags & LIR_OpArrayCopy::dst_range_check) {
2342 __ addw(tmp, dst_pos, length);
2343 __ ldrw(rscratch1, dst_length_addr);
2344 __ cmpw(tmp, rscratch1);
2345 __ br(Assembler::HI, *stub->entry());
2346 }
2347
2348 if (flags & LIR_OpArrayCopy::type_check) {
2349 // We don't know the array types are compatible
2350 if (basic_type != T_OBJECT) {
2351 // Simple test for basic type arrays
2352 __ cmp_klass(src, dst, tmp, rscratch1);
2353 __ br(Assembler::NE, *stub->entry());
2354 } else {
2355 // For object arrays, if src is a sub class of dst then we can
2356 // safely do the copy.
2357 Label cont, slow;
2358
2359 #define PUSH(r1, r2) \
2360 stp(r1, r2, __ pre(sp, -2 * wordSize));
2361
2362 #define POP(r1, r2) \
2363 ldp(r1, r2, __ post(sp, 2 * wordSize));
2364
2365 __ PUSH(src, dst);
2366
2367 __ load_klass(src, src);
2368 __ load_klass(dst, dst);
2369
2370 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2371
2372 __ PUSH(src, dst);
2454 }
2455
2456 __ b(*stub->entry());
2457
2458 __ bind(cont);
2459 __ POP(src, dst);
2460 }
2461 }
2462
2463 #ifdef ASSERT
2464 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2465 // Sanity check the known type with the incoming class. For the
2466 // primitive case the types must match exactly with src.klass and
2467 // dst.klass each exactly matching the default type. For the
2468 // object array case, if no type check is needed then either the
2469 // dst type is exactly the expected type and the src type is a
2470 // subtype which we can't check or src is the same array as dst
2471 // but not necessarily exactly of type default_type.
2472 Label known_ok, halt;
2473 __ mov_metadata(tmp, default_type->constant_encoding());
2474
2475 if (basic_type != T_OBJECT) {
2476 __ cmp_klass(dst, tmp, rscratch1);
2477 __ br(Assembler::NE, halt);
2478 __ cmp_klass(src, tmp, rscratch1);
2479 __ br(Assembler::EQ, known_ok);
2480 } else {
2481 __ cmp_klass(dst, tmp, rscratch1);
2482 __ br(Assembler::EQ, known_ok);
2483 __ cmp(src, dst);
2484 __ br(Assembler::EQ, known_ok);
2485 }
2486 __ bind(halt);
2487 __ stop("incorrect type information in arraycopy");
2488 __ bind(known_ok);
2489 }
2490 #endif
2491
2492 #ifndef PRODUCT
2493 if (PrintC1Statistics) {
2494 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2495 }
2496 #endif
2497
2498 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2499 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2500 assert_different_registers(c_rarg0, dst, dst_pos, length);
2501 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2542 // done
2543 } else if (op->code() == lir_unlock) {
2544 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2545 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2546 } else {
2547 Unimplemented();
2548 }
2549 __ bind(*op->stub()->continuation());
2550 }
2551
2552 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2553 Register obj = op->obj()->as_pointer_register();
2554 Register result = op->result_opr()->as_pointer_register();
2555
2556 CodeEmitInfo* info = op->info();
2557 if (info != nullptr) {
2558 add_debug_info_for_null_check_here(info);
2559 }
2560
2561 if (UseCompressedClassPointers) {
2562 if (UseCompactObjectHeaders) {
2563 // Check if we can take the (common) fast path, if obj is unlocked.
2564 __ ldr(result, Address(obj, oopDesc::mark_offset_in_bytes()));
2565 __ tst(result, markWord::monitor_value);
2566 __ br(Assembler::NE, *op->stub()->entry());
2567 __ bind(*op->stub()->continuation());
2568
2569 // Shift to get proper narrow Klass*.
2570 __ lsr(result, result, markWord::klass_shift);
2571 } else {
2572 __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2573 }
2574 __ decode_klass_not_null(result);
2575 } else {
2576 __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2577 }
2578 }
2579
2580 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2581 ciMethod* method = op->profiled_method();
2582 int bci = op->profiled_bci();
2583 ciMethod* callee = op->profiled_callee();
2584
2585 // Update counter for all call types
2586 ciMethodData* md = method->method_data_or_null();
2587 assert(md != nullptr, "Sanity");
2588 ciProfileData* data = md->bci_to_data(bci);
2589 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2590 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2591 Register mdo = op->mdo()->as_register();
2592 __ mov_metadata(mdo, md->constant_encoding());
2593 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
|