1212 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1213 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1214 __ b(*op->stub()->entry());
1215 } else {
1216 Register tmp1 = op->tmp1()->as_register();
1217 Register tmp2 = op->tmp2()->as_register();
1218 Register tmp3 = op->tmp3()->as_register();
1219 if (len == tmp1) {
1220 tmp1 = tmp3;
1221 } else if (len == tmp2) {
1222 tmp2 = tmp3;
1223 } else if (len == tmp3) {
1224 // everything is ok
1225 } else {
1226 __ mov(tmp3, len);
1227 }
1228 __ allocate_array(op->obj()->as_register(),
1229 len,
1230 tmp1,
1231 tmp2,
1232 arrayOopDesc::header_size(op->type()),
1233 array_element_size(op->type()),
1234 op->klass()->as_register(),
1235 *op->stub()->entry());
1236 }
1237 __ bind(*op->stub()->continuation());
1238 }
1239
1240 void LIR_Assembler::type_profile_helper(Register mdo,
1241 ciMethodData *md, ciProfileData *data,
1242 Register recv, Label* update_done) {
1243 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1244 Label next_test;
1245 // See if the receiver is receiver[n].
1246 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1247 __ ldr(rscratch1, Address(rscratch2));
1248 __ cmp(recv, rscratch1);
1249 __ br(Assembler::NE, next_test);
1250 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1251 __ addptr(data_addr, DataLayout::counter_increment);
1252 __ b(*update_done);
2335 }
2336
2337 if (flags & LIR_OpArrayCopy::src_range_check) {
2338 __ addw(tmp, src_pos, length);
2339 __ ldrw(rscratch1, src_length_addr);
2340 __ cmpw(tmp, rscratch1);
2341 __ br(Assembler::HI, *stub->entry());
2342 }
2343 if (flags & LIR_OpArrayCopy::dst_range_check) {
2344 __ addw(tmp, dst_pos, length);
2345 __ ldrw(rscratch1, dst_length_addr);
2346 __ cmpw(tmp, rscratch1);
2347 __ br(Assembler::HI, *stub->entry());
2348 }
2349
2350 if (flags & LIR_OpArrayCopy::type_check) {
2351 // We don't know the array types are compatible
2352 if (basic_type != T_OBJECT) {
2353 // Simple test for basic type arrays
2354 if (UseCompressedClassPointers) {
2355 __ ldrw(tmp, src_klass_addr);
2356 __ ldrw(rscratch1, dst_klass_addr);
2357 __ cmpw(tmp, rscratch1);
2358 } else {
2359 __ ldr(tmp, src_klass_addr);
2360 __ ldr(rscratch1, dst_klass_addr);
2361 __ cmp(tmp, rscratch1);
2362 }
2363 __ br(Assembler::NE, *stub->entry());
2364 } else {
2365 // For object arrays, if src is a sub class of dst then we can
2366 // safely do the copy.
2367 Label cont, slow;
2368
2369 #define PUSH(r1, r2) \
2370 stp(r1, r2, __ pre(sp, -2 * wordSize));
2371
2372 #define POP(r1, r2) \
2373 ldp(r1, r2, __ post(sp, 2 * wordSize));
2374
2375 __ PUSH(src, dst);
2376
2377 __ load_klass(src, src);
2378 __ load_klass(dst, dst);
2379
2380 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2464 }
2465
2466 __ b(*stub->entry());
2467
2468 __ bind(cont);
2469 __ POP(src, dst);
2470 }
2471 }
2472
2473 #ifdef ASSERT
2474 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2475 // Sanity check the known type with the incoming class. For the
2476 // primitive case the types must match exactly with src.klass and
2477 // dst.klass each exactly matching the default type. For the
2478 // object array case, if no type check is needed then either the
2479 // dst type is exactly the expected type and the src type is a
2480 // subtype which we can't check or src is the same array as dst
2481 // but not necessarily exactly of type default_type.
2482 Label known_ok, halt;
2483 __ mov_metadata(tmp, default_type->constant_encoding());
2484 if (UseCompressedClassPointers) {
2485 __ encode_klass_not_null(tmp);
2486 }
2487
2488 if (basic_type != T_OBJECT) {
2489
2490 if (UseCompressedClassPointers) {
2491 __ ldrw(rscratch1, dst_klass_addr);
2492 __ cmpw(tmp, rscratch1);
2493 } else {
2494 __ ldr(rscratch1, dst_klass_addr);
2495 __ cmp(tmp, rscratch1);
2496 }
2497 __ br(Assembler::NE, halt);
2498 if (UseCompressedClassPointers) {
2499 __ ldrw(rscratch1, src_klass_addr);
2500 __ cmpw(tmp, rscratch1);
2501 } else {
2502 __ ldr(rscratch1, src_klass_addr);
2503 __ cmp(tmp, rscratch1);
2504 }
2505 __ br(Assembler::EQ, known_ok);
2506 } else {
2507 if (UseCompressedClassPointers) {
2508 __ ldrw(rscratch1, dst_klass_addr);
2509 __ cmpw(tmp, rscratch1);
2510 } else {
2511 __ ldr(rscratch1, dst_klass_addr);
2512 __ cmp(tmp, rscratch1);
2513 }
2514 __ br(Assembler::EQ, known_ok);
2515 __ cmp(src, dst);
2516 __ br(Assembler::EQ, known_ok);
2517 }
2518 __ bind(halt);
2519 __ stop("incorrect type information in arraycopy");
2520 __ bind(known_ok);
2521 }
2522 #endif
2523
2524 #ifndef PRODUCT
2525 if (PrintC1Statistics) {
2526 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2527 }
2528 #endif
2529
2530 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2531 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2532 assert_different_registers(c_rarg0, dst, dst_pos, length);
2533 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2574 // done
2575 } else if (op->code() == lir_unlock) {
2576 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2577 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2578 } else {
2579 Unimplemented();
2580 }
2581 __ bind(*op->stub()->continuation());
2582 }
2583
2584 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2585 Register obj = op->obj()->as_pointer_register();
2586 Register result = op->result_opr()->as_pointer_register();
2587
2588 CodeEmitInfo* info = op->info();
2589 if (info != nullptr) {
2590 add_debug_info_for_null_check_here(info);
2591 }
2592
2593 if (UseCompressedClassPointers) {
2594 __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2595 __ decode_klass_not_null(result);
2596 } else {
2597 __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2598 }
2599 }
2600
2601 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2602 ciMethod* method = op->profiled_method();
2603 int bci = op->profiled_bci();
2604 ciMethod* callee = op->profiled_callee();
2605
2606 // Update counter for all call types
2607 ciMethodData* md = method->method_data_or_null();
2608 assert(md != nullptr, "Sanity");
2609 ciProfileData* data = md->bci_to_data(bci);
2610 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2611 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2612 Register mdo = op->mdo()->as_register();
2613 __ mov_metadata(mdo, md->constant_encoding());
2614 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
|
1212 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1213 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1214 __ b(*op->stub()->entry());
1215 } else {
1216 Register tmp1 = op->tmp1()->as_register();
1217 Register tmp2 = op->tmp2()->as_register();
1218 Register tmp3 = op->tmp3()->as_register();
1219 if (len == tmp1) {
1220 tmp1 = tmp3;
1221 } else if (len == tmp2) {
1222 tmp2 = tmp3;
1223 } else if (len == tmp3) {
1224 // everything is ok
1225 } else {
1226 __ mov(tmp3, len);
1227 }
1228 __ allocate_array(op->obj()->as_register(),
1229 len,
1230 tmp1,
1231 tmp2,
1232 arrayOopDesc::base_offset_in_bytes(op->type()),
1233 array_element_size(op->type()),
1234 op->klass()->as_register(),
1235 *op->stub()->entry());
1236 }
1237 __ bind(*op->stub()->continuation());
1238 }
1239
1240 void LIR_Assembler::type_profile_helper(Register mdo,
1241 ciMethodData *md, ciProfileData *data,
1242 Register recv, Label* update_done) {
1243 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1244 Label next_test;
1245 // See if the receiver is receiver[n].
1246 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1247 __ ldr(rscratch1, Address(rscratch2));
1248 __ cmp(recv, rscratch1);
1249 __ br(Assembler::NE, next_test);
1250 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1251 __ addptr(data_addr, DataLayout::counter_increment);
1252 __ b(*update_done);
2335 }
2336
2337 if (flags & LIR_OpArrayCopy::src_range_check) {
2338 __ addw(tmp, src_pos, length);
2339 __ ldrw(rscratch1, src_length_addr);
2340 __ cmpw(tmp, rscratch1);
2341 __ br(Assembler::HI, *stub->entry());
2342 }
2343 if (flags & LIR_OpArrayCopy::dst_range_check) {
2344 __ addw(tmp, dst_pos, length);
2345 __ ldrw(rscratch1, dst_length_addr);
2346 __ cmpw(tmp, rscratch1);
2347 __ br(Assembler::HI, *stub->entry());
2348 }
2349
2350 if (flags & LIR_OpArrayCopy::type_check) {
2351 // We don't know the array types are compatible
2352 if (basic_type != T_OBJECT) {
2353 // Simple test for basic type arrays
2354 if (UseCompressedClassPointers) {
2355 __ load_nklass(tmp, src);
2356 __ load_nklass(rscratch1, dst);
2357 __ cmpw(tmp, rscratch1);
2358 } else {
2359 __ ldr(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
2360 __ ldr(rscratch1, Address(dst, oopDesc::klass_offset_in_bytes()));
2361 __ cmp(tmp, rscratch1);
2362 }
2363 __ br(Assembler::NE, *stub->entry());
2364 } else {
2365 // For object arrays, if src is a sub class of dst then we can
2366 // safely do the copy.
2367 Label cont, slow;
2368
2369 #define PUSH(r1, r2) \
2370 stp(r1, r2, __ pre(sp, -2 * wordSize));
2371
2372 #define POP(r1, r2) \
2373 ldp(r1, r2, __ post(sp, 2 * wordSize));
2374
2375 __ PUSH(src, dst);
2376
2377 __ load_klass(src, src);
2378 __ load_klass(dst, dst);
2379
2380 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2464 }
2465
2466 __ b(*stub->entry());
2467
2468 __ bind(cont);
2469 __ POP(src, dst);
2470 }
2471 }
2472
2473 #ifdef ASSERT
2474 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2475 // Sanity check the known type with the incoming class. For the
2476 // primitive case the types must match exactly with src.klass and
2477 // dst.klass each exactly matching the default type. For the
2478 // object array case, if no type check is needed then either the
2479 // dst type is exactly the expected type and the src type is a
2480 // subtype which we can't check or src is the same array as dst
2481 // but not necessarily exactly of type default_type.
2482 Label known_ok, halt;
2483 __ mov_metadata(tmp, default_type->constant_encoding());
2484
2485 if (basic_type != T_OBJECT) {
2486 __ cmp_klass(dst, tmp, rscratch1);
2487 __ br(Assembler::NE, halt);
2488 __ cmp_klass(src, tmp, rscratch1);
2489 __ br(Assembler::EQ, known_ok);
2490 } else {
2491 __ cmp_klass(dst, tmp, rscratch1);
2492 __ br(Assembler::EQ, known_ok);
2493 __ cmp(src, dst);
2494 __ br(Assembler::EQ, known_ok);
2495 }
2496 __ bind(halt);
2497 __ stop("incorrect type information in arraycopy");
2498 __ bind(known_ok);
2499 }
2500 #endif
2501
2502 #ifndef PRODUCT
2503 if (PrintC1Statistics) {
2504 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2505 }
2506 #endif
2507
2508 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2509 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2510 assert_different_registers(c_rarg0, dst, dst_pos, length);
2511 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2552 // done
2553 } else if (op->code() == lir_unlock) {
2554 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2555 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2556 } else {
2557 Unimplemented();
2558 }
2559 __ bind(*op->stub()->continuation());
2560 }
2561
2562 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2563 Register obj = op->obj()->as_pointer_register();
2564 Register result = op->result_opr()->as_pointer_register();
2565
2566 CodeEmitInfo* info = op->info();
2567 if (info != nullptr) {
2568 add_debug_info_for_null_check_here(info);
2569 }
2570
2571 if (UseCompressedClassPointers) {
2572 if (UseCompactObjectHeaders) {
2573 // Check if we can take the (common) fast path, if obj is unlocked.
2574 __ ldr(result, Address(obj, oopDesc::mark_offset_in_bytes()));
2575 __ tst(result, markWord::monitor_value);
2576 __ br(Assembler::NE, *op->stub()->entry());
2577 __ bind(*op->stub()->continuation());
2578
2579 // Shift to get proper narrow Klass*.
2580 __ lsr(result, result, markWord::klass_shift);
2581 } else {
2582 __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2583 }
2584 __ decode_klass_not_null(result);
2585 } else {
2586 __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2587 }
2588 }
2589
2590 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2591 ciMethod* method = op->profiled_method();
2592 int bci = op->profiled_bci();
2593 ciMethod* callee = op->profiled_callee();
2594
2595 // Update counter for all call types
2596 ciMethodData* md = method->method_data_or_null();
2597 assert(md != nullptr, "Sanity");
2598 ciProfileData* data = md->bci_to_data(bci);
2599 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2600 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2601 Register mdo = op->mdo()->as_register();
2602 __ mov_metadata(mdo, md->constant_encoding());
2603 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
|