< prev index next >

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

Print this page

1212       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1213       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1214     __ b(*op->stub()->entry());
1215   } else {
1216     Register tmp1 = op->tmp1()->as_register();
1217     Register tmp2 = op->tmp2()->as_register();
1218     Register tmp3 = op->tmp3()->as_register();
1219     if (len == tmp1) {
1220       tmp1 = tmp3;
1221     } else if (len == tmp2) {
1222       tmp2 = tmp3;
1223     } else if (len == tmp3) {
1224       // everything is ok
1225     } else {
1226       __ mov(tmp3, len);
1227     }
1228     __ allocate_array(op->obj()->as_register(),
1229                       len,
1230                       tmp1,
1231                       tmp2,
1232                       arrayOopDesc::header_size(op->type()),
1233                       array_element_size(op->type()),
1234                       op->klass()->as_register(),
1235                       *op->stub()->entry());
1236   }
1237   __ bind(*op->stub()->continuation());
1238 }
1239 
1240 void LIR_Assembler::type_profile_helper(Register mdo,
1241                                         ciMethodData *md, ciProfileData *data,
1242                                         Register recv, Label* update_done) {
1243   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1244     Label next_test;
1245     // See if the receiver is receiver[n].
1246     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1247     __ ldr(rscratch1, Address(rscratch2));
1248     __ cmp(recv, rscratch1);
1249     __ br(Assembler::NE, next_test);
1250     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1251     __ addptr(data_addr, DataLayout::counter_increment);
1252     __ b(*update_done);

2272 
2273     // r0 is -1^K where K == partial copied count
2274     __ eonw(rscratch1, r0, zr);
2275     // adjust length down and src/end pos up by partial copied count
2276     __ subw(length, length, rscratch1);
2277     __ addw(src_pos, src_pos, rscratch1);
2278     __ addw(dst_pos, dst_pos, rscratch1);
2279     __ b(*stub->entry());
2280 
2281     __ bind(*stub->continuation());
2282     return;
2283   }
2284 
2285   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2286 
2287   int elem_size = type2aelembytes(basic_type);
2288   int scale = exact_log2(elem_size);
2289 
2290   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2291   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2292   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2293   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2294 
2295   // test for null
2296   if (flags & LIR_OpArrayCopy::src_null_check) {
2297     __ cbz(src, *stub->entry());
2298   }
2299   if (flags & LIR_OpArrayCopy::dst_null_check) {
2300     __ cbz(dst, *stub->entry());
2301   }
2302 
2303   // If the compiler was not able to prove that exact type of the source or the destination
2304   // of the arraycopy is an array type, check at runtime if the source or the destination is
2305   // an instance type.
2306   if (flags & LIR_OpArrayCopy::type_check) {
2307     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2308       __ load_klass(tmp, dst);
2309       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2310       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2311       __ br(Assembler::GE, *stub->entry());
2312     }
2313 

2334     __ br(Assembler::LT, *stub->entry());
2335   }
2336 
2337   if (flags & LIR_OpArrayCopy::src_range_check) {
2338     __ addw(tmp, src_pos, length);
2339     __ ldrw(rscratch1, src_length_addr);
2340     __ cmpw(tmp, rscratch1);
2341     __ br(Assembler::HI, *stub->entry());
2342   }
2343   if (flags & LIR_OpArrayCopy::dst_range_check) {
2344     __ addw(tmp, dst_pos, length);
2345     __ ldrw(rscratch1, dst_length_addr);
2346     __ cmpw(tmp, rscratch1);
2347     __ br(Assembler::HI, *stub->entry());
2348   }
2349 
2350   if (flags & LIR_OpArrayCopy::type_check) {
2351     // We don't know the array types are compatible
2352     if (basic_type != T_OBJECT) {
2353       // Simple test for basic type arrays
2354       if (UseCompressedClassPointers) {
2355         __ ldrw(tmp, src_klass_addr);
2356         __ ldrw(rscratch1, dst_klass_addr);
2357         __ cmpw(tmp, rscratch1);
2358       } else {
2359         __ ldr(tmp, src_klass_addr);
2360         __ ldr(rscratch1, dst_klass_addr);
2361         __ cmp(tmp, rscratch1);
2362       }
2363       __ br(Assembler::NE, *stub->entry());
2364     } else {
2365       // For object arrays, if src is a sub class of dst then we can
2366       // safely do the copy.
2367       Label cont, slow;
2368 
2369 #define PUSH(r1, r2)                                    \
2370       stp(r1, r2, __ pre(sp, -2 * wordSize));
2371 
2372 #define POP(r1, r2)                                     \
2373       ldp(r1, r2, __ post(sp, 2 * wordSize));
2374 
2375       __ PUSH(src, dst);
2376 
2377       __ load_klass(src, src);
2378       __ load_klass(dst, dst);
2379 
2380       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2381 
2382       __ PUSH(src, dst);

2464       }
2465 
2466       __ b(*stub->entry());
2467 
2468       __ bind(cont);
2469       __ POP(src, dst);
2470     }
2471   }
2472 
2473 #ifdef ASSERT
2474   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2475     // Sanity check the known type with the incoming class.  For the
2476     // primitive case the types must match exactly with src.klass and
2477     // dst.klass each exactly matching the default type.  For the
2478     // object array case, if no type check is needed then either the
2479     // dst type is exactly the expected type and the src type is a
2480     // subtype which we can't check or src is the same array as dst
2481     // but not necessarily exactly of type default_type.
2482     Label known_ok, halt;
2483     __ mov_metadata(tmp, default_type->constant_encoding());
2484     if (UseCompressedClassPointers) {
2485       __ encode_klass_not_null(tmp);
2486     }
2487 
2488     if (basic_type != T_OBJECT) {
2489 
2490       if (UseCompressedClassPointers) {
2491         __ ldrw(rscratch1, dst_klass_addr);
2492         __ cmpw(tmp, rscratch1);
2493       } else {
2494         __ ldr(rscratch1, dst_klass_addr);
2495         __ cmp(tmp, rscratch1);
2496       }
2497       __ br(Assembler::NE, halt);
2498       if (UseCompressedClassPointers) {
2499         __ ldrw(rscratch1, src_klass_addr);
2500         __ cmpw(tmp, rscratch1);
2501       } else {
2502         __ ldr(rscratch1, src_klass_addr);
2503         __ cmp(tmp, rscratch1);
2504       }
2505       __ br(Assembler::EQ, known_ok);
2506     } else {
2507       if (UseCompressedClassPointers) {
2508         __ ldrw(rscratch1, dst_klass_addr);
2509         __ cmpw(tmp, rscratch1);
2510       } else {
2511         __ ldr(rscratch1, dst_klass_addr);
2512         __ cmp(tmp, rscratch1);
2513       }
2514       __ br(Assembler::EQ, known_ok);
2515       __ cmp(src, dst);
2516       __ br(Assembler::EQ, known_ok);
2517     }
2518     __ bind(halt);
2519     __ stop("incorrect type information in arraycopy");
2520     __ bind(known_ok);
2521   }
2522 #endif
2523 
2524 #ifndef PRODUCT
2525   if (PrintC1Statistics) {
2526     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2527   }
2528 #endif
2529 
2530   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2531   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2532   assert_different_registers(c_rarg0, dst, dst_pos, length);
2533   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));

2574     // done
2575   } else if (op->code() == lir_unlock) {
2576     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2577     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2578   } else {
2579     Unimplemented();
2580   }
2581   __ bind(*op->stub()->continuation());
2582 }
2583 
2584 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2585   Register obj = op->obj()->as_pointer_register();
2586   Register result = op->result_opr()->as_pointer_register();
2587 
2588   CodeEmitInfo* info = op->info();
2589   if (info != nullptr) {
2590     add_debug_info_for_null_check_here(info);
2591   }
2592 
2593   if (UseCompressedClassPointers) {
2594     __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));











2595     __ decode_klass_not_null(result);
2596   } else {
2597     __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2598   }
2599 }
2600 
2601 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2602   ciMethod* method = op->profiled_method();
2603   int bci          = op->profiled_bci();
2604   ciMethod* callee = op->profiled_callee();
2605 
2606   // Update counter for all call types
2607   ciMethodData* md = method->method_data_or_null();
2608   assert(md != nullptr, "Sanity");
2609   ciProfileData* data = md->bci_to_data(bci);
2610   assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2611   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2612   Register mdo  = op->mdo()->as_register();
2613   __ mov_metadata(mdo, md->constant_encoding());
2614   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));

1212       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1213       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1214     __ b(*op->stub()->entry());
1215   } else {
1216     Register tmp1 = op->tmp1()->as_register();
1217     Register tmp2 = op->tmp2()->as_register();
1218     Register tmp3 = op->tmp3()->as_register();
1219     if (len == tmp1) {
1220       tmp1 = tmp3;
1221     } else if (len == tmp2) {
1222       tmp2 = tmp3;
1223     } else if (len == tmp3) {
1224       // everything is ok
1225     } else {
1226       __ mov(tmp3, len);
1227     }
1228     __ allocate_array(op->obj()->as_register(),
1229                       len,
1230                       tmp1,
1231                       tmp2,
1232                       arrayOopDesc::base_offset_in_bytes(op->type()),
1233                       array_element_size(op->type()),
1234                       op->klass()->as_register(),
1235                       *op->stub()->entry());
1236   }
1237   __ bind(*op->stub()->continuation());
1238 }
1239 
1240 void LIR_Assembler::type_profile_helper(Register mdo,
1241                                         ciMethodData *md, ciProfileData *data,
1242                                         Register recv, Label* update_done) {
1243   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1244     Label next_test;
1245     // See if the receiver is receiver[n].
1246     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1247     __ ldr(rscratch1, Address(rscratch2));
1248     __ cmp(recv, rscratch1);
1249     __ br(Assembler::NE, next_test);
1250     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1251     __ addptr(data_addr, DataLayout::counter_increment);
1252     __ b(*update_done);

2272 
2273     // r0 is -1^K where K == partial copied count
2274     __ eonw(rscratch1, r0, zr);
2275     // adjust length down and src/end pos up by partial copied count
2276     __ subw(length, length, rscratch1);
2277     __ addw(src_pos, src_pos, rscratch1);
2278     __ addw(dst_pos, dst_pos, rscratch1);
2279     __ b(*stub->entry());
2280 
2281     __ bind(*stub->continuation());
2282     return;
2283   }
2284 
2285   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2286 
2287   int elem_size = type2aelembytes(basic_type);
2288   int scale = exact_log2(elem_size);
2289 
2290   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2291   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());


2292 
2293   // test for null
2294   if (flags & LIR_OpArrayCopy::src_null_check) {
2295     __ cbz(src, *stub->entry());
2296   }
2297   if (flags & LIR_OpArrayCopy::dst_null_check) {
2298     __ cbz(dst, *stub->entry());
2299   }
2300 
2301   // If the compiler was not able to prove that exact type of the source or the destination
2302   // of the arraycopy is an array type, check at runtime if the source or the destination is
2303   // an instance type.
2304   if (flags & LIR_OpArrayCopy::type_check) {
2305     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2306       __ load_klass(tmp, dst);
2307       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2308       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2309       __ br(Assembler::GE, *stub->entry());
2310     }
2311 

2332     __ br(Assembler::LT, *stub->entry());
2333   }
2334 
2335   if (flags & LIR_OpArrayCopy::src_range_check) {
2336     __ addw(tmp, src_pos, length);
2337     __ ldrw(rscratch1, src_length_addr);
2338     __ cmpw(tmp, rscratch1);
2339     __ br(Assembler::HI, *stub->entry());
2340   }
2341   if (flags & LIR_OpArrayCopy::dst_range_check) {
2342     __ addw(tmp, dst_pos, length);
2343     __ ldrw(rscratch1, dst_length_addr);
2344     __ cmpw(tmp, rscratch1);
2345     __ br(Assembler::HI, *stub->entry());
2346   }
2347 
2348   if (flags & LIR_OpArrayCopy::type_check) {
2349     // We don't know the array types are compatible
2350     if (basic_type != T_OBJECT) {
2351       // Simple test for basic type arrays
2352       __ cmp_klass(src, dst, tmp, rscratch1);








2353       __ br(Assembler::NE, *stub->entry());
2354     } else {
2355       // For object arrays, if src is a sub class of dst then we can
2356       // safely do the copy.
2357       Label cont, slow;
2358 
2359 #define PUSH(r1, r2)                                    \
2360       stp(r1, r2, __ pre(sp, -2 * wordSize));
2361 
2362 #define POP(r1, r2)                                     \
2363       ldp(r1, r2, __ post(sp, 2 * wordSize));
2364 
2365       __ PUSH(src, dst);
2366 
2367       __ load_klass(src, src);
2368       __ load_klass(dst, dst);
2369 
2370       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2371 
2372       __ PUSH(src, dst);

2454       }
2455 
2456       __ b(*stub->entry());
2457 
2458       __ bind(cont);
2459       __ POP(src, dst);
2460     }
2461   }
2462 
2463 #ifdef ASSERT
2464   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2465     // Sanity check the known type with the incoming class.  For the
2466     // primitive case the types must match exactly with src.klass and
2467     // dst.klass each exactly matching the default type.  For the
2468     // object array case, if no type check is needed then either the
2469     // dst type is exactly the expected type and the src type is a
2470     // subtype which we can't check or src is the same array as dst
2471     // but not necessarily exactly of type default_type.
2472     Label known_ok, halt;
2473     __ mov_metadata(tmp, default_type->constant_encoding());



2474 
2475     if (basic_type != T_OBJECT) {
2476       __ cmp_klass(dst, tmp, rscratch1);







2477       __ br(Assembler::NE, halt);
2478       __ cmp_klass(src, tmp, rscratch1);






2479       __ br(Assembler::EQ, known_ok);
2480     } else {
2481       __ cmp_klass(dst, tmp, rscratch1);






2482       __ br(Assembler::EQ, known_ok);
2483       __ cmp(src, dst);
2484       __ br(Assembler::EQ, known_ok);
2485     }
2486     __ bind(halt);
2487     __ stop("incorrect type information in arraycopy");
2488     __ bind(known_ok);
2489   }
2490 #endif
2491 
2492 #ifndef PRODUCT
2493   if (PrintC1Statistics) {
2494     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2495   }
2496 #endif
2497 
2498   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2499   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2500   assert_different_registers(c_rarg0, dst, dst_pos, length);
2501   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));

2542     // done
2543   } else if (op->code() == lir_unlock) {
2544     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2545     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2546   } else {
2547     Unimplemented();
2548   }
2549   __ bind(*op->stub()->continuation());
2550 }
2551 
2552 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2553   Register obj = op->obj()->as_pointer_register();
2554   Register result = op->result_opr()->as_pointer_register();
2555 
2556   CodeEmitInfo* info = op->info();
2557   if (info != nullptr) {
2558     add_debug_info_for_null_check_here(info);
2559   }
2560 
2561   if (UseCompressedClassPointers) {
2562     if (UseCompactObjectHeaders) {
2563       // Check if we can take the (common) fast path, if obj is unlocked.
2564       __ ldr(result, Address(obj, oopDesc::mark_offset_in_bytes()));
2565       __ tst(result, markWord::monitor_value);
2566       __ br(Assembler::NE, *op->stub()->entry());
2567       __ bind(*op->stub()->continuation());
2568 
2569       // Shift to get proper narrow Klass*.
2570       __ lsr(result, result, markWord::klass_shift);
2571     } else {
2572       __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2573     }
2574     __ decode_klass_not_null(result);
2575   } else {
2576     __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2577   }
2578 }
2579 
2580 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2581   ciMethod* method = op->profiled_method();
2582   int bci          = op->profiled_bci();
2583   ciMethod* callee = op->profiled_callee();
2584 
2585   // Update counter for all call types
2586   ciMethodData* md = method->method_data_or_null();
2587   assert(md != nullptr, "Sanity");
2588   ciProfileData* data = md->bci_to_data(bci);
2589   assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2590   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2591   Register mdo  = op->mdo()->as_register();
2592   __ mov_metadata(mdo, md->constant_encoding());
2593   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
< prev index next >