< prev index next >

src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp

Print this page

 249   // locals[nlocals-1..0]
 250   // monitors[0..number_of_locks]
 251   //
 252   // locals is a direct copy of the interpreter frame so in the osr buffer
 253   // so first slot in the local array is the last local from the interpreter
 254   // and last slot is local[0] (receiver) from the interpreter
 255   //
 256   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 257   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 258   // in the interpreter frame (the method lock if a sync method)
 259 
 260   // Initialize monitors in the compiled activation.
 261   //   r2: pointer to osr buffer
 262   //
 263   // All other registers are dead at this point and the locals will be
 264   // copied into place by code emitted in the IR.
 265 
 266   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 267   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 268     int monitor_offset = BytesPerWord * method()->max_locals() +
 269       (2 * BytesPerWord) * (number_of_locks - 1);
 270     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 271     // the OSR buffer using 2 word entries: first the lock and then
 272     // the oop.
 273     for (int i = 0; i < number_of_locks; i++) {
 274       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 275 #ifdef ASSERT
 276       // verify the interpreter's monitor has a non-null object
 277       {
 278         Label L;
 279         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 280         __ cbnz(rscratch1, L);
 281         __ stop("locked object is NULL");
 282         __ bind(L);
 283       }
 284 #endif
 285       __ ldp(r19, r20, Address(OSR_buf, slot_offset));
 286       __ str(r19, frame_map()->address_for_monitor_lock(i));
 287       __ str(r20, frame_map()->address_for_monitor_object(i));
 288     }
 289   }
 290 }
 291 
 292 
 293 // inline cache check; done before the frame is built.
 294 int LIR_Assembler::check_icache() {
 295   Register receiver = FrameMap::receiver_opr->as_register();
 296   Register ic_klass = IC_Klass;
 297   int start_offset = __ offset();
 298   __ inline_cache_check(receiver, ic_klass);
 299 
 300   // if icache check fails, then jump to runtime routine
 301   // Note: RECEIVER must still contain the receiver!
 302   Label dont;
 303   __ br(Assembler::EQ, dont);
 304   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 305 
 306   // We align the verified entry point unless the method body
 307   // (including its inline cache check) will fit in a single 64-byte

 413   }
 414 #endif
 415 
 416   int offset = code_offset();
 417 
 418   // Fetch the exception from TLS and clear out exception related thread state
 419   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 420   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 421   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 422 
 423   __ bind(_unwind_handler_entry);
 424   __ verify_not_null_oop(r0);
 425   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 426     __ mov(r19, r0);  // Preserve the exception
 427   }
 428 
 429   // Perform needed unlocking
 430   MonitorExitStub* stub = NULL;
 431   if (method()->is_synchronized()) {
 432     monitor_address(0, FrameMap::r0_opr);
 433     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);

 434     if (UseHeavyMonitors) {
 435       __ b(*stub->entry());
 436     } else {
 437       __ unlock_object(r5, r4, r0, *stub->entry());
 438     }
 439     __ bind(*stub->continuation());
 440   }
 441 
 442   if (compilation()->env()->dtrace_method_probes()) {
 443     __ mov(c_rarg0, rthread);
 444     __ mov_metadata(c_rarg1, method()->constant_encoding());
 445     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 446   }
 447 
 448   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 449     __ mov(r0, r19);  // Restore the exception
 450   }
 451 
 452   // remove the activation and dispatch to the unwind handler
 453   __ block_comment("remove_frame and dispatch to the unwind handler");

1212       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1213       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1214     __ b(*op->stub()->entry());
1215   } else {
1216     Register tmp1 = op->tmp1()->as_register();
1217     Register tmp2 = op->tmp2()->as_register();
1218     Register tmp3 = op->tmp3()->as_register();
1219     if (len == tmp1) {
1220       tmp1 = tmp3;
1221     } else if (len == tmp2) {
1222       tmp2 = tmp3;
1223     } else if (len == tmp3) {
1224       // everything is ok
1225     } else {
1226       __ mov(tmp3, len);
1227     }
1228     __ allocate_array(op->obj()->as_register(),
1229                       len,
1230                       tmp1,
1231                       tmp2,
1232                       arrayOopDesc::header_size(op->type()),
1233                       array_element_size(op->type()),
1234                       op->klass()->as_register(),
1235                       *op->stub()->entry());
1236   }
1237   __ bind(*op->stub()->continuation());
1238 }
1239 
1240 void LIR_Assembler::type_profile_helper(Register mdo,
1241                                         ciMethodData *md, ciProfileData *data,
1242                                         Register recv, Label* update_done) {
1243   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1244     Label next_test;
1245     // See if the receiver is receiver[n].
1246     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1247     __ ldr(rscratch1, Address(rscratch2));
1248     __ cmp(recv, rscratch1);
1249     __ br(Assembler::NE, next_test);
1250     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1251     __ addptr(data_addr, DataLayout::counter_increment);
1252     __ b(*update_done);

2270 
2271     // r0 is -1^K where K == partial copied count
2272     __ eonw(rscratch1, r0, zr);
2273     // adjust length down and src/end pos up by partial copied count
2274     __ subw(length, length, rscratch1);
2275     __ addw(src_pos, src_pos, rscratch1);
2276     __ addw(dst_pos, dst_pos, rscratch1);
2277     __ b(*stub->entry());
2278 
2279     __ bind(*stub->continuation());
2280     return;
2281   }
2282 
2283   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2284 
2285   int elem_size = type2aelembytes(basic_type);
2286   int scale = exact_log2(elem_size);
2287 
2288   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2289   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2290   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2291   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2292 
2293   // test for NULL
2294   if (flags & LIR_OpArrayCopy::src_null_check) {
2295     __ cbz(src, *stub->entry());
2296   }
2297   if (flags & LIR_OpArrayCopy::dst_null_check) {
2298     __ cbz(dst, *stub->entry());
2299   }
2300 
2301   // If the compiler was not able to prove that exact type of the source or the destination
2302   // of the arraycopy is an array type, check at runtime if the source or the destination is
2303   // an instance type.
2304   if (flags & LIR_OpArrayCopy::type_check) {
2305     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2306       __ load_klass(tmp, dst);
2307       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2308       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2309       __ br(Assembler::GE, *stub->entry());
2310     }
2311 

2332     __ br(Assembler::LT, *stub->entry());
2333   }
2334 
2335   if (flags & LIR_OpArrayCopy::src_range_check) {
2336     __ addw(tmp, src_pos, length);
2337     __ ldrw(rscratch1, src_length_addr);
2338     __ cmpw(tmp, rscratch1);
2339     __ br(Assembler::HI, *stub->entry());
2340   }
2341   if (flags & LIR_OpArrayCopy::dst_range_check) {
2342     __ addw(tmp, dst_pos, length);
2343     __ ldrw(rscratch1, dst_length_addr);
2344     __ cmpw(tmp, rscratch1);
2345     __ br(Assembler::HI, *stub->entry());
2346   }
2347 
2348   if (flags & LIR_OpArrayCopy::type_check) {
2349     // We don't know the array types are compatible
2350     if (basic_type != T_OBJECT) {
2351       // Simple test for basic type arrays
2352       if (UseCompressedClassPointers) {
2353         __ ldrw(tmp, src_klass_addr);
2354         __ ldrw(rscratch1, dst_klass_addr);
2355         __ cmpw(tmp, rscratch1);
2356       } else {
2357         __ ldr(tmp, src_klass_addr);
2358         __ ldr(rscratch1, dst_klass_addr);
2359         __ cmp(tmp, rscratch1);
2360       }
2361       __ br(Assembler::NE, *stub->entry());
2362     } else {
2363       // For object arrays, if src is a sub class of dst then we can
2364       // safely do the copy.
2365       Label cont, slow;
2366 
2367 #define PUSH(r1, r2)                                    \
2368       stp(r1, r2, __ pre(sp, -2 * wordSize));
2369 
2370 #define POP(r1, r2)                                     \
2371       ldp(r1, r2, __ post(sp, 2 * wordSize));
2372 
2373       __ PUSH(src, dst);
2374 
2375       __ load_klass(src, src);
2376       __ load_klass(dst, dst);


2377 
2378       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2379 
2380       __ PUSH(src, dst);
2381       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2382       __ POP(src, dst);
2383 
2384       __ cbnz(src, cont);
2385 
2386       __ bind(slow);
2387       __ POP(src, dst);
2388 
2389       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2390       if (copyfunc_addr != NULL) { // use stub if available
2391         // src is not a sub class of dst so we have to do a
2392         // per-element check.
2393 
2394         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2395         if ((flags & mask) != mask) {
2396           // Check that at least both of them object arrays.

2466       __ bind(cont);
2467       __ POP(src, dst);
2468     }
2469   }
2470 
2471 #ifdef ASSERT
2472   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2473     // Sanity check the known type with the incoming class.  For the
2474     // primitive case the types must match exactly with src.klass and
2475     // dst.klass each exactly matching the default type.  For the
2476     // object array case, if no type check is needed then either the
2477     // dst type is exactly the expected type and the src type is a
2478     // subtype which we can't check or src is the same array as dst
2479     // but not necessarily exactly of type default_type.
2480     Label known_ok, halt;
2481     __ mov_metadata(tmp, default_type->constant_encoding());
2482     if (UseCompressedClassPointers) {
2483       __ encode_klass_not_null(tmp);
2484     }
2485 

2486     if (basic_type != T_OBJECT) {
2487 
2488       if (UseCompressedClassPointers) {
2489         __ ldrw(rscratch1, dst_klass_addr);
2490         __ cmpw(tmp, rscratch1);
2491       } else {
2492         __ ldr(rscratch1, dst_klass_addr);
2493         __ cmp(tmp, rscratch1);
2494       }
2495       __ br(Assembler::NE, halt);
2496       if (UseCompressedClassPointers) {
2497         __ ldrw(rscratch1, src_klass_addr);
2498         __ cmpw(tmp, rscratch1);
2499       } else {
2500         __ ldr(rscratch1, src_klass_addr);
2501         __ cmp(tmp, rscratch1);
2502       }
2503       __ br(Assembler::EQ, known_ok);
2504     } else {
2505       if (UseCompressedClassPointers) {
2506         __ ldrw(rscratch1, dst_klass_addr);
2507         __ cmpw(tmp, rscratch1);
2508       } else {
2509         __ ldr(rscratch1, dst_klass_addr);
2510         __ cmp(tmp, rscratch1);
2511       }
2512       __ br(Assembler::EQ, known_ok);
2513       __ cmp(src, dst);
2514       __ br(Assembler::EQ, known_ok);
2515     }
2516     __ bind(halt);
2517     __ stop("incorrect type information in arraycopy");
2518     __ bind(known_ok);
2519   }
2520 #endif
2521 
2522 #ifndef PRODUCT
2523   if (PrintC1Statistics) {
2524     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2525   }
2526 #endif
2527 
2528   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2529   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2530   assert_different_registers(c_rarg0, dst, dst_pos, length);
2531   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));

2546    __ call_VM_leaf(entry, 3);
2547  }
2548 
2549   __ bind(*stub->continuation());
2550 }
2551 
2552 
2553 
2554 
2555 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2556   Register obj = op->obj_opr()->as_register();  // may not be an oop
2557   Register hdr = op->hdr_opr()->as_register();
2558   Register lock = op->lock_opr()->as_register();
2559   if (UseHeavyMonitors) {
2560     if (op->info() != NULL) {
2561       add_debug_info_for_null_check_here(op->info());
2562       __ null_check(obj, -1);
2563     }
2564     __ b(*op->stub()->entry());
2565   } else if (op->code() == lir_lock) {
2566     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2567     // add debug info for NullPointerException only if one is possible
2568     int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
2569     if (op->info() != NULL) {
2570       add_debug_info_for_null_check(null_check_offset, op->info());
2571     }
2572     // done
2573   } else if (op->code() == lir_unlock) {
2574     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2575     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2576   } else {
2577     Unimplemented();
2578   }
2579   __ bind(*op->stub()->continuation());
2580 }
2581 
2582 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2583   Register obj = op->obj()->as_pointer_register();
2584   Register result = op->result_opr()->as_pointer_register();

2585 
2586   CodeEmitInfo* info = op->info();
2587   if (info != NULL) {
2588     add_debug_info_for_null_check_here(info);
2589   }
2590 
2591   if (UseCompressedClassPointers) {
2592     __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2593     __ decode_klass_not_null(result);
2594   } else {
2595     __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2596   }








2597 }
2598 
2599 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2600   ciMethod* method = op->profiled_method();
2601   int bci          = op->profiled_bci();
2602   ciMethod* callee = op->profiled_callee();
2603 
2604   // Update counter for all call types
2605   ciMethodData* md = method->method_data_or_null();
2606   assert(md != NULL, "Sanity");
2607   ciProfileData* data = md->bci_to_data(bci);
2608   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2609   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2610   Register mdo  = op->mdo()->as_register();
2611   __ mov_metadata(mdo, md->constant_encoding());
2612   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2613   // Perform additional virtual call profiling for invokevirtual and
2614   // invokeinterface bytecodes
2615   if (op->should_profile_receiver_type()) {
2616     assert(op->recv()->is_single_cpu(), "recv must be allocated");

2637       }
2638 
2639       // Receiver type not found in profile data; select an empty slot
2640 
2641       // Note that this is less efficient than it should be because it
2642       // always does a write to the receiver part of the
2643       // VirtualCallData rather than just the first time
2644       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2645         ciKlass* receiver = vc_data->receiver(i);
2646         if (receiver == NULL) {
2647           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2648           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2649           __ lea(rscratch2, recv_addr);
2650           __ str(rscratch1, Address(rscratch2));
2651           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2652           __ addptr(data_addr, DataLayout::counter_increment);
2653           return;
2654         }
2655       }
2656     } else {
2657       __ load_klass(recv, recv);

2658       Label update_done;
2659       type_profile_helper(mdo, md, data, recv, &update_done);
2660       // Receiver did not match any saved receiver and there is no empty row for it.
2661       // Increment total counter to indicate polymorphic case.
2662       __ addptr(counter_addr, DataLayout::counter_increment);
2663 
2664       __ bind(update_done);
2665     }
2666   } else {
2667     // Static call
2668     __ addptr(counter_addr, DataLayout::counter_increment);
2669   }
2670 }
2671 
2672 
2673 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2674   Unimplemented();
2675 }
2676 
2677 
2678 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2679   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2680 }
2681 
2682 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2683   assert(op->crc()->is_single_cpu(),  "crc must be register");
2684   assert(op->val()->is_single_cpu(),  "byte value must be register");
2685   assert(op->result_opr()->is_single_cpu(), "result must be register");
2686   Register crc = op->crc()->as_register();
2687   Register val = op->val()->as_register();
2688   Register res = op->result_opr()->as_register();
2689 
2690   assert_different_registers(val, crc, res);
2691   uint64_t offset;
2692   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2693   __ add(res, res, offset);
2694 
2695   __ mvnw(crc, crc); // ~crc
2696   __ update_byte_crc32(crc, val, res);
2697   __ mvnw(res, crc); // ~crc
2698 }
2699 

2731     }
2732     if (do_update) {
2733 #ifndef ASSERT
2734       __ b(next);
2735     }
2736 #else
2737       __ b(next);
2738     }
2739   } else {
2740     __ cbnz(tmp, update);
2741     __ stop("unexpected null obj");
2742 #endif
2743   }
2744 
2745   __ bind(update);
2746 
2747   if (do_update) {
2748 #ifdef ASSERT
2749     if (exact_klass != NULL) {
2750       Label ok;
2751       __ load_klass(tmp, tmp);

2752       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2753       __ eor(rscratch1, tmp, rscratch1);
2754       __ cbz(rscratch1, ok);
2755       __ stop("exact klass and actual klass differ");
2756       __ bind(ok);
2757     }
2758 #endif
2759     if (!no_conflict) {
2760       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2761         if (exact_klass != NULL) {
2762           __ mov_metadata(tmp, exact_klass->constant_encoding());
2763         } else {
2764           __ load_klass(tmp, tmp);

2765         }
2766 
2767         __ ldr(rscratch2, mdo_addr);
2768         __ eor(tmp, tmp, rscratch2);
2769         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2770         // klass seen before, nothing to do. The unknown bit may have been
2771         // set already but no need to check.
2772         __ cbz(rscratch1, next);
2773 
2774         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2775 
2776         if (TypeEntries::is_type_none(current_klass)) {
2777           __ cbz(rscratch2, none);
2778           __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2779           __ br(Assembler::EQ, none);
2780           // There is a chance that the checks above (re-reading profiling
2781           // data from memory) fail if another thread has just set the
2782           // profiling to this obj's klass
2783           __ dmb(Assembler::ISHLD);
2784           __ ldr(rscratch2, mdo_addr);

 249   // locals[nlocals-1..0]
 250   // monitors[0..number_of_locks]
 251   //
 252   // locals is a direct copy of the interpreter frame so in the osr buffer
 253   // so first slot in the local array is the last local from the interpreter
 254   // and last slot is local[0] (receiver) from the interpreter
 255   //
 256   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 257   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 258   // in the interpreter frame (the method lock if a sync method)
 259 
 260   // Initialize monitors in the compiled activation.
 261   //   r2: pointer to osr buffer
 262   //
 263   // All other registers are dead at this point and the locals will be
 264   // copied into place by code emitted in the IR.
 265 
 266   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 267   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 268     int monitor_offset = BytesPerWord * method()->max_locals() +
 269       BytesPerWord * (number_of_locks - 1);



 270     for (int i = 0; i < number_of_locks; i++) {
 271       int slot_offset = monitor_offset - (i * BytesPerWord);
 272 #ifdef ASSERT
 273       // verify the interpreter's monitor has a non-null object
 274       {
 275         Label L;
 276         __ ldr(rscratch1, Address(OSR_buf, slot_offset));
 277         __ cbnz(rscratch1, L);
 278         __ stop("locked object is NULL");
 279         __ bind(L);
 280       }
 281 #endif
 282       __ ldr(r19, Address(OSR_buf, slot_offset));
 283       __ str(r19, frame_map()->address_for_monitor_object(i));

 284     }
 285   }
 286 }
 287 
 288 
 289 // inline cache check; done before the frame is built.
 290 int LIR_Assembler::check_icache() {
 291   Register receiver = FrameMap::receiver_opr->as_register();
 292   Register ic_klass = IC_Klass;
 293   int start_offset = __ offset();
 294   __ inline_cache_check(receiver, ic_klass);
 295 
 296   // if icache check fails, then jump to runtime routine
 297   // Note: RECEIVER must still contain the receiver!
 298   Label dont;
 299   __ br(Assembler::EQ, dont);
 300   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 301 
 302   // We align the verified entry point unless the method body
 303   // (including its inline cache check) will fit in a single 64-byte

 409   }
 410 #endif
 411 
 412   int offset = code_offset();
 413 
 414   // Fetch the exception from TLS and clear out exception related thread state
 415   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 416   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 417   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 418 
 419   __ bind(_unwind_handler_entry);
 420   __ verify_not_null_oop(r0);
 421   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 422     __ mov(r19, r0);  // Preserve the exception
 423   }
 424 
 425   // Perform needed unlocking
 426   MonitorExitStub* stub = NULL;
 427   if (method()->is_synchronized()) {
 428     monitor_address(0, FrameMap::r0_opr);
 429     __ ldr(r4, Address(r0, BasicObjectLock::obj_offset_in_bytes()));
 430     stub = new MonitorExitStub(FrameMap::r4_opr);
 431     if (UseHeavyMonitors) {
 432       __ b(*stub->entry());
 433     } else {
 434       __ unlock_object(r5, r4, r0, *stub->entry());
 435     }
 436     __ bind(*stub->continuation());
 437   }
 438 
 439   if (compilation()->env()->dtrace_method_probes()) {
 440     __ mov(c_rarg0, rthread);
 441     __ mov_metadata(c_rarg1, method()->constant_encoding());
 442     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 443   }
 444 
 445   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 446     __ mov(r0, r19);  // Restore the exception
 447   }
 448 
 449   // remove the activation and dispatch to the unwind handler
 450   __ block_comment("remove_frame and dispatch to the unwind handler");

1209       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1210       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1211     __ b(*op->stub()->entry());
1212   } else {
1213     Register tmp1 = op->tmp1()->as_register();
1214     Register tmp2 = op->tmp2()->as_register();
1215     Register tmp3 = op->tmp3()->as_register();
1216     if (len == tmp1) {
1217       tmp1 = tmp3;
1218     } else if (len == tmp2) {
1219       tmp2 = tmp3;
1220     } else if (len == tmp3) {
1221       // everything is ok
1222     } else {
1223       __ mov(tmp3, len);
1224     }
1225     __ allocate_array(op->obj()->as_register(),
1226                       len,
1227                       tmp1,
1228                       tmp2,
1229                       arrayOopDesc::base_offset_in_bytes(op->type()),
1230                       array_element_size(op->type()),
1231                       op->klass()->as_register(),
1232                       *op->stub()->entry());
1233   }
1234   __ bind(*op->stub()->continuation());
1235 }
1236 
1237 void LIR_Assembler::type_profile_helper(Register mdo,
1238                                         ciMethodData *md, ciProfileData *data,
1239                                         Register recv, Label* update_done) {
1240   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1241     Label next_test;
1242     // See if the receiver is receiver[n].
1243     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1244     __ ldr(rscratch1, Address(rscratch2));
1245     __ cmp(recv, rscratch1);
1246     __ br(Assembler::NE, next_test);
1247     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1248     __ addptr(data_addr, DataLayout::counter_increment);
1249     __ b(*update_done);

2267 
2268     // r0 is -1^K where K == partial copied count
2269     __ eonw(rscratch1, r0, zr);
2270     // adjust length down and src/end pos up by partial copied count
2271     __ subw(length, length, rscratch1);
2272     __ addw(src_pos, src_pos, rscratch1);
2273     __ addw(dst_pos, dst_pos, rscratch1);
2274     __ b(*stub->entry());
2275 
2276     __ bind(*stub->continuation());
2277     return;
2278   }
2279 
2280   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2281 
2282   int elem_size = type2aelembytes(basic_type);
2283   int scale = exact_log2(elem_size);
2284 
2285   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2286   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());


2287 
2288   // test for NULL
2289   if (flags & LIR_OpArrayCopy::src_null_check) {
2290     __ cbz(src, *stub->entry());
2291   }
2292   if (flags & LIR_OpArrayCopy::dst_null_check) {
2293     __ cbz(dst, *stub->entry());
2294   }
2295 
2296   // If the compiler was not able to prove that exact type of the source or the destination
2297   // of the arraycopy is an array type, check at runtime if the source or the destination is
2298   // an instance type.
2299   if (flags & LIR_OpArrayCopy::type_check) {
2300     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2301       __ load_klass(tmp, dst);
2302       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2303       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2304       __ br(Assembler::GE, *stub->entry());
2305     }
2306 

2327     __ br(Assembler::LT, *stub->entry());
2328   }
2329 
2330   if (flags & LIR_OpArrayCopy::src_range_check) {
2331     __ addw(tmp, src_pos, length);
2332     __ ldrw(rscratch1, src_length_addr);
2333     __ cmpw(tmp, rscratch1);
2334     __ br(Assembler::HI, *stub->entry());
2335   }
2336   if (flags & LIR_OpArrayCopy::dst_range_check) {
2337     __ addw(tmp, dst_pos, length);
2338     __ ldrw(rscratch1, dst_length_addr);
2339     __ cmpw(tmp, rscratch1);
2340     __ br(Assembler::HI, *stub->entry());
2341   }
2342 
2343   if (flags & LIR_OpArrayCopy::type_check) {
2344     // We don't know the array types are compatible
2345     if (basic_type != T_OBJECT) {
2346       // Simple test for basic type arrays
2347       assert(UseCompressedClassPointers, "Lilliput");
2348       __ load_nklass(tmp, src);
2349       __ load_nklass(rscratch1, dst);
2350       __ cmpw(tmp, rscratch1);





2351       __ br(Assembler::NE, *stub->entry());
2352     } else {
2353       // For object arrays, if src is a sub class of dst then we can
2354       // safely do the copy.
2355       Label cont, slow;
2356 
2357 #define PUSH(r1, r2)                                    \
2358       stp(r1, r2, __ pre(sp, -2 * wordSize));
2359 
2360 #define POP(r1, r2)                                     \
2361       ldp(r1, r2, __ post(sp, 2 * wordSize));
2362 
2363       __ PUSH(src, dst);
2364 
2365       __ load_klass(tmp, src);
2366       __ mov(src, tmp);
2367       __ load_klass(tmp, dst);
2368       __ mov(dst, tmp);
2369 
2370       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2371 
2372       __ PUSH(src, dst);
2373       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2374       __ POP(src, dst);
2375 
2376       __ cbnz(src, cont);
2377 
2378       __ bind(slow);
2379       __ POP(src, dst);
2380 
2381       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2382       if (copyfunc_addr != NULL) { // use stub if available
2383         // src is not a sub class of dst so we have to do a
2384         // per-element check.
2385 
2386         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2387         if ((flags & mask) != mask) {
2388           // Check that at least both of them object arrays.

2458       __ bind(cont);
2459       __ POP(src, dst);
2460     }
2461   }
2462 
2463 #ifdef ASSERT
2464   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2465     // Sanity check the known type with the incoming class.  For the
2466     // primitive case the types must match exactly with src.klass and
2467     // dst.klass each exactly matching the default type.  For the
2468     // object array case, if no type check is needed then either the
2469     // dst type is exactly the expected type and the src type is a
2470     // subtype which we can't check or src is the same array as dst
2471     // but not necessarily exactly of type default_type.
2472     Label known_ok, halt;
2473     __ mov_metadata(tmp, default_type->constant_encoding());
2474     if (UseCompressedClassPointers) {
2475       __ encode_klass_not_null(tmp);
2476     }
2477 
2478     assert(UseCompressedClassPointers, "Lilliput");
2479     if (basic_type != T_OBJECT) {
2480       __ load_nklass(rscratch1, dst);
2481       __ cmpw(tmp, rscratch1);






2482       __ br(Assembler::NE, halt);
2483       __ load_nklass(rscratch1, src);
2484       __ cmpw(tmp, rscratch1);





2485       __ br(Assembler::EQ, known_ok);
2486     } else {
2487       __ load_nklass(rscratch1, dst);
2488       __ cmpw(tmp, rscratch1);





2489       __ br(Assembler::EQ, known_ok);
2490       __ cmp(src, dst);
2491       __ br(Assembler::EQ, known_ok);
2492     }
2493     __ bind(halt);
2494     __ stop("incorrect type information in arraycopy");
2495     __ bind(known_ok);
2496   }
2497 #endif
2498 
2499 #ifndef PRODUCT
2500   if (PrintC1Statistics) {
2501     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2502   }
2503 #endif
2504 
2505   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2506   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2507   assert_different_registers(c_rarg0, dst, dst_pos, length);
2508   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));

2523    __ call_VM_leaf(entry, 3);
2524  }
2525 
2526   __ bind(*stub->continuation());
2527 }
2528 
2529 
2530 
2531 
2532 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2533   Register obj = op->obj_opr()->as_register();  // may not be an oop
2534   Register hdr = op->hdr_opr()->as_register();
2535   Register lock = op->lock_opr()->as_register();
2536   if (UseHeavyMonitors) {
2537     if (op->info() != NULL) {
2538       add_debug_info_for_null_check_here(op->info());
2539       __ null_check(obj, -1);
2540     }
2541     __ b(*op->stub()->entry());
2542   } else if (op->code() == lir_lock) {

2543     // add debug info for NullPointerException only if one is possible
2544     int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
2545     if (op->info() != NULL) {
2546       add_debug_info_for_null_check(null_check_offset, op->info());
2547     }
2548     // done
2549   } else if (op->code() == lir_unlock) {

2550     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2551   } else {
2552     Unimplemented();
2553   }
2554   __ bind(*op->stub()->continuation());
2555 }
2556 
2557 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2558   Register obj = op->obj()->as_pointer_register();
2559   Register result = op->result_opr()->as_pointer_register();
2560   Register tmp = rscratch1;
2561 
2562   CodeEmitInfo* info = op->info();
2563   if (info != NULL) {
2564     add_debug_info_for_null_check_here(info);
2565   }
2566 
2567   assert(UseCompressedClassPointers, "expects UseCompressedClassPointers");
2568 
2569   // Check if we can take the (common) fast path, if obj is unlocked.
2570   __ ldr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
2571   __ eor(tmp, tmp, markWord::unlocked_value);
2572   __ tst(tmp, markWord::lock_mask_in_place);
2573   __ br(Assembler::NE, *op->stub()->entry());
2574 
2575   // Fast-path: shift and decode Klass*.
2576   __ mov(result, tmp);
2577   __ lsr(result, result, markWord::klass_shift);
2578 
2579   __ bind(*op->stub()->continuation());
2580   __ decode_klass_not_null(result);
2581 }
2582 
2583 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2584   ciMethod* method = op->profiled_method();
2585   int bci          = op->profiled_bci();
2586   ciMethod* callee = op->profiled_callee();
2587 
2588   // Update counter for all call types
2589   ciMethodData* md = method->method_data_or_null();
2590   assert(md != NULL, "Sanity");
2591   ciProfileData* data = md->bci_to_data(bci);
2592   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2593   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2594   Register mdo  = op->mdo()->as_register();
2595   __ mov_metadata(mdo, md->constant_encoding());
2596   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2597   // Perform additional virtual call profiling for invokevirtual and
2598   // invokeinterface bytecodes
2599   if (op->should_profile_receiver_type()) {
2600     assert(op->recv()->is_single_cpu(), "recv must be allocated");

2621       }
2622 
2623       // Receiver type not found in profile data; select an empty slot
2624 
2625       // Note that this is less efficient than it should be because it
2626       // always does a write to the receiver part of the
2627       // VirtualCallData rather than just the first time
2628       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2629         ciKlass* receiver = vc_data->receiver(i);
2630         if (receiver == NULL) {
2631           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2632           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2633           __ lea(rscratch2, recv_addr);
2634           __ str(rscratch1, Address(rscratch2));
2635           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2636           __ addptr(data_addr, DataLayout::counter_increment);
2637           return;
2638         }
2639       }
2640     } else {
2641       __ load_klass(rscratch1, recv);
2642       __ mov(recv, rscratch1);
2643       Label update_done;
2644       type_profile_helper(mdo, md, data, recv, &update_done);
2645       // Receiver did not match any saved receiver and there is no empty row for it.
2646       // Increment total counter to indicate polymorphic case.
2647       __ addptr(counter_addr, DataLayout::counter_increment);
2648 
2649       __ bind(update_done);
2650     }
2651   } else {
2652     // Static call
2653     __ addptr(counter_addr, DataLayout::counter_increment);
2654   }
2655 }
2656 
2657 
2658 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2659   Unimplemented();
2660 }
2661 
2662 
2663 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2664   __ lea(dst->as_register(), frame_map()->address_for_monitor_object(monitor_no));
2665 }
2666 
2667 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2668   assert(op->crc()->is_single_cpu(),  "crc must be register");
2669   assert(op->val()->is_single_cpu(),  "byte value must be register");
2670   assert(op->result_opr()->is_single_cpu(), "result must be register");
2671   Register crc = op->crc()->as_register();
2672   Register val = op->val()->as_register();
2673   Register res = op->result_opr()->as_register();
2674 
2675   assert_different_registers(val, crc, res);
2676   uint64_t offset;
2677   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2678   __ add(res, res, offset);
2679 
2680   __ mvnw(crc, crc); // ~crc
2681   __ update_byte_crc32(crc, val, res);
2682   __ mvnw(res, crc); // ~crc
2683 }
2684 

2716     }
2717     if (do_update) {
2718 #ifndef ASSERT
2719       __ b(next);
2720     }
2721 #else
2722       __ b(next);
2723     }
2724   } else {
2725     __ cbnz(tmp, update);
2726     __ stop("unexpected null obj");
2727 #endif
2728   }
2729 
2730   __ bind(update);
2731 
2732   if (do_update) {
2733 #ifdef ASSERT
2734     if (exact_klass != NULL) {
2735       Label ok;
2736       __ load_klass(rscratch1, tmp);
2737       __ mov(tmp, rscratch1);
2738       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2739       __ eor(rscratch1, tmp, rscratch1);
2740       __ cbz(rscratch1, ok);
2741       __ stop("exact klass and actual klass differ");
2742       __ bind(ok);
2743     }
2744 #endif
2745     if (!no_conflict) {
2746       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2747         if (exact_klass != NULL) {
2748           __ mov_metadata(tmp, exact_klass->constant_encoding());
2749         } else {
2750           __ load_klass(rscratch1, tmp);
2751           __ mov(tmp, rscratch1);
2752         }
2753 
2754         __ ldr(rscratch2, mdo_addr);
2755         __ eor(tmp, tmp, rscratch2);
2756         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2757         // klass seen before, nothing to do. The unknown bit may have been
2758         // set already but no need to check.
2759         __ cbz(rscratch1, next);
2760 
2761         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2762 
2763         if (TypeEntries::is_type_none(current_klass)) {
2764           __ cbz(rscratch2, none);
2765           __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2766           __ br(Assembler::EQ, none);
2767           // There is a chance that the checks above (re-reading profiling
2768           // data from memory) fail if another thread has just set the
2769           // profiling to this obj's klass
2770           __ dmb(Assembler::ISHLD);
2771           __ ldr(rscratch2, mdo_addr);
< prev index next >