249 // locals[nlocals-1..0]
250 // monitors[0..number_of_locks]
251 //
252 // locals is a direct copy of the interpreter frame so in the osr buffer
253 // so first slot in the local array is the last local from the interpreter
254 // and last slot is local[0] (receiver) from the interpreter
255 //
256 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
257 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
258 // in the interpreter frame (the method lock if a sync method)
259
260 // Initialize monitors in the compiled activation.
261 // r2: pointer to osr buffer
262 //
263 // All other registers are dead at this point and the locals will be
264 // copied into place by code emitted in the IR.
265
266 Register OSR_buf = osrBufferPointer()->as_pointer_register();
267 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
268 int monitor_offset = BytesPerWord * method()->max_locals() +
269 (2 * BytesPerWord) * (number_of_locks - 1);
270 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
271 // the OSR buffer using 2 word entries: first the lock and then
272 // the oop.
273 for (int i = 0; i < number_of_locks; i++) {
274 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
275 #ifdef ASSERT
276 // verify the interpreter's monitor has a non-null object
277 {
278 Label L;
279 __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
280 __ cbnz(rscratch1, L);
281 __ stop("locked object is NULL");
282 __ bind(L);
283 }
284 #endif
285 __ ldp(r19, r20, Address(OSR_buf, slot_offset));
286 __ str(r19, frame_map()->address_for_monitor_lock(i));
287 __ str(r20, frame_map()->address_for_monitor_object(i));
288 }
289 }
290 }
291
292
293 // inline cache check; done before the frame is built.
294 int LIR_Assembler::check_icache() {
295 Register receiver = FrameMap::receiver_opr->as_register();
296 Register ic_klass = IC_Klass;
297 int start_offset = __ offset();
298 __ inline_cache_check(receiver, ic_klass);
299
300 // if icache check fails, then jump to runtime routine
301 // Note: RECEIVER must still contain the receiver!
302 Label dont;
303 __ br(Assembler::EQ, dont);
304 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
305
306 // We align the verified entry point unless the method body
307 // (including its inline cache check) will fit in a single 64-byte
413 }
414 #endif
415
416 int offset = code_offset();
417
418 // Fetch the exception from TLS and clear out exception related thread state
419 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
420 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
421 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
422
423 __ bind(_unwind_handler_entry);
424 __ verify_not_null_oop(r0);
425 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
426 __ mov(r19, r0); // Preserve the exception
427 }
428
429 // Perform needed unlocking
430 MonitorExitStub* stub = NULL;
431 if (method()->is_synchronized()) {
432 monitor_address(0, FrameMap::r0_opr);
433 stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
434 if (UseHeavyMonitors) {
435 __ b(*stub->entry());
436 } else {
437 __ unlock_object(r5, r4, r0, *stub->entry());
438 }
439 __ bind(*stub->continuation());
440 }
441
442 if (compilation()->env()->dtrace_method_probes()) {
443 __ mov(c_rarg0, rthread);
444 __ mov_metadata(c_rarg1, method()->constant_encoding());
445 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
446 }
447
448 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
449 __ mov(r0, r19); // Restore the exception
450 }
451
452 // remove the activation and dispatch to the unwind handler
453 __ block_comment("remove_frame and dispatch to the unwind handler");
1212 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1213 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1214 __ b(*op->stub()->entry());
1215 } else {
1216 Register tmp1 = op->tmp1()->as_register();
1217 Register tmp2 = op->tmp2()->as_register();
1218 Register tmp3 = op->tmp3()->as_register();
1219 if (len == tmp1) {
1220 tmp1 = tmp3;
1221 } else if (len == tmp2) {
1222 tmp2 = tmp3;
1223 } else if (len == tmp3) {
1224 // everything is ok
1225 } else {
1226 __ mov(tmp3, len);
1227 }
1228 __ allocate_array(op->obj()->as_register(),
1229 len,
1230 tmp1,
1231 tmp2,
1232 arrayOopDesc::header_size(op->type()),
1233 array_element_size(op->type()),
1234 op->klass()->as_register(),
1235 *op->stub()->entry());
1236 }
1237 __ bind(*op->stub()->continuation());
1238 }
1239
1240 void LIR_Assembler::type_profile_helper(Register mdo,
1241 ciMethodData *md, ciProfileData *data,
1242 Register recv, Label* update_done) {
1243 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1244 Label next_test;
1245 // See if the receiver is receiver[n].
1246 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1247 __ ldr(rscratch1, Address(rscratch2));
1248 __ cmp(recv, rscratch1);
1249 __ br(Assembler::NE, next_test);
1250 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1251 __ addptr(data_addr, DataLayout::counter_increment);
1252 __ b(*update_done);
2270
2271 // r0 is -1^K where K == partial copied count
2272 __ eonw(rscratch1, r0, zr);
2273 // adjust length down and src/end pos up by partial copied count
2274 __ subw(length, length, rscratch1);
2275 __ addw(src_pos, src_pos, rscratch1);
2276 __ addw(dst_pos, dst_pos, rscratch1);
2277 __ b(*stub->entry());
2278
2279 __ bind(*stub->continuation());
2280 return;
2281 }
2282
2283 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2284
2285 int elem_size = type2aelembytes(basic_type);
2286 int scale = exact_log2(elem_size);
2287
2288 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2289 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2290 Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2291 Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2292
2293 // test for NULL
2294 if (flags & LIR_OpArrayCopy::src_null_check) {
2295 __ cbz(src, *stub->entry());
2296 }
2297 if (flags & LIR_OpArrayCopy::dst_null_check) {
2298 __ cbz(dst, *stub->entry());
2299 }
2300
2301 // If the compiler was not able to prove that exact type of the source or the destination
2302 // of the arraycopy is an array type, check at runtime if the source or the destination is
2303 // an instance type.
2304 if (flags & LIR_OpArrayCopy::type_check) {
2305 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2306 __ load_klass(tmp, dst);
2307 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2308 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2309 __ br(Assembler::GE, *stub->entry());
2310 }
2311
2332 __ br(Assembler::LT, *stub->entry());
2333 }
2334
2335 if (flags & LIR_OpArrayCopy::src_range_check) {
2336 __ addw(tmp, src_pos, length);
2337 __ ldrw(rscratch1, src_length_addr);
2338 __ cmpw(tmp, rscratch1);
2339 __ br(Assembler::HI, *stub->entry());
2340 }
2341 if (flags & LIR_OpArrayCopy::dst_range_check) {
2342 __ addw(tmp, dst_pos, length);
2343 __ ldrw(rscratch1, dst_length_addr);
2344 __ cmpw(tmp, rscratch1);
2345 __ br(Assembler::HI, *stub->entry());
2346 }
2347
2348 if (flags & LIR_OpArrayCopy::type_check) {
2349 // We don't know the array types are compatible
2350 if (basic_type != T_OBJECT) {
2351 // Simple test for basic type arrays
2352 if (UseCompressedClassPointers) {
2353 __ ldrw(tmp, src_klass_addr);
2354 __ ldrw(rscratch1, dst_klass_addr);
2355 __ cmpw(tmp, rscratch1);
2356 } else {
2357 __ ldr(tmp, src_klass_addr);
2358 __ ldr(rscratch1, dst_klass_addr);
2359 __ cmp(tmp, rscratch1);
2360 }
2361 __ br(Assembler::NE, *stub->entry());
2362 } else {
2363 // For object arrays, if src is a sub class of dst then we can
2364 // safely do the copy.
2365 Label cont, slow;
2366
2367 #define PUSH(r1, r2) \
2368 stp(r1, r2, __ pre(sp, -2 * wordSize));
2369
2370 #define POP(r1, r2) \
2371 ldp(r1, r2, __ post(sp, 2 * wordSize));
2372
2373 __ PUSH(src, dst);
2374
2375 __ load_klass(src, src);
2376 __ load_klass(dst, dst);
2377
2378 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2379
2380 __ PUSH(src, dst);
2381 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2382 __ POP(src, dst);
2383
2384 __ cbnz(src, cont);
2385
2386 __ bind(slow);
2387 __ POP(src, dst);
2388
2389 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2390 if (copyfunc_addr != NULL) { // use stub if available
2391 // src is not a sub class of dst so we have to do a
2392 // per-element check.
2393
2394 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2395 if ((flags & mask) != mask) {
2396 // Check that at least both of them object arrays.
2466 __ bind(cont);
2467 __ POP(src, dst);
2468 }
2469 }
2470
2471 #ifdef ASSERT
2472 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2473 // Sanity check the known type with the incoming class. For the
2474 // primitive case the types must match exactly with src.klass and
2475 // dst.klass each exactly matching the default type. For the
2476 // object array case, if no type check is needed then either the
2477 // dst type is exactly the expected type and the src type is a
2478 // subtype which we can't check or src is the same array as dst
2479 // but not necessarily exactly of type default_type.
2480 Label known_ok, halt;
2481 __ mov_metadata(tmp, default_type->constant_encoding());
2482 if (UseCompressedClassPointers) {
2483 __ encode_klass_not_null(tmp);
2484 }
2485
2486 if (basic_type != T_OBJECT) {
2487
2488 if (UseCompressedClassPointers) {
2489 __ ldrw(rscratch1, dst_klass_addr);
2490 __ cmpw(tmp, rscratch1);
2491 } else {
2492 __ ldr(rscratch1, dst_klass_addr);
2493 __ cmp(tmp, rscratch1);
2494 }
2495 __ br(Assembler::NE, halt);
2496 if (UseCompressedClassPointers) {
2497 __ ldrw(rscratch1, src_klass_addr);
2498 __ cmpw(tmp, rscratch1);
2499 } else {
2500 __ ldr(rscratch1, src_klass_addr);
2501 __ cmp(tmp, rscratch1);
2502 }
2503 __ br(Assembler::EQ, known_ok);
2504 } else {
2505 if (UseCompressedClassPointers) {
2506 __ ldrw(rscratch1, dst_klass_addr);
2507 __ cmpw(tmp, rscratch1);
2508 } else {
2509 __ ldr(rscratch1, dst_klass_addr);
2510 __ cmp(tmp, rscratch1);
2511 }
2512 __ br(Assembler::EQ, known_ok);
2513 __ cmp(src, dst);
2514 __ br(Assembler::EQ, known_ok);
2515 }
2516 __ bind(halt);
2517 __ stop("incorrect type information in arraycopy");
2518 __ bind(known_ok);
2519 }
2520 #endif
2521
2522 #ifndef PRODUCT
2523 if (PrintC1Statistics) {
2524 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2525 }
2526 #endif
2527
2528 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2529 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2530 assert_different_registers(c_rarg0, dst, dst_pos, length);
2531 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2546 __ call_VM_leaf(entry, 3);
2547 }
2548
2549 __ bind(*stub->continuation());
2550 }
2551
2552
2553
2554
2555 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2556 Register obj = op->obj_opr()->as_register(); // may not be an oop
2557 Register hdr = op->hdr_opr()->as_register();
2558 Register lock = op->lock_opr()->as_register();
2559 if (UseHeavyMonitors) {
2560 if (op->info() != NULL) {
2561 add_debug_info_for_null_check_here(op->info());
2562 __ null_check(obj, -1);
2563 }
2564 __ b(*op->stub()->entry());
2565 } else if (op->code() == lir_lock) {
2566 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2567 // add debug info for NullPointerException only if one is possible
2568 int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
2569 if (op->info() != NULL) {
2570 add_debug_info_for_null_check(null_check_offset, op->info());
2571 }
2572 // done
2573 } else if (op->code() == lir_unlock) {
2574 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2575 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2576 } else {
2577 Unimplemented();
2578 }
2579 __ bind(*op->stub()->continuation());
2580 }
2581
2582 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2583 Register obj = op->obj()->as_pointer_register();
2584 Register result = op->result_opr()->as_pointer_register();
2585
2586 CodeEmitInfo* info = op->info();
2587 if (info != NULL) {
2588 add_debug_info_for_null_check_here(info);
2589 }
2590
2591 if (UseCompressedClassPointers) {
2592 __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2593 __ decode_klass_not_null(result);
2594 } else {
2595 __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2596 }
2597 }
2598
2599 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2600 ciMethod* method = op->profiled_method();
2601 int bci = op->profiled_bci();
2602 ciMethod* callee = op->profiled_callee();
2603
2604 // Update counter for all call types
2605 ciMethodData* md = method->method_data_or_null();
2606 assert(md != NULL, "Sanity");
2607 ciProfileData* data = md->bci_to_data(bci);
2608 assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2609 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2610 Register mdo = op->mdo()->as_register();
2611 __ mov_metadata(mdo, md->constant_encoding());
2612 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2613 // Perform additional virtual call profiling for invokevirtual and
2614 // invokeinterface bytecodes
2615 if (op->should_profile_receiver_type()) {
2616 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2637 }
2638
2639 // Receiver type not found in profile data; select an empty slot
2640
2641 // Note that this is less efficient than it should be because it
2642 // always does a write to the receiver part of the
2643 // VirtualCallData rather than just the first time
2644 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2645 ciKlass* receiver = vc_data->receiver(i);
2646 if (receiver == NULL) {
2647 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2648 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2649 __ lea(rscratch2, recv_addr);
2650 __ str(rscratch1, Address(rscratch2));
2651 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2652 __ addptr(data_addr, DataLayout::counter_increment);
2653 return;
2654 }
2655 }
2656 } else {
2657 __ load_klass(recv, recv);
2658 Label update_done;
2659 type_profile_helper(mdo, md, data, recv, &update_done);
2660 // Receiver did not match any saved receiver and there is no empty row for it.
2661 // Increment total counter to indicate polymorphic case.
2662 __ addptr(counter_addr, DataLayout::counter_increment);
2663
2664 __ bind(update_done);
2665 }
2666 } else {
2667 // Static call
2668 __ addptr(counter_addr, DataLayout::counter_increment);
2669 }
2670 }
2671
2672
2673 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2674 Unimplemented();
2675 }
2676
2677
2678 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2679 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2680 }
2681
2682 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2683 assert(op->crc()->is_single_cpu(), "crc must be register");
2684 assert(op->val()->is_single_cpu(), "byte value must be register");
2685 assert(op->result_opr()->is_single_cpu(), "result must be register");
2686 Register crc = op->crc()->as_register();
2687 Register val = op->val()->as_register();
2688 Register res = op->result_opr()->as_register();
2689
2690 assert_different_registers(val, crc, res);
2691 uint64_t offset;
2692 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2693 __ add(res, res, offset);
2694
2695 __ mvnw(crc, crc); // ~crc
2696 __ update_byte_crc32(crc, val, res);
2697 __ mvnw(res, crc); // ~crc
2698 }
2699
2731 }
2732 if (do_update) {
2733 #ifndef ASSERT
2734 __ b(next);
2735 }
2736 #else
2737 __ b(next);
2738 }
2739 } else {
2740 __ cbnz(tmp, update);
2741 __ stop("unexpected null obj");
2742 #endif
2743 }
2744
2745 __ bind(update);
2746
2747 if (do_update) {
2748 #ifdef ASSERT
2749 if (exact_klass != NULL) {
2750 Label ok;
2751 __ load_klass(tmp, tmp);
2752 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2753 __ eor(rscratch1, tmp, rscratch1);
2754 __ cbz(rscratch1, ok);
2755 __ stop("exact klass and actual klass differ");
2756 __ bind(ok);
2757 }
2758 #endif
2759 if (!no_conflict) {
2760 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2761 if (exact_klass != NULL) {
2762 __ mov_metadata(tmp, exact_klass->constant_encoding());
2763 } else {
2764 __ load_klass(tmp, tmp);
2765 }
2766
2767 __ ldr(rscratch2, mdo_addr);
2768 __ eor(tmp, tmp, rscratch2);
2769 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2770 // klass seen before, nothing to do. The unknown bit may have been
2771 // set already but no need to check.
2772 __ cbz(rscratch1, next);
2773
2774 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2775
2776 if (TypeEntries::is_type_none(current_klass)) {
2777 __ cbz(rscratch2, none);
2778 __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2779 __ br(Assembler::EQ, none);
2780 // There is a chance that the checks above (re-reading profiling
2781 // data from memory) fail if another thread has just set the
2782 // profiling to this obj's klass
2783 __ dmb(Assembler::ISHLD);
2784 __ ldr(rscratch2, mdo_addr);
|
249 // locals[nlocals-1..0]
250 // monitors[0..number_of_locks]
251 //
252 // locals is a direct copy of the interpreter frame so in the osr buffer
253 // so first slot in the local array is the last local from the interpreter
254 // and last slot is local[0] (receiver) from the interpreter
255 //
256 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
257 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
258 // in the interpreter frame (the method lock if a sync method)
259
260 // Initialize monitors in the compiled activation.
261 // r2: pointer to osr buffer
262 //
263 // All other registers are dead at this point and the locals will be
264 // copied into place by code emitted in the IR.
265
266 Register OSR_buf = osrBufferPointer()->as_pointer_register();
267 { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
268 int monitor_offset = BytesPerWord * method()->max_locals() +
269 BytesPerWord * (number_of_locks - 1);
270 for (int i = 0; i < number_of_locks; i++) {
271 int slot_offset = monitor_offset - (i * BytesPerWord);
272 #ifdef ASSERT
273 // verify the interpreter's monitor has a non-null object
274 {
275 Label L;
276 __ ldr(rscratch1, Address(OSR_buf, slot_offset));
277 __ cbnz(rscratch1, L);
278 __ stop("locked object is NULL");
279 __ bind(L);
280 }
281 #endif
282 __ ldr(r19, Address(OSR_buf, slot_offset));
283 __ str(r19, frame_map()->address_for_monitor_object(i));
284 }
285 }
286 }
287
288
289 // inline cache check; done before the frame is built.
290 int LIR_Assembler::check_icache() {
291 Register receiver = FrameMap::receiver_opr->as_register();
292 Register ic_klass = IC_Klass;
293 int start_offset = __ offset();
294 __ inline_cache_check(receiver, ic_klass);
295
296 // if icache check fails, then jump to runtime routine
297 // Note: RECEIVER must still contain the receiver!
298 Label dont;
299 __ br(Assembler::EQ, dont);
300 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
301
302 // We align the verified entry point unless the method body
303 // (including its inline cache check) will fit in a single 64-byte
409 }
410 #endif
411
412 int offset = code_offset();
413
414 // Fetch the exception from TLS and clear out exception related thread state
415 __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
416 __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
417 __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
418
419 __ bind(_unwind_handler_entry);
420 __ verify_not_null_oop(r0);
421 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
422 __ mov(r19, r0); // Preserve the exception
423 }
424
425 // Perform needed unlocking
426 MonitorExitStub* stub = NULL;
427 if (method()->is_synchronized()) {
428 monitor_address(0, FrameMap::r0_opr);
429 __ ldr(r4, Address(r0, BasicObjectLock::obj_offset_in_bytes()));
430 stub = new MonitorExitStub(FrameMap::r4_opr);
431 if (UseHeavyMonitors) {
432 __ b(*stub->entry());
433 } else {
434 __ unlock_object(r5, r4, r0, *stub->entry());
435 }
436 __ bind(*stub->continuation());
437 }
438
439 if (compilation()->env()->dtrace_method_probes()) {
440 __ mov(c_rarg0, rthread);
441 __ mov_metadata(c_rarg1, method()->constant_encoding());
442 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
443 }
444
445 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
446 __ mov(r0, r19); // Restore the exception
447 }
448
449 // remove the activation and dispatch to the unwind handler
450 __ block_comment("remove_frame and dispatch to the unwind handler");
1209 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1210 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1211 __ b(*op->stub()->entry());
1212 } else {
1213 Register tmp1 = op->tmp1()->as_register();
1214 Register tmp2 = op->tmp2()->as_register();
1215 Register tmp3 = op->tmp3()->as_register();
1216 if (len == tmp1) {
1217 tmp1 = tmp3;
1218 } else if (len == tmp2) {
1219 tmp2 = tmp3;
1220 } else if (len == tmp3) {
1221 // everything is ok
1222 } else {
1223 __ mov(tmp3, len);
1224 }
1225 __ allocate_array(op->obj()->as_register(),
1226 len,
1227 tmp1,
1228 tmp2,
1229 arrayOopDesc::base_offset_in_bytes(op->type()),
1230 array_element_size(op->type()),
1231 op->klass()->as_register(),
1232 *op->stub()->entry());
1233 }
1234 __ bind(*op->stub()->continuation());
1235 }
1236
1237 void LIR_Assembler::type_profile_helper(Register mdo,
1238 ciMethodData *md, ciProfileData *data,
1239 Register recv, Label* update_done) {
1240 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1241 Label next_test;
1242 // See if the receiver is receiver[n].
1243 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1244 __ ldr(rscratch1, Address(rscratch2));
1245 __ cmp(recv, rscratch1);
1246 __ br(Assembler::NE, next_test);
1247 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1248 __ addptr(data_addr, DataLayout::counter_increment);
1249 __ b(*update_done);
2267
2268 // r0 is -1^K where K == partial copied count
2269 __ eonw(rscratch1, r0, zr);
2270 // adjust length down and src/end pos up by partial copied count
2271 __ subw(length, length, rscratch1);
2272 __ addw(src_pos, src_pos, rscratch1);
2273 __ addw(dst_pos, dst_pos, rscratch1);
2274 __ b(*stub->entry());
2275
2276 __ bind(*stub->continuation());
2277 return;
2278 }
2279
2280 assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2281
2282 int elem_size = type2aelembytes(basic_type);
2283 int scale = exact_log2(elem_size);
2284
2285 Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2286 Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2287
2288 // test for NULL
2289 if (flags & LIR_OpArrayCopy::src_null_check) {
2290 __ cbz(src, *stub->entry());
2291 }
2292 if (flags & LIR_OpArrayCopy::dst_null_check) {
2293 __ cbz(dst, *stub->entry());
2294 }
2295
2296 // If the compiler was not able to prove that exact type of the source or the destination
2297 // of the arraycopy is an array type, check at runtime if the source or the destination is
2298 // an instance type.
2299 if (flags & LIR_OpArrayCopy::type_check) {
2300 if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2301 __ load_klass(tmp, dst);
2302 __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2303 __ cmpw(rscratch1, Klass::_lh_neutral_value);
2304 __ br(Assembler::GE, *stub->entry());
2305 }
2306
2327 __ br(Assembler::LT, *stub->entry());
2328 }
2329
2330 if (flags & LIR_OpArrayCopy::src_range_check) {
2331 __ addw(tmp, src_pos, length);
2332 __ ldrw(rscratch1, src_length_addr);
2333 __ cmpw(tmp, rscratch1);
2334 __ br(Assembler::HI, *stub->entry());
2335 }
2336 if (flags & LIR_OpArrayCopy::dst_range_check) {
2337 __ addw(tmp, dst_pos, length);
2338 __ ldrw(rscratch1, dst_length_addr);
2339 __ cmpw(tmp, rscratch1);
2340 __ br(Assembler::HI, *stub->entry());
2341 }
2342
2343 if (flags & LIR_OpArrayCopy::type_check) {
2344 // We don't know the array types are compatible
2345 if (basic_type != T_OBJECT) {
2346 // Simple test for basic type arrays
2347 assert(UseCompressedClassPointers, "Lilliput");
2348 __ load_nklass(tmp, src);
2349 __ load_nklass(rscratch1, dst);
2350 __ cmpw(tmp, rscratch1);
2351 __ br(Assembler::NE, *stub->entry());
2352 } else {
2353 // For object arrays, if src is a sub class of dst then we can
2354 // safely do the copy.
2355 Label cont, slow;
2356
2357 #define PUSH(r1, r2) \
2358 stp(r1, r2, __ pre(sp, -2 * wordSize));
2359
2360 #define POP(r1, r2) \
2361 ldp(r1, r2, __ post(sp, 2 * wordSize));
2362
2363 __ PUSH(src, dst);
2364
2365 __ load_klass(tmp, src);
2366 __ mov(src, tmp);
2367 __ load_klass(tmp, dst);
2368 __ mov(dst, tmp);
2369
2370 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2371
2372 __ PUSH(src, dst);
2373 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2374 __ POP(src, dst);
2375
2376 __ cbnz(src, cont);
2377
2378 __ bind(slow);
2379 __ POP(src, dst);
2380
2381 address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2382 if (copyfunc_addr != NULL) { // use stub if available
2383 // src is not a sub class of dst so we have to do a
2384 // per-element check.
2385
2386 int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2387 if ((flags & mask) != mask) {
2388 // Check that at least both of them object arrays.
2458 __ bind(cont);
2459 __ POP(src, dst);
2460 }
2461 }
2462
2463 #ifdef ASSERT
2464 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2465 // Sanity check the known type with the incoming class. For the
2466 // primitive case the types must match exactly with src.klass and
2467 // dst.klass each exactly matching the default type. For the
2468 // object array case, if no type check is needed then either the
2469 // dst type is exactly the expected type and the src type is a
2470 // subtype which we can't check or src is the same array as dst
2471 // but not necessarily exactly of type default_type.
2472 Label known_ok, halt;
2473 __ mov_metadata(tmp, default_type->constant_encoding());
2474 if (UseCompressedClassPointers) {
2475 __ encode_klass_not_null(tmp);
2476 }
2477
2478 assert(UseCompressedClassPointers, "Lilliput");
2479 if (basic_type != T_OBJECT) {
2480 __ load_nklass(rscratch1, dst);
2481 __ cmpw(tmp, rscratch1);
2482 __ br(Assembler::NE, halt);
2483 __ load_nklass(rscratch1, src);
2484 __ cmpw(tmp, rscratch1);
2485 __ br(Assembler::EQ, known_ok);
2486 } else {
2487 __ load_nklass(rscratch1, dst);
2488 __ cmpw(tmp, rscratch1);
2489 __ br(Assembler::EQ, known_ok);
2490 __ cmp(src, dst);
2491 __ br(Assembler::EQ, known_ok);
2492 }
2493 __ bind(halt);
2494 __ stop("incorrect type information in arraycopy");
2495 __ bind(known_ok);
2496 }
2497 #endif
2498
2499 #ifndef PRODUCT
2500 if (PrintC1Statistics) {
2501 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2502 }
2503 #endif
2504
2505 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2506 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2507 assert_different_registers(c_rarg0, dst, dst_pos, length);
2508 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2523 __ call_VM_leaf(entry, 3);
2524 }
2525
2526 __ bind(*stub->continuation());
2527 }
2528
2529
2530
2531
2532 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2533 Register obj = op->obj_opr()->as_register(); // may not be an oop
2534 Register hdr = op->hdr_opr()->as_register();
2535 Register lock = op->lock_opr()->as_register();
2536 if (UseHeavyMonitors) {
2537 if (op->info() != NULL) {
2538 add_debug_info_for_null_check_here(op->info());
2539 __ null_check(obj, -1);
2540 }
2541 __ b(*op->stub()->entry());
2542 } else if (op->code() == lir_lock) {
2543 // add debug info for NullPointerException only if one is possible
2544 int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
2545 if (op->info() != NULL) {
2546 add_debug_info_for_null_check(null_check_offset, op->info());
2547 }
2548 // done
2549 } else if (op->code() == lir_unlock) {
2550 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2551 } else {
2552 Unimplemented();
2553 }
2554 __ bind(*op->stub()->continuation());
2555 }
2556
2557 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2558 Register obj = op->obj()->as_pointer_register();
2559 Register result = op->result_opr()->as_pointer_register();
2560 Register tmp = rscratch1;
2561
2562 CodeEmitInfo* info = op->info();
2563 if (info != NULL) {
2564 add_debug_info_for_null_check_here(info);
2565 }
2566
2567 assert(UseCompressedClassPointers, "expects UseCompressedClassPointers");
2568
2569 // Check if we can take the (common) fast path, if obj is unlocked.
2570 __ ldr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
2571 __ eor(tmp, tmp, markWord::unlocked_value);
2572 __ tst(tmp, markWord::lock_mask_in_place);
2573 __ br(Assembler::NE, *op->stub()->entry());
2574
2575 // Fast-path: shift and decode Klass*.
2576 __ mov(result, tmp);
2577 __ lsr(result, result, markWord::klass_shift);
2578
2579 __ bind(*op->stub()->continuation());
2580 __ decode_klass_not_null(result);
2581 }
2582
2583 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2584 ciMethod* method = op->profiled_method();
2585 int bci = op->profiled_bci();
2586 ciMethod* callee = op->profiled_callee();
2587
2588 // Update counter for all call types
2589 ciMethodData* md = method->method_data_or_null();
2590 assert(md != NULL, "Sanity");
2591 ciProfileData* data = md->bci_to_data(bci);
2592 assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2593 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2594 Register mdo = op->mdo()->as_register();
2595 __ mov_metadata(mdo, md->constant_encoding());
2596 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2597 // Perform additional virtual call profiling for invokevirtual and
2598 // invokeinterface bytecodes
2599 if (op->should_profile_receiver_type()) {
2600 assert(op->recv()->is_single_cpu(), "recv must be allocated");
2621 }
2622
2623 // Receiver type not found in profile data; select an empty slot
2624
2625 // Note that this is less efficient than it should be because it
2626 // always does a write to the receiver part of the
2627 // VirtualCallData rather than just the first time
2628 for (i = 0; i < VirtualCallData::row_limit(); i++) {
2629 ciKlass* receiver = vc_data->receiver(i);
2630 if (receiver == NULL) {
2631 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2632 __ mov_metadata(rscratch1, known_klass->constant_encoding());
2633 __ lea(rscratch2, recv_addr);
2634 __ str(rscratch1, Address(rscratch2));
2635 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2636 __ addptr(data_addr, DataLayout::counter_increment);
2637 return;
2638 }
2639 }
2640 } else {
2641 __ load_klass(rscratch1, recv);
2642 __ mov(recv, rscratch1);
2643 Label update_done;
2644 type_profile_helper(mdo, md, data, recv, &update_done);
2645 // Receiver did not match any saved receiver and there is no empty row for it.
2646 // Increment total counter to indicate polymorphic case.
2647 __ addptr(counter_addr, DataLayout::counter_increment);
2648
2649 __ bind(update_done);
2650 }
2651 } else {
2652 // Static call
2653 __ addptr(counter_addr, DataLayout::counter_increment);
2654 }
2655 }
2656
2657
2658 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2659 Unimplemented();
2660 }
2661
2662
2663 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2664 __ lea(dst->as_register(), frame_map()->address_for_monitor_object(monitor_no));
2665 }
2666
2667 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2668 assert(op->crc()->is_single_cpu(), "crc must be register");
2669 assert(op->val()->is_single_cpu(), "byte value must be register");
2670 assert(op->result_opr()->is_single_cpu(), "result must be register");
2671 Register crc = op->crc()->as_register();
2672 Register val = op->val()->as_register();
2673 Register res = op->result_opr()->as_register();
2674
2675 assert_different_registers(val, crc, res);
2676 uint64_t offset;
2677 __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2678 __ add(res, res, offset);
2679
2680 __ mvnw(crc, crc); // ~crc
2681 __ update_byte_crc32(crc, val, res);
2682 __ mvnw(res, crc); // ~crc
2683 }
2684
2716 }
2717 if (do_update) {
2718 #ifndef ASSERT
2719 __ b(next);
2720 }
2721 #else
2722 __ b(next);
2723 }
2724 } else {
2725 __ cbnz(tmp, update);
2726 __ stop("unexpected null obj");
2727 #endif
2728 }
2729
2730 __ bind(update);
2731
2732 if (do_update) {
2733 #ifdef ASSERT
2734 if (exact_klass != NULL) {
2735 Label ok;
2736 __ load_klass(rscratch1, tmp);
2737 __ mov(tmp, rscratch1);
2738 __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2739 __ eor(rscratch1, tmp, rscratch1);
2740 __ cbz(rscratch1, ok);
2741 __ stop("exact klass and actual klass differ");
2742 __ bind(ok);
2743 }
2744 #endif
2745 if (!no_conflict) {
2746 if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2747 if (exact_klass != NULL) {
2748 __ mov_metadata(tmp, exact_klass->constant_encoding());
2749 } else {
2750 __ load_klass(rscratch1, tmp);
2751 __ mov(tmp, rscratch1);
2752 }
2753
2754 __ ldr(rscratch2, mdo_addr);
2755 __ eor(tmp, tmp, rscratch2);
2756 __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2757 // klass seen before, nothing to do. The unknown bit may have been
2758 // set already but no need to check.
2759 __ cbz(rscratch1, next);
2760
2761 __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2762
2763 if (TypeEntries::is_type_none(current_klass)) {
2764 __ cbz(rscratch2, none);
2765 __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2766 __ br(Assembler::EQ, none);
2767 // There is a chance that the checks above (re-reading profiling
2768 // data from memory) fail if another thread has just set the
2769 // profiling to this obj's klass
2770 __ dmb(Assembler::ISHLD);
2771 __ ldr(rscratch2, mdo_addr);
|