21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "code/compiledIC.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "utilities/powerOfTwo.hpp"
46 #include "vmreg_aarch64.inline.hpp"
47
48
49 #ifndef PRODUCT
50 #define COMMENT(x) do { __ block_comment(x); } while (0)
51 #else
52 #define COMMENT(x)
53 #endif
54
55 NEEDS_CLEANUP // remove this definitions ?
56 const Register IC_Klass = rscratch2; // where the IC klass is cached
57 const Register SYNC_header = r0; // synchronization header
58 const Register SHIFT_count = r0; // where count for shift operations must be
59
60 #define __ _masm->
225 assert((addr.offset() & mask) == 0, "scaled offsets only");
226 return __ legitimize_address(addr, size, tmp);
227 }
228
229 void LIR_Assembler::osr_entry() {
230 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
231 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
232 ValueStack* entry_state = osr_entry->state();
233 int number_of_locks = entry_state->locks_size();
234
235 // we jump here if osr happens with the interpreter
236 // state set up to continue at the beginning of the
237 // loop that triggered osr - in particular, we have
238 // the following registers setup:
239 //
240 // r2: osr buffer
241 //
242
243 // build frame
244 ciMethod* m = compilation()->method();
245 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
246
247 // OSR buffer is
248 //
249 // locals[nlocals-1..0]
250 // monitors[0..number_of_locks]
251 //
252 // locals is a direct copy of the interpreter frame so in the osr buffer
253 // so first slot in the local array is the last local from the interpreter
254 // and last slot is local[0] (receiver) from the interpreter
255 //
256 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
257 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
258 // in the interpreter frame (the method lock if a sync method)
259
260 // Initialize monitors in the compiled activation.
261 // r2: pointer to osr buffer
262 //
263 // All other registers are dead at this point and the locals will be
264 // copied into place by code emitted in the IR.
265
1212 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1213 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1214 __ b(*op->stub()->entry());
1215 } else {
1216 Register tmp1 = op->tmp1()->as_register();
1217 Register tmp2 = op->tmp2()->as_register();
1218 Register tmp3 = op->tmp3()->as_register();
1219 if (len == tmp1) {
1220 tmp1 = tmp3;
1221 } else if (len == tmp2) {
1222 tmp2 = tmp3;
1223 } else if (len == tmp3) {
1224 // everything is ok
1225 } else {
1226 __ mov(tmp3, len);
1227 }
1228 __ allocate_array(op->obj()->as_register(),
1229 len,
1230 tmp1,
1231 tmp2,
1232 arrayOopDesc::header_size(op->type()),
1233 array_element_size(op->type()),
1234 op->klass()->as_register(),
1235 *op->stub()->entry());
1236 }
1237 __ bind(*op->stub()->continuation());
1238 }
1239
1240 void LIR_Assembler::type_profile_helper(Register mdo,
1241 ciMethodData *md, ciProfileData *data,
1242 Register recv, Label* update_done) {
1243 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1244 Label next_test;
1245 // See if the receiver is receiver[n].
1246 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1247 __ ldr(rscratch1, Address(rscratch2));
1248 __ cmp(recv, rscratch1);
1249 __ br(Assembler::NE, next_test);
1250 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1251 __ addptr(data_addr, DataLayout::counter_increment);
1252 __ b(*update_done);
2333 }
2334
2335 if (flags & LIR_OpArrayCopy::src_range_check) {
2336 __ addw(tmp, src_pos, length);
2337 __ ldrw(rscratch1, src_length_addr);
2338 __ cmpw(tmp, rscratch1);
2339 __ br(Assembler::HI, *stub->entry());
2340 }
2341 if (flags & LIR_OpArrayCopy::dst_range_check) {
2342 __ addw(tmp, dst_pos, length);
2343 __ ldrw(rscratch1, dst_length_addr);
2344 __ cmpw(tmp, rscratch1);
2345 __ br(Assembler::HI, *stub->entry());
2346 }
2347
2348 if (flags & LIR_OpArrayCopy::type_check) {
2349 // We don't know the array types are compatible
2350 if (basic_type != T_OBJECT) {
2351 // Simple test for basic type arrays
2352 if (UseCompressedClassPointers) {
2353 __ ldrw(tmp, src_klass_addr);
2354 __ ldrw(rscratch1, dst_klass_addr);
2355 __ cmpw(tmp, rscratch1);
2356 } else {
2357 __ ldr(tmp, src_klass_addr);
2358 __ ldr(rscratch1, dst_klass_addr);
2359 __ cmp(tmp, rscratch1);
2360 }
2361 __ br(Assembler::NE, *stub->entry());
2362 } else {
2363 // For object arrays, if src is a sub class of dst then we can
2364 // safely do the copy.
2365 Label cont, slow;
2366
2367 #define PUSH(r1, r2) \
2368 stp(r1, r2, __ pre(sp, -2 * wordSize));
2369
2370 #define POP(r1, r2) \
2371 ldp(r1, r2, __ post(sp, 2 * wordSize));
2372
2373 __ PUSH(src, dst);
2374
2375 __ load_klass(src, src);
2376 __ load_klass(dst, dst);
2377
2378 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2462 }
2463
2464 __ b(*stub->entry());
2465
2466 __ bind(cont);
2467 __ POP(src, dst);
2468 }
2469 }
2470
2471 #ifdef ASSERT
2472 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2473 // Sanity check the known type with the incoming class. For the
2474 // primitive case the types must match exactly with src.klass and
2475 // dst.klass each exactly matching the default type. For the
2476 // object array case, if no type check is needed then either the
2477 // dst type is exactly the expected type and the src type is a
2478 // subtype which we can't check or src is the same array as dst
2479 // but not necessarily exactly of type default_type.
2480 Label known_ok, halt;
2481 __ mov_metadata(tmp, default_type->constant_encoding());
2482 if (UseCompressedClassPointers) {
2483 __ encode_klass_not_null(tmp);
2484 }
2485
2486 if (basic_type != T_OBJECT) {
2487
2488 if (UseCompressedClassPointers) {
2489 __ ldrw(rscratch1, dst_klass_addr);
2490 __ cmpw(tmp, rscratch1);
2491 } else {
2492 __ ldr(rscratch1, dst_klass_addr);
2493 __ cmp(tmp, rscratch1);
2494 }
2495 __ br(Assembler::NE, halt);
2496 if (UseCompressedClassPointers) {
2497 __ ldrw(rscratch1, src_klass_addr);
2498 __ cmpw(tmp, rscratch1);
2499 } else {
2500 __ ldr(rscratch1, src_klass_addr);
2501 __ cmp(tmp, rscratch1);
2502 }
2503 __ br(Assembler::EQ, known_ok);
2504 } else {
2505 if (UseCompressedClassPointers) {
2506 __ ldrw(rscratch1, dst_klass_addr);
2507 __ cmpw(tmp, rscratch1);
2508 } else {
2509 __ ldr(rscratch1, dst_klass_addr);
2510 __ cmp(tmp, rscratch1);
2511 }
2512 __ br(Assembler::EQ, known_ok);
2513 __ cmp(src, dst);
2514 __ br(Assembler::EQ, known_ok);
2515 }
2516 __ bind(halt);
2517 __ stop("incorrect type information in arraycopy");
2518 __ bind(known_ok);
2519 }
2520 #endif
2521
2522 #ifndef PRODUCT
2523 if (PrintC1Statistics) {
2524 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2525 }
2526 #endif
2527
2528 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2529 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2530 assert_different_registers(c_rarg0, dst, dst_pos, length);
2531 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2572 // done
2573 } else if (op->code() == lir_unlock) {
2574 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2575 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2576 } else {
2577 Unimplemented();
2578 }
2579 __ bind(*op->stub()->continuation());
2580 }
2581
2582 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2583 Register obj = op->obj()->as_pointer_register();
2584 Register result = op->result_opr()->as_pointer_register();
2585
2586 CodeEmitInfo* info = op->info();
2587 if (info != NULL) {
2588 add_debug_info_for_null_check_here(info);
2589 }
2590
2591 if (UseCompressedClassPointers) {
2592 __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2593 __ decode_klass_not_null(result);
2594 } else {
2595 __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2596 }
2597 }
2598
2599 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2600 ciMethod* method = op->profiled_method();
2601 int bci = op->profiled_bci();
2602 ciMethod* callee = op->profiled_callee();
2603
2604 // Update counter for all call types
2605 ciMethodData* md = method->method_data_or_null();
2606 assert(md != NULL, "Sanity");
2607 ciProfileData* data = md->bci_to_data(bci);
2608 assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2609 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2610 Register mdo = op->mdo()->as_register();
2611 __ mov_metadata(mdo, md->constant_encoding());
2612 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
|
21 * or visit www.oracle.com if you need additional information or have any
22 * questions.
23 *
24 */
25
26 #include "precompiled.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "asm/assembler.hpp"
29 #include "c1/c1_CodeStubs.hpp"
30 #include "c1/c1_Compilation.hpp"
31 #include "c1/c1_LIRAssembler.hpp"
32 #include "c1/c1_MacroAssembler.hpp"
33 #include "c1/c1_Runtime1.hpp"
34 #include "c1/c1_ValueStack.hpp"
35 #include "ci/ciArrayKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "code/compiledIC.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_aarch64.hpp"
41 #include "oops/compressedKlass.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_aarch64.inline.hpp"
48
49
50 #ifndef PRODUCT
51 #define COMMENT(x) do { __ block_comment(x); } while (0)
52 #else
53 #define COMMENT(x)
54 #endif
55
56 NEEDS_CLEANUP // remove this definitions ?
57 const Register IC_Klass = rscratch2; // where the IC klass is cached
58 const Register SYNC_header = r0; // synchronization header
59 const Register SHIFT_count = r0; // where count for shift operations must be
60
61 #define __ _masm->
226 assert((addr.offset() & mask) == 0, "scaled offsets only");
227 return __ legitimize_address(addr, size, tmp);
228 }
229
230 void LIR_Assembler::osr_entry() {
231 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
232 BlockBegin* osr_entry = compilation()->hir()->osr_entry();
233 ValueStack* entry_state = osr_entry->state();
234 int number_of_locks = entry_state->locks_size();
235
236 // we jump here if osr happens with the interpreter
237 // state set up to continue at the beginning of the
238 // loop that triggered osr - in particular, we have
239 // the following registers setup:
240 //
241 // r2: osr buffer
242 //
243
244 // build frame
245 ciMethod* m = compilation()->method();
246 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes(), compilation()->max_monitors());
247
248 // OSR buffer is
249 //
250 // locals[nlocals-1..0]
251 // monitors[0..number_of_locks]
252 //
253 // locals is a direct copy of the interpreter frame so in the osr buffer
254 // so first slot in the local array is the last local from the interpreter
255 // and last slot is local[0] (receiver) from the interpreter
256 //
257 // Similarly with locks. The first lock slot in the osr buffer is the nth lock
258 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
259 // in the interpreter frame (the method lock if a sync method)
260
261 // Initialize monitors in the compiled activation.
262 // r2: pointer to osr buffer
263 //
264 // All other registers are dead at this point and the locals will be
265 // copied into place by code emitted in the IR.
266
1213 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1214 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1215 __ b(*op->stub()->entry());
1216 } else {
1217 Register tmp1 = op->tmp1()->as_register();
1218 Register tmp2 = op->tmp2()->as_register();
1219 Register tmp3 = op->tmp3()->as_register();
1220 if (len == tmp1) {
1221 tmp1 = tmp3;
1222 } else if (len == tmp2) {
1223 tmp2 = tmp3;
1224 } else if (len == tmp3) {
1225 // everything is ok
1226 } else {
1227 __ mov(tmp3, len);
1228 }
1229 __ allocate_array(op->obj()->as_register(),
1230 len,
1231 tmp1,
1232 tmp2,
1233 arrayOopDesc::base_offset_in_bytes(op->type()),
1234 array_element_size(op->type()),
1235 op->klass()->as_register(),
1236 *op->stub()->entry());
1237 }
1238 __ bind(*op->stub()->continuation());
1239 }
1240
1241 void LIR_Assembler::type_profile_helper(Register mdo,
1242 ciMethodData *md, ciProfileData *data,
1243 Register recv, Label* update_done) {
1244 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1245 Label next_test;
1246 // See if the receiver is receiver[n].
1247 __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1248 __ ldr(rscratch1, Address(rscratch2));
1249 __ cmp(recv, rscratch1);
1250 __ br(Assembler::NE, next_test);
1251 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1252 __ addptr(data_addr, DataLayout::counter_increment);
1253 __ b(*update_done);
2334 }
2335
2336 if (flags & LIR_OpArrayCopy::src_range_check) {
2337 __ addw(tmp, src_pos, length);
2338 __ ldrw(rscratch1, src_length_addr);
2339 __ cmpw(tmp, rscratch1);
2340 __ br(Assembler::HI, *stub->entry());
2341 }
2342 if (flags & LIR_OpArrayCopy::dst_range_check) {
2343 __ addw(tmp, dst_pos, length);
2344 __ ldrw(rscratch1, dst_length_addr);
2345 __ cmpw(tmp, rscratch1);
2346 __ br(Assembler::HI, *stub->entry());
2347 }
2348
2349 if (flags & LIR_OpArrayCopy::type_check) {
2350 // We don't know the array types are compatible
2351 if (basic_type != T_OBJECT) {
2352 // Simple test for basic type arrays
2353 if (UseCompressedClassPointers) {
2354 __ load_nklass(tmp, src);
2355 __ load_nklass(rscratch1, dst);
2356 __ cmpw(tmp, rscratch1);
2357 } else {
2358 __ ldr(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
2359 __ ldr(rscratch1, Address(dst, oopDesc::klass_offset_in_bytes()));
2360 __ cmp(tmp, rscratch1);
2361 }
2362 __ br(Assembler::NE, *stub->entry());
2363 } else {
2364 // For object arrays, if src is a sub class of dst then we can
2365 // safely do the copy.
2366 Label cont, slow;
2367
2368 #define PUSH(r1, r2) \
2369 stp(r1, r2, __ pre(sp, -2 * wordSize));
2370
2371 #define POP(r1, r2) \
2372 ldp(r1, r2, __ post(sp, 2 * wordSize));
2373
2374 __ PUSH(src, dst);
2375
2376 __ load_klass(src, src);
2377 __ load_klass(dst, dst);
2378
2379 __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2463 }
2464
2465 __ b(*stub->entry());
2466
2467 __ bind(cont);
2468 __ POP(src, dst);
2469 }
2470 }
2471
2472 #ifdef ASSERT
2473 if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2474 // Sanity check the known type with the incoming class. For the
2475 // primitive case the types must match exactly with src.klass and
2476 // dst.klass each exactly matching the default type. For the
2477 // object array case, if no type check is needed then either the
2478 // dst type is exactly the expected type and the src type is a
2479 // subtype which we can't check or src is the same array as dst
2480 // but not necessarily exactly of type default_type.
2481 Label known_ok, halt;
2482 __ mov_metadata(tmp, default_type->constant_encoding());
2483
2484 if (basic_type != T_OBJECT) {
2485 __ cmp_klass(dst, tmp, rscratch1);
2486 __ br(Assembler::NE, halt);
2487 __ cmp_klass(src, tmp, rscratch1);
2488 __ br(Assembler::EQ, known_ok);
2489 } else {
2490 __ cmp_klass(dst, tmp, rscratch1);
2491 __ br(Assembler::EQ, known_ok);
2492 __ cmp(src, dst);
2493 __ br(Assembler::EQ, known_ok);
2494 }
2495 __ bind(halt);
2496 __ stop("incorrect type information in arraycopy");
2497 __ bind(known_ok);
2498 }
2499 #endif
2500
2501 #ifndef PRODUCT
2502 if (PrintC1Statistics) {
2503 __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2504 }
2505 #endif
2506
2507 __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2508 __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2509 assert_different_registers(c_rarg0, dst, dst_pos, length);
2510 __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2551 // done
2552 } else if (op->code() == lir_unlock) {
2553 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2554 __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2555 } else {
2556 Unimplemented();
2557 }
2558 __ bind(*op->stub()->continuation());
2559 }
2560
2561 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2562 Register obj = op->obj()->as_pointer_register();
2563 Register result = op->result_opr()->as_pointer_register();
2564
2565 CodeEmitInfo* info = op->info();
2566 if (info != NULL) {
2567 add_debug_info_for_null_check_here(info);
2568 }
2569
2570 if (UseCompressedClassPointers) {
2571 if (UseCompactObjectHeaders) {
2572 // Check if we can take the (common) fast path, if obj is unlocked.
2573 __ ldr(result, Address(obj, oopDesc::mark_offset_in_bytes()));
2574 __ tst(result, markWord::monitor_value);
2575 __ br(Assembler::NE, *op->stub()->entry());
2576 __ bind(*op->stub()->continuation());
2577
2578 // Shift to get proper narrow Klass*.
2579 __ lsr(result, result, markWord::klass_shift);
2580 } else {
2581 __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2582 }
2583 __ decode_klass_not_null(result);
2584 } else {
2585 __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2586 }
2587 }
2588
2589 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2590 ciMethod* method = op->profiled_method();
2591 int bci = op->profiled_bci();
2592 ciMethod* callee = op->profiled_callee();
2593
2594 // Update counter for all call types
2595 ciMethodData* md = method->method_data_or_null();
2596 assert(md != NULL, "Sanity");
2597 ciProfileData* data = md->bci_to_data(bci);
2598 assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2599 assert(op->mdo()->is_single_cpu(), "mdo must be allocated");
2600 Register mdo = op->mdo()->as_register();
2601 __ mov_metadata(mdo, md->constant_encoding());
2602 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
|