3305
3306 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2,
3307 MacroAssembler::PreservationLevel preservation_level) {
3308 Label resolved;
3309
3310 // A null weak handle resolves to null.
3311 cmpdi(CR0, result, 0);
3312 beq(CR0, resolved);
3313
3314 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, result, noreg, result, tmp1, tmp2,
3315 preservation_level);
3316 bind(resolved);
3317 }
3318
3319 void MacroAssembler::load_method_holder(Register holder, Register method) {
3320 ld(holder, in_bytes(Method::const_offset()), method);
3321 ld(holder, in_bytes(ConstMethod::constants_offset()), holder);
3322 ld(holder, ConstantPool::pool_holder_offset(), holder);
3323 }
3324
3325 // Clear Array
3326 // For very short arrays. tmp == R0 is allowed.
3327 void MacroAssembler::clear_memory_unrolled(Register base_ptr, int cnt_dwords, Register tmp, int offset) {
3328 if (cnt_dwords > 0) { li(tmp, 0); }
3329 for (int i = 0; i < cnt_dwords; ++i) { std(tmp, offset + i * 8, base_ptr); }
3330 }
3331
3332 // Version for constant short array length. Kills base_ptr. tmp == R0 is allowed.
3333 void MacroAssembler::clear_memory_constlen(Register base_ptr, int cnt_dwords, Register tmp) {
3334 if (cnt_dwords < 8) {
3335 clear_memory_unrolled(base_ptr, cnt_dwords, tmp);
3336 return;
3337 }
3338
3339 Label loop;
3340 const long loopcnt = cnt_dwords >> 1,
3341 remainder = cnt_dwords & 1;
3342
3343 li(tmp, loopcnt);
3344 mtctr(tmp);
3393
3394 bind(fastloop);
3395 dcbz(base_ptr); // Clear 128byte aligned block.
3396 addi(base_ptr, base_ptr, cl_size);
3397 bdnz(fastloop);
3398
3399 bind(small_rest);
3400 cmpdi(CR0, cnt_dwords, 0); // size 0?
3401 beq(CR0, done); // rest == 0
3402 li(tmp, 0);
3403 mtctr(cnt_dwords); // Load counter.
3404
3405 bind(restloop); // Clear rest.
3406 std(tmp, 0, base_ptr); // Clear 8byte aligned block.
3407 addi(base_ptr, base_ptr, 8);
3408 bdnz(restloop);
3409
3410 bind(done);
3411 }
3412
3413 /////////////////////////////////////////// String intrinsics ////////////////////////////////////////////
3414
3415 // Helpers for Intrinsic Emitters
3416 //
3417 // Revert the byte order of a 32bit value in a register
3418 // src: 0x44556677
3419 // dst: 0x77665544
3420 // Three steps to obtain the result:
3421 // 1) Rotate src (as doubleword) left 5 bytes. That puts the leftmost byte of the src word
3422 // into the rightmost byte position. Afterwards, everything left of the rightmost byte is cleared.
3423 // This value initializes dst.
3424 // 2) Rotate src (as word) left 3 bytes. That puts the rightmost byte of the src word into the leftmost
3425 // byte position. Furthermore, byte 5 is rotated into byte 6 position where it is supposed to go.
3426 // This value is mask inserted into dst with a [0..23] mask of 1s.
3427 // 3) Rotate src (as word) left 1 byte. That puts byte 6 into byte 5 position.
3428 // This value is mask inserted into dst with a [8..15] mask of 1s.
3429 void MacroAssembler::load_reverse_32(Register dst, Register src) {
3430 assert_different_registers(dst, src);
3431
3432 rldicl(dst, src, (4+1)*8, 56); // Rotate byte 4 into position 7 (rightmost), clear all to the left.
4712 }
4713
4714 // Function to flip between unlocked and locked state (fast locking).
4715 // Branches to failed if the state is not as expected with CR0 NE.
4716 // Falls through upon success with CR0 EQ.
4717 // This requires fewer instructions and registers and is easier to use than the
4718 // cmpxchg based implementation.
4719 void MacroAssembler::atomically_flip_locked_state(bool is_unlock, Register obj, Register tmp, Label& failed, int semantics) {
4720 assert_different_registers(obj, tmp, R0);
4721 Label retry;
4722
4723 if (semantics & MemBarRel) {
4724 release();
4725 }
4726
4727 bind(retry);
4728 STATIC_ASSERT(markWord::locked_value == 0); // Or need to change this!
4729 if (!is_unlock) {
4730 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_acquire_lock());
4731 xori(tmp, tmp, markWord::unlocked_value); // flip unlocked bit
4732 andi_(R0, tmp, markWord::lock_mask_in_place);
4733 bne(CR0, failed); // failed if new header doesn't contain locked_value (which is 0)
4734 } else {
4735 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_release_lock());
4736 andi_(R0, tmp, markWord::lock_mask_in_place);
4737 bne(CR0, failed); // failed if old header doesn't contain locked_value (which is 0)
4738 ori(tmp, tmp, markWord::unlocked_value); // set unlocked bit
4739 }
4740 stdcx_(tmp, obj);
4741 bne(CR0, retry);
4742
4743 if (semantics & MemBarFenceAfter) {
4744 fence();
4745 } else if (semantics & MemBarAcq) {
4746 isync();
4747 }
4748 }
4749
4750 // Implements fast-locking.
4751 //
4752 // - obj: the object to be locked
4753 // - t1, t2: temporary register
4867 beq(CR0, not_unlocked);
4868 stop("fast_unlock already unlocked");
4869 bind(not_unlocked);
4870 #endif
4871
4872 // Try to unlock. Transition lock bits 0b00 => 0b01
4873 atomically_flip_locked_state(/* is_unlock */ true, obj, t, push_and_slow, MacroAssembler::MemBarRel);
4874 b(unlocked);
4875
4876 bind(push_and_slow);
4877
4878 // Restore lock-stack and handle the unlock in runtime.
4879 lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4880 DEBUG_ONLY(stdx(obj, R16_thread, top);)
4881 addi(top, top, oopSize);
4882 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4883 b(slow);
4884
4885 bind(unlocked);
4886 }
|
3305
3306 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2,
3307 MacroAssembler::PreservationLevel preservation_level) {
3308 Label resolved;
3309
3310 // A null weak handle resolves to null.
3311 cmpdi(CR0, result, 0);
3312 beq(CR0, resolved);
3313
3314 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, result, noreg, result, tmp1, tmp2,
3315 preservation_level);
3316 bind(resolved);
3317 }
3318
3319 void MacroAssembler::load_method_holder(Register holder, Register method) {
3320 ld(holder, in_bytes(Method::const_offset()), method);
3321 ld(holder, in_bytes(ConstMethod::constants_offset()), holder);
3322 ld(holder, ConstantPool::pool_holder_offset(), holder);
3323 }
3324
3325 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
3326 assert_different_registers(markword, R0);
3327 andi(R0, markword, markWord::inline_type_pattern_mask);
3328 cmpwi(CR0, R0, markWord::inline_type_pattern);
3329 beq(CR0, is_inline_type);
3330 }
3331
3332 void MacroAssembler::test_oop_is_not_inline_type(Register object, Label& not_inline_type, bool can_be_null) {
3333 if (can_be_null) {
3334 cmpdi(CR0, object, 0);
3335 beq(CR0, not_inline_type);
3336 }
3337 ld(R0, oopDesc::mark_offset_in_bytes(), object);
3338 andi(R0, R0, markWord::inline_type_pattern_mask);
3339 cmpwi(CR0, R0, markWord::inline_type_pattern);
3340 bne(CR0, not_inline_type);
3341 }
3342
3343 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Label& is_null_free_inline_type) {
3344 testbitdi(CR0, R0, flags, ResolvedFieldEntry::is_null_free_inline_type_shift);
3345 bne(CR0, is_null_free_inline_type);
3346 }
3347
3348 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Label& not_null_free_inline_type) {
3349 testbitdi(CR0, R0, flags, ResolvedFieldEntry::is_null_free_inline_type_shift);
3350 beq(CR0, not_null_free_inline_type);
3351 }
3352
3353 void MacroAssembler::test_field_is_flat(Register flags, Label& is_flat) {
3354 testbitdi(CR0, R0, flags, ResolvedFieldEntry::is_flat_shift);
3355 bne(CR0, is_flat);
3356 }
3357
3358 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set,
3359 Label& jmp_label, bool maybe_far) {
3360 // load mark word
3361 ld(temp_reg, oopDesc::mark_offset_in_bytes(), oop);
3362 if (!UseObjectMonitorTable) {
3363 Label test_mark_word;
3364 // if unlocked bit is set we can directly use the mark word
3365 andi_(R0, temp_reg, markWord::unlocked_value);
3366 bne(CR0, test_mark_word);
3367 // slow path use klass prototype
3368 load_prototype_header(temp_reg, oop);
3369
3370 bind(test_mark_word);
3371 }
3372 andi_(R0, temp_reg, test_bit);
3373 if (maybe_far) {
3374 bc_far_optimized(jmp_set ? Assembler::bcondCRbiIs0 : Assembler::bcondCRbiIs1,
3375 bi0(CR0, Assembler::equal), jmp_label);
3376 } else {
3377 if (jmp_set) {
3378 bne(CR0, jmp_label);
3379 } else {
3380 beq(CR0, jmp_label);
3381 }
3382 }
3383 }
3384
3385 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg, Label& is_flat_array, bool maybe_far) {
3386 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array, maybe_far);
3387 }
3388
3389 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg, Label& is_non_flat_array) {
3390 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array);
3391 }
3392
3393 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array, bool maybe_far) {
3394 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array, maybe_far);
3395 }
3396
3397 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label& is_non_null_free_array) {
3398 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
3399 }
3400
3401 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) {
3402 testbitdi(CR0, R0, lh, exact_log2(Klass::_lh_array_tag_flat_value_bit_inplace));
3403 bne(CR0, is_flat_array);
3404 }
3405
3406 void MacroAssembler::load_metadata(Register dst, Register src) {
3407 if (UseCompactObjectHeaders) {
3408 load_narrow_klass_compact(dst, src);
3409 } else {
3410 lwz(dst, oopDesc::klass_offset_in_bytes(), src);
3411 }
3412 }
3413
3414 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3415 load_klass(dst, src);
3416 ld(dst, Klass::prototype_header_offset(), dst);
3417 }
3418
3419 void MacroAssembler::flat_field_copy(DecoratorSet decorators, Register src, Register dst, Register inline_layout_info) {
3420 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3421 bs->flat_field_copy(this, decorators, src, dst, inline_layout_info);
3422 }
3423
3424 void MacroAssembler::payload_offset(Register inline_klass, Register offset) {
3425 ld(offset, in_bytes(InlineKlass::adr_members_offset()), inline_klass);
3426 lwz(offset, in_bytes(InlineKlass::payload_offset_offset()), offset);
3427 }
3428
3429 void MacroAssembler::payload_address(Register oop, Register data, Register inline_klass, Register t1) {
3430 // ((address) (void*) o) + vk->payload_offset();
3431 payload_offset(inline_klass, t1);
3432 add(data, oop, t1);
3433 }
3434
3435 void MacroAssembler::inline_layout_info(Register holder_klass, Register index, Register layout_info) {
3436 assert_different_registers(holder_klass, index, layout_info);
3437 InlineLayoutInfo array[2];
3438 int size = (char*)&array[1] - (char*)&array[0]; // computing size of array elements
3439 if (is_power_of_2(size)) {
3440 sldi(index, index, log2i_exact(size)); // Scale index by power of 2
3441 } else {
3442 mulld(index, index, size); // Scale the index to be the entry index * array_element_size
3443 }
3444 ld(layout_info, InstanceKlass::inline_layout_info_array_offset(), holder_klass);
3445 addi(layout_info, layout_info, Array<InlineLayoutInfo>::base_offset_in_bytes());
3446 add(layout_info, layout_info, index);
3447 }
3448
3449
3450 // Clear Array
3451 // For very short arrays. tmp == R0 is allowed.
3452 void MacroAssembler::clear_memory_unrolled(Register base_ptr, int cnt_dwords, Register tmp, int offset) {
3453 if (cnt_dwords > 0) { li(tmp, 0); }
3454 for (int i = 0; i < cnt_dwords; ++i) { std(tmp, offset + i * 8, base_ptr); }
3455 }
3456
3457 // Version for constant short array length. Kills base_ptr. tmp == R0 is allowed.
3458 void MacroAssembler::clear_memory_constlen(Register base_ptr, int cnt_dwords, Register tmp) {
3459 if (cnt_dwords < 8) {
3460 clear_memory_unrolled(base_ptr, cnt_dwords, tmp);
3461 return;
3462 }
3463
3464 Label loop;
3465 const long loopcnt = cnt_dwords >> 1,
3466 remainder = cnt_dwords & 1;
3467
3468 li(tmp, loopcnt);
3469 mtctr(tmp);
3518
3519 bind(fastloop);
3520 dcbz(base_ptr); // Clear 128byte aligned block.
3521 addi(base_ptr, base_ptr, cl_size);
3522 bdnz(fastloop);
3523
3524 bind(small_rest);
3525 cmpdi(CR0, cnt_dwords, 0); // size 0?
3526 beq(CR0, done); // rest == 0
3527 li(tmp, 0);
3528 mtctr(cnt_dwords); // Load counter.
3529
3530 bind(restloop); // Clear rest.
3531 std(tmp, 0, base_ptr); // Clear 8byte aligned block.
3532 addi(base_ptr, base_ptr, 8);
3533 bdnz(restloop);
3534
3535 bind(done);
3536 }
3537
3538 // base: Address of a buffer to be filled, 8 bytes aligned. Killed.
3539 // cnt: Count in 8-byte unit.
3540 // value: Value to be filled with.
3541 void MacroAssembler::fill_words(Register base, Register cnt, Register value) {
3542 Label loop, loop_end, done;
3543
3544 // 2x unrolled loop
3545 srdi_(R0, cnt, 1);
3546 beq(CR0, loop_end); // less than 2 elements
3547 mtctr(R0);
3548
3549 bind(loop);
3550 std(value, 0, base);
3551 std(value, 8, base);
3552 addi(base, base, 16);
3553 bdnz(loop);
3554
3555 bind(loop_end);
3556 andi_(R0, cnt, 1);
3557 beq(CR0, done);
3558 std(value, 0, base); // last element
3559
3560 bind(done);
3561 }
3562
3563 /////////////////////////////////////////// String intrinsics ////////////////////////////////////////////
3564
3565 // Helpers for Intrinsic Emitters
3566 //
3567 // Revert the byte order of a 32bit value in a register
3568 // src: 0x44556677
3569 // dst: 0x77665544
3570 // Three steps to obtain the result:
3571 // 1) Rotate src (as doubleword) left 5 bytes. That puts the leftmost byte of the src word
3572 // into the rightmost byte position. Afterwards, everything left of the rightmost byte is cleared.
3573 // This value initializes dst.
3574 // 2) Rotate src (as word) left 3 bytes. That puts the rightmost byte of the src word into the leftmost
3575 // byte position. Furthermore, byte 5 is rotated into byte 6 position where it is supposed to go.
3576 // This value is mask inserted into dst with a [0..23] mask of 1s.
3577 // 3) Rotate src (as word) left 1 byte. That puts byte 6 into byte 5 position.
3578 // This value is mask inserted into dst with a [8..15] mask of 1s.
3579 void MacroAssembler::load_reverse_32(Register dst, Register src) {
3580 assert_different_registers(dst, src);
3581
3582 rldicl(dst, src, (4+1)*8, 56); // Rotate byte 4 into position 7 (rightmost), clear all to the left.
4862 }
4863
4864 // Function to flip between unlocked and locked state (fast locking).
4865 // Branches to failed if the state is not as expected with CR0 NE.
4866 // Falls through upon success with CR0 EQ.
4867 // This requires fewer instructions and registers and is easier to use than the
4868 // cmpxchg based implementation.
4869 void MacroAssembler::atomically_flip_locked_state(bool is_unlock, Register obj, Register tmp, Label& failed, int semantics) {
4870 assert_different_registers(obj, tmp, R0);
4871 Label retry;
4872
4873 if (semantics & MemBarRel) {
4874 release();
4875 }
4876
4877 bind(retry);
4878 STATIC_ASSERT(markWord::locked_value == 0); // Or need to change this!
4879 if (!is_unlock) {
4880 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_acquire_lock());
4881 xori(tmp, tmp, markWord::unlocked_value); // flip unlocked bit
4882 andi_(R0, tmp, markWord::lock_mask_in_place | markWord::inline_type_bit_in_place);
4883 bne(CR0, failed); // failed if new header doesn't contain locked_value (which is 0) or belongs to an inline type
4884 } else {
4885 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_release_lock());
4886 andi_(R0, tmp, markWord::lock_mask_in_place);
4887 bne(CR0, failed); // failed if old header doesn't contain locked_value (which is 0)
4888 ori(tmp, tmp, markWord::unlocked_value); // set unlocked bit
4889 }
4890 stdcx_(tmp, obj);
4891 bne(CR0, retry);
4892
4893 if (semantics & MemBarFenceAfter) {
4894 fence();
4895 } else if (semantics & MemBarAcq) {
4896 isync();
4897 }
4898 }
4899
4900 // Implements fast-locking.
4901 //
4902 // - obj: the object to be locked
4903 // - t1, t2: temporary register
5017 beq(CR0, not_unlocked);
5018 stop("fast_unlock already unlocked");
5019 bind(not_unlocked);
5020 #endif
5021
5022 // Try to unlock. Transition lock bits 0b00 => 0b01
5023 atomically_flip_locked_state(/* is_unlock */ true, obj, t, push_and_slow, MacroAssembler::MemBarRel);
5024 b(unlocked);
5025
5026 bind(push_and_slow);
5027
5028 // Restore lock-stack and handle the unlock in runtime.
5029 lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
5030 DEBUG_ONLY(stdx(obj, R16_thread, top);)
5031 addi(top, top, oopSize);
5032 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
5033 b(slow);
5034
5035 bind(unlocked);
5036 }
5037
5038 // Unimplemented methods for inline types.
5039 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
5040 Unimplemented();
5041 }
5042
5043 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
5044 Unimplemented();
5045 }
5046
5047 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
5048 VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
5049 RegState reg_state[]) {
5050 Unimplemented();
5051 }
5052
5053 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
5054 VMRegPair* from, int from_count, int& from_index, VMReg to,
5055 RegState reg_state[], Register val_array) {
5056 Unimplemented();
5057 }
5058
5059 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
5060 Unimplemented();
5061 }
5062
5063 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
5064 Unimplemented();
5065 }
|