3301
3302 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2,
3303 MacroAssembler::PreservationLevel preservation_level) {
3304 Label resolved;
3305
3306 // A null weak handle resolves to null.
3307 cmpdi(CR0, result, 0);
3308 beq(CR0, resolved);
3309
3310 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, result, noreg, result, tmp1, tmp2,
3311 preservation_level);
3312 bind(resolved);
3313 }
3314
3315 void MacroAssembler::load_method_holder(Register holder, Register method) {
3316 ld(holder, in_bytes(Method::const_offset()), method);
3317 ld(holder, in_bytes(ConstMethod::constants_offset()), holder);
3318 ld(holder, ConstantPool::pool_holder_offset(), holder);
3319 }
3320
3321 // Clear Array
3322 // For very short arrays. tmp == R0 is allowed.
3323 void MacroAssembler::clear_memory_unrolled(Register base_ptr, int cnt_dwords, Register tmp, int offset) {
3324 if (cnt_dwords > 0) { li(tmp, 0); }
3325 for (int i = 0; i < cnt_dwords; ++i) { std(tmp, offset + i * 8, base_ptr); }
3326 }
3327
3328 // Version for constant short array length. Kills base_ptr. tmp == R0 is allowed.
3329 void MacroAssembler::clear_memory_constlen(Register base_ptr, int cnt_dwords, Register tmp) {
3330 if (cnt_dwords < 8) {
3331 clear_memory_unrolled(base_ptr, cnt_dwords, tmp);
3332 return;
3333 }
3334
3335 Label loop;
3336 const long loopcnt = cnt_dwords >> 1,
3337 remainder = cnt_dwords & 1;
3338
3339 li(tmp, loopcnt);
3340 mtctr(tmp);
3389
3390 bind(fastloop);
3391 dcbz(base_ptr); // Clear 128byte aligned block.
3392 addi(base_ptr, base_ptr, cl_size);
3393 bdnz(fastloop);
3394
3395 bind(small_rest);
3396 cmpdi(CR0, cnt_dwords, 0); // size 0?
3397 beq(CR0, done); // rest == 0
3398 li(tmp, 0);
3399 mtctr(cnt_dwords); // Load counter.
3400
3401 bind(restloop); // Clear rest.
3402 std(tmp, 0, base_ptr); // Clear 8byte aligned block.
3403 addi(base_ptr, base_ptr, 8);
3404 bdnz(restloop);
3405
3406 bind(done);
3407 }
3408
3409 /////////////////////////////////////////// String intrinsics ////////////////////////////////////////////
3410
3411 // Helpers for Intrinsic Emitters
3412 //
3413 // Revert the byte order of a 32bit value in a register
3414 // src: 0x44556677
3415 // dst: 0x77665544
3416 // Three steps to obtain the result:
3417 // 1) Rotate src (as doubleword) left 5 bytes. That puts the leftmost byte of the src word
3418 // into the rightmost byte position. Afterwards, everything left of the rightmost byte is cleared.
3419 // This value initializes dst.
3420 // 2) Rotate src (as word) left 3 bytes. That puts the rightmost byte of the src word into the leftmost
3421 // byte position. Furthermore, byte 5 is rotated into byte 6 position where it is supposed to go.
3422 // This value is mask inserted into dst with a [0..23] mask of 1s.
3423 // 3) Rotate src (as word) left 1 byte. That puts byte 6 into byte 5 position.
3424 // This value is mask inserted into dst with a [8..15] mask of 1s.
3425 void MacroAssembler::load_reverse_32(Register dst, Register src) {
3426 assert_different_registers(dst, src);
3427
3428 rldicl(dst, src, (4+1)*8, 56); // Rotate byte 4 into position 7 (rightmost), clear all to the left.
4708 }
4709
4710 // Function to flip between unlocked and locked state (fast locking).
4711 // Branches to failed if the state is not as expected with CR0 NE.
4712 // Falls through upon success with CR0 EQ.
4713 // This requires fewer instructions and registers and is easier to use than the
4714 // cmpxchg based implementation.
4715 void MacroAssembler::atomically_flip_locked_state(bool is_unlock, Register obj, Register tmp, Label& failed, int semantics) {
4716 assert_different_registers(obj, tmp, R0);
4717 Label retry;
4718
4719 if (semantics & MemBarRel) {
4720 release();
4721 }
4722
4723 bind(retry);
4724 STATIC_ASSERT(markWord::locked_value == 0); // Or need to change this!
4725 if (!is_unlock) {
4726 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_acquire_lock());
4727 xori(tmp, tmp, markWord::unlocked_value); // flip unlocked bit
4728 andi_(R0, tmp, markWord::lock_mask_in_place);
4729 bne(CR0, failed); // failed if new header doesn't contain locked_value (which is 0)
4730 } else {
4731 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_release_lock());
4732 andi_(R0, tmp, markWord::lock_mask_in_place);
4733 bne(CR0, failed); // failed if old header doesn't contain locked_value (which is 0)
4734 ori(tmp, tmp, markWord::unlocked_value); // set unlocked bit
4735 }
4736 stdcx_(tmp, obj);
4737 bne(CR0, retry);
4738
4739 if (semantics & MemBarFenceAfter) {
4740 fence();
4741 } else if (semantics & MemBarAcq) {
4742 isync();
4743 }
4744 }
4745
4746 // Implements fast-locking.
4747 //
4748 // - obj: the object to be locked
4749 // - t1, t2: temporary register
4863 beq(CR0, not_unlocked);
4864 stop("fast_unlock already unlocked");
4865 bind(not_unlocked);
4866 #endif
4867
4868 // Try to unlock. Transition lock bits 0b00 => 0b01
4869 atomically_flip_locked_state(/* is_unlock */ true, obj, t, push_and_slow, MacroAssembler::MemBarRel);
4870 b(unlocked);
4871
4872 bind(push_and_slow);
4873
4874 // Restore lock-stack and handle the unlock in runtime.
4875 lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4876 DEBUG_ONLY(stdx(obj, R16_thread, top);)
4877 addi(top, top, oopSize);
4878 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4879 b(slow);
4880
4881 bind(unlocked);
4882 }
|
3301
3302 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2,
3303 MacroAssembler::PreservationLevel preservation_level) {
3304 Label resolved;
3305
3306 // A null weak handle resolves to null.
3307 cmpdi(CR0, result, 0);
3308 beq(CR0, resolved);
3309
3310 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, result, noreg, result, tmp1, tmp2,
3311 preservation_level);
3312 bind(resolved);
3313 }
3314
3315 void MacroAssembler::load_method_holder(Register holder, Register method) {
3316 ld(holder, in_bytes(Method::const_offset()), method);
3317 ld(holder, in_bytes(ConstMethod::constants_offset()), holder);
3318 ld(holder, ConstantPool::pool_holder_offset(), holder);
3319 }
3320
3321 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
3322 assert_different_registers(markword, R0);
3323 andi(R0, markword, markWord::inline_type_pattern_mask);
3324 cmpwi(CR0, R0, markWord::inline_type_pattern);
3325 beq(CR0, is_inline_type);
3326 }
3327
3328 void MacroAssembler::test_oop_is_not_inline_type(Register object, Label& not_inline_type, bool can_be_null) {
3329 if (can_be_null) {
3330 cmpdi(CR0, object, 0);
3331 beq(CR0, not_inline_type);
3332 }
3333 ld(R0, oopDesc::mark_offset_in_bytes(), object);
3334 andi(R0, R0, markWord::inline_type_pattern_mask);
3335 cmpwi(CR0, R0, markWord::inline_type_pattern);
3336 bne(CR0, not_inline_type);
3337 }
3338
3339 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Label& is_null_free_inline_type) {
3340 testbitdi(CR0, R0, flags, ResolvedFieldEntry::is_null_free_inline_type_shift);
3341 bne(CR0, is_null_free_inline_type);
3342 }
3343
3344 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Label& not_null_free_inline_type) {
3345 testbitdi(CR0, R0, flags, ResolvedFieldEntry::is_null_free_inline_type_shift);
3346 beq(CR0, not_null_free_inline_type);
3347 }
3348
3349 void MacroAssembler::test_field_is_flat(Register flags, Label& is_flat) {
3350 testbitdi(CR0, R0, flags, ResolvedFieldEntry::is_flat_shift);
3351 bne(CR0, is_flat);
3352 }
3353
3354 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set,
3355 Label& jmp_label, bool maybe_far) {
3356 // load mark word
3357 ld(temp_reg, oopDesc::mark_offset_in_bytes(), oop);
3358 if (!UseObjectMonitorTable) {
3359 Label test_mark_word;
3360 // if unlocked bit is set we can directly use the mark word
3361 andi_(R0, temp_reg, markWord::unlocked_value);
3362 bne(CR0, test_mark_word);
3363 // slow path use klass prototype
3364 load_prototype_header(temp_reg, oop);
3365
3366 bind(test_mark_word);
3367 }
3368 andi_(R0, temp_reg, test_bit);
3369 if (maybe_far) {
3370 bc_far_optimized(jmp_set ? Assembler::bcondCRbiIs0 : Assembler::bcondCRbiIs1,
3371 bi0(CR0, Assembler::equal), jmp_label);
3372 } else {
3373 if (jmp_set) {
3374 bne(CR0, jmp_label);
3375 } else {
3376 beq(CR0, jmp_label);
3377 }
3378 }
3379 }
3380
3381 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg, Label& is_flat_array, bool maybe_far) {
3382 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array, maybe_far);
3383 }
3384
3385 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg, Label& is_non_flat_array) {
3386 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array);
3387 }
3388
3389 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array, bool maybe_far) {
3390 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array, maybe_far);
3391 }
3392
3393 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label& is_non_null_free_array) {
3394 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
3395 }
3396
3397 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) {
3398 testbitdi(CR0, R0, lh, exact_log2(Klass::_lh_array_tag_flat_value_bit_inplace));
3399 bne(CR0, is_flat_array);
3400 }
3401
3402 void MacroAssembler::load_metadata(Register dst, Register src) {
3403 if (UseCompactObjectHeaders) {
3404 load_narrow_klass_compact(dst, src);
3405 } else {
3406 lwz(dst, oopDesc::klass_offset_in_bytes(), src);
3407 }
3408 }
3409
3410 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3411 load_klass(dst, src);
3412 ld(dst, Klass::prototype_header_offset(), dst);
3413 }
3414
3415 void MacroAssembler::flat_field_copy(DecoratorSet decorators, Register src, Register dst, Register inline_layout_info) {
3416 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3417 bs->flat_field_copy(this, decorators, src, dst, inline_layout_info);
3418 }
3419
3420 void MacroAssembler::payload_offset(Register inline_klass, Register offset) {
3421 ld(offset, in_bytes(InlineKlass::adr_members_offset()), inline_klass);
3422 lwz(offset, in_bytes(InlineKlass::payload_offset_offset()), offset);
3423 }
3424
3425 void MacroAssembler::payload_address(Register oop, Register data, Register inline_klass, Register t1) {
3426 // ((address) (void*) o) + vk->payload_offset();
3427 payload_offset(inline_klass, t1);
3428 add(data, oop, t1);
3429 }
3430
3431 void MacroAssembler::inline_layout_info(Register holder_klass, Register index, Register layout_info) {
3432 assert_different_registers(holder_klass, index, layout_info);
3433 InlineLayoutInfo array[2];
3434 int size = (char*)&array[1] - (char*)&array[0]; // computing size of array elements
3435 if (is_power_of_2(size)) {
3436 sldi(index, index, log2i_exact(size)); // Scale index by power of 2
3437 } else {
3438 mulld(index, index, size); // Scale the index to be the entry index * array_element_size
3439 }
3440 ld(layout_info, InstanceKlass::inline_layout_info_array_offset(), holder_klass);
3441 addi(layout_info, layout_info, Array<InlineLayoutInfo>::base_offset_in_bytes());
3442 add(layout_info, layout_info, index);
3443 }
3444
3445
3446 // Clear Array
3447 // For very short arrays. tmp == R0 is allowed.
3448 void MacroAssembler::clear_memory_unrolled(Register base_ptr, int cnt_dwords, Register tmp, int offset) {
3449 if (cnt_dwords > 0) { li(tmp, 0); }
3450 for (int i = 0; i < cnt_dwords; ++i) { std(tmp, offset + i * 8, base_ptr); }
3451 }
3452
3453 // Version for constant short array length. Kills base_ptr. tmp == R0 is allowed.
3454 void MacroAssembler::clear_memory_constlen(Register base_ptr, int cnt_dwords, Register tmp) {
3455 if (cnt_dwords < 8) {
3456 clear_memory_unrolled(base_ptr, cnt_dwords, tmp);
3457 return;
3458 }
3459
3460 Label loop;
3461 const long loopcnt = cnt_dwords >> 1,
3462 remainder = cnt_dwords & 1;
3463
3464 li(tmp, loopcnt);
3465 mtctr(tmp);
3514
3515 bind(fastloop);
3516 dcbz(base_ptr); // Clear 128byte aligned block.
3517 addi(base_ptr, base_ptr, cl_size);
3518 bdnz(fastloop);
3519
3520 bind(small_rest);
3521 cmpdi(CR0, cnt_dwords, 0); // size 0?
3522 beq(CR0, done); // rest == 0
3523 li(tmp, 0);
3524 mtctr(cnt_dwords); // Load counter.
3525
3526 bind(restloop); // Clear rest.
3527 std(tmp, 0, base_ptr); // Clear 8byte aligned block.
3528 addi(base_ptr, base_ptr, 8);
3529 bdnz(restloop);
3530
3531 bind(done);
3532 }
3533
3534 // base: Address of a buffer to be filled, 8 bytes aligned. Killed.
3535 // cnt: Count in 8-byte unit.
3536 // value: Value to be filled with.
3537 void MacroAssembler::fill_words(Register base, Register cnt, Register value) {
3538 Label loop, loop_end, done;
3539
3540 // 2x unrolled loop
3541 srdi_(R0, cnt, 1);
3542 beq(CR0, loop_end); // less than 2 elements
3543 mtctr(R0);
3544
3545 bind(loop);
3546 std(value, 0, base);
3547 std(value, 8, base);
3548 addi(base, base, 16);
3549 bdnz(loop);
3550
3551 bind(loop_end);
3552 andi_(R0, cnt, 1);
3553 beq(CR0, done);
3554 std(value, 0, base); // last element
3555
3556 bind(done);
3557 }
3558
3559 /////////////////////////////////////////// String intrinsics ////////////////////////////////////////////
3560
3561 // Helpers for Intrinsic Emitters
3562 //
3563 // Revert the byte order of a 32bit value in a register
3564 // src: 0x44556677
3565 // dst: 0x77665544
3566 // Three steps to obtain the result:
3567 // 1) Rotate src (as doubleword) left 5 bytes. That puts the leftmost byte of the src word
3568 // into the rightmost byte position. Afterwards, everything left of the rightmost byte is cleared.
3569 // This value initializes dst.
3570 // 2) Rotate src (as word) left 3 bytes. That puts the rightmost byte of the src word into the leftmost
3571 // byte position. Furthermore, byte 5 is rotated into byte 6 position where it is supposed to go.
3572 // This value is mask inserted into dst with a [0..23] mask of 1s.
3573 // 3) Rotate src (as word) left 1 byte. That puts byte 6 into byte 5 position.
3574 // This value is mask inserted into dst with a [8..15] mask of 1s.
3575 void MacroAssembler::load_reverse_32(Register dst, Register src) {
3576 assert_different_registers(dst, src);
3577
3578 rldicl(dst, src, (4+1)*8, 56); // Rotate byte 4 into position 7 (rightmost), clear all to the left.
4858 }
4859
4860 // Function to flip between unlocked and locked state (fast locking).
4861 // Branches to failed if the state is not as expected with CR0 NE.
4862 // Falls through upon success with CR0 EQ.
4863 // This requires fewer instructions and registers and is easier to use than the
4864 // cmpxchg based implementation.
4865 void MacroAssembler::atomically_flip_locked_state(bool is_unlock, Register obj, Register tmp, Label& failed, int semantics) {
4866 assert_different_registers(obj, tmp, R0);
4867 Label retry;
4868
4869 if (semantics & MemBarRel) {
4870 release();
4871 }
4872
4873 bind(retry);
4874 STATIC_ASSERT(markWord::locked_value == 0); // Or need to change this!
4875 if (!is_unlock) {
4876 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_acquire_lock());
4877 xori(tmp, tmp, markWord::unlocked_value); // flip unlocked bit
4878 andi_(R0, tmp, markWord::lock_mask_in_place | markWord::inline_type_bit_in_place);
4879 bne(CR0, failed); // failed if new header doesn't contain locked_value (which is 0) or belongs to an inline type
4880 } else {
4881 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_release_lock());
4882 andi_(R0, tmp, markWord::lock_mask_in_place);
4883 bne(CR0, failed); // failed if old header doesn't contain locked_value (which is 0)
4884 ori(tmp, tmp, markWord::unlocked_value); // set unlocked bit
4885 }
4886 stdcx_(tmp, obj);
4887 bne(CR0, retry);
4888
4889 if (semantics & MemBarFenceAfter) {
4890 fence();
4891 } else if (semantics & MemBarAcq) {
4892 isync();
4893 }
4894 }
4895
4896 // Implements fast-locking.
4897 //
4898 // - obj: the object to be locked
4899 // - t1, t2: temporary register
5013 beq(CR0, not_unlocked);
5014 stop("fast_unlock already unlocked");
5015 bind(not_unlocked);
5016 #endif
5017
5018 // Try to unlock. Transition lock bits 0b00 => 0b01
5019 atomically_flip_locked_state(/* is_unlock */ true, obj, t, push_and_slow, MacroAssembler::MemBarRel);
5020 b(unlocked);
5021
5022 bind(push_and_slow);
5023
5024 // Restore lock-stack and handle the unlock in runtime.
5025 lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
5026 DEBUG_ONLY(stdx(obj, R16_thread, top);)
5027 addi(top, top, oopSize);
5028 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
5029 b(slow);
5030
5031 bind(unlocked);
5032 }
5033
5034 // Unimplemented methods for inline types.
5035 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
5036 Unimplemented();
5037 }
5038
5039 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
5040 Unimplemented();
5041 }
5042
5043 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
5044 VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
5045 RegState reg_state[]) {
5046 Unimplemented();
5047 }
5048
5049 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
5050 VMRegPair* from, int from_count, int& from_index, VMReg to,
5051 RegState reg_state[], Register val_array) {
5052 Unimplemented();
5053 }
5054
5055 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
5056 Unimplemented();
5057 }
5058
5059 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
5060 Unimplemented();
5061 }
|