3301
3302 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2,
3303 MacroAssembler::PreservationLevel preservation_level) {
3304 Label resolved;
3305
3306 // A null weak handle resolves to null.
3307 cmpdi(CR0, result, 0);
3308 beq(CR0, resolved);
3309
3310 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, result, noreg, result, tmp1, tmp2,
3311 preservation_level);
3312 bind(resolved);
3313 }
3314
3315 void MacroAssembler::load_method_holder(Register holder, Register method) {
3316 ld(holder, in_bytes(Method::const_offset()), method);
3317 ld(holder, in_bytes(ConstMethod::constants_offset()), holder);
3318 ld(holder, ConstantPool::pool_holder_offset(), holder);
3319 }
3320
3321 // Clear Array
3322 // For very short arrays. tmp == R0 is allowed.
3323 void MacroAssembler::clear_memory_unrolled(Register base_ptr, int cnt_dwords, Register tmp, int offset) {
3324 if (cnt_dwords > 0) { li(tmp, 0); }
3325 for (int i = 0; i < cnt_dwords; ++i) { std(tmp, offset + i * 8, base_ptr); }
3326 }
3327
3328 // Version for constant short array length. Kills base_ptr. tmp == R0 is allowed.
3329 void MacroAssembler::clear_memory_constlen(Register base_ptr, int cnt_dwords, Register tmp) {
3330 if (cnt_dwords < 8) {
3331 clear_memory_unrolled(base_ptr, cnt_dwords, tmp);
3332 return;
3333 }
3334
3335 Label loop;
3336 const long loopcnt = cnt_dwords >> 1,
3337 remainder = cnt_dwords & 1;
3338
3339 li(tmp, loopcnt);
3340 mtctr(tmp);
3389
3390 bind(fastloop);
3391 dcbz(base_ptr); // Clear 128byte aligned block.
3392 addi(base_ptr, base_ptr, cl_size);
3393 bdnz(fastloop);
3394
3395 bind(small_rest);
3396 cmpdi(CR0, cnt_dwords, 0); // size 0?
3397 beq(CR0, done); // rest == 0
3398 li(tmp, 0);
3399 mtctr(cnt_dwords); // Load counter.
3400
3401 bind(restloop); // Clear rest.
3402 std(tmp, 0, base_ptr); // Clear 8byte aligned block.
3403 addi(base_ptr, base_ptr, 8);
3404 bdnz(restloop);
3405
3406 bind(done);
3407 }
3408
3409 /////////////////////////////////////////// String intrinsics ////////////////////////////////////////////
3410
3411 // Helpers for Intrinsic Emitters
3412 //
3413 // Revert the byte order of a 32bit value in a register
3414 // src: 0x44556677
3415 // dst: 0x77665544
3416 // Three steps to obtain the result:
3417 // 1) Rotate src (as doubleword) left 5 bytes. That puts the leftmost byte of the src word
3418 // into the rightmost byte position. Afterwards, everything left of the rightmost byte is cleared.
3419 // This value initializes dst.
3420 // 2) Rotate src (as word) left 3 bytes. That puts the rightmost byte of the src word into the leftmost
3421 // byte position. Furthermore, byte 5 is rotated into byte 6 position where it is supposed to go.
3422 // This value is mask inserted into dst with a [0..23] mask of 1s.
3423 // 3) Rotate src (as word) left 1 byte. That puts byte 6 into byte 5 position.
3424 // This value is mask inserted into dst with a [8..15] mask of 1s.
3425 void MacroAssembler::load_reverse_32(Register dst, Register src) {
3426 assert_different_registers(dst, src);
3427
3428 rldicl(dst, src, (4+1)*8, 56); // Rotate byte 4 into position 7 (rightmost), clear all to the left.
4708 }
4709
4710 // Function to flip between unlocked and locked state (fast locking).
4711 // Branches to failed if the state is not as expected with CR0 NE.
4712 // Falls through upon success with CR0 EQ.
4713 // This requires fewer instructions and registers and is easier to use than the
4714 // cmpxchg based implementation.
4715 void MacroAssembler::atomically_flip_locked_state(bool is_unlock, Register obj, Register tmp, Label& failed, int semantics) {
4716 assert_different_registers(obj, tmp, R0);
4717 Label retry;
4718
4719 if (semantics & MemBarRel) {
4720 release();
4721 }
4722
4723 bind(retry);
4724 STATIC_ASSERT(markWord::locked_value == 0); // Or need to change this!
4725 if (!is_unlock) {
4726 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_acquire_lock());
4727 xori(tmp, tmp, markWord::unlocked_value); // flip unlocked bit
4728 andi_(R0, tmp, markWord::lock_mask_in_place);
4729 bne(CR0, failed); // failed if new header doesn't contain locked_value (which is 0)
4730 } else {
4731 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_release_lock());
4732 andi_(R0, tmp, markWord::lock_mask_in_place);
4733 bne(CR0, failed); // failed if old header doesn't contain locked_value (which is 0)
4734 ori(tmp, tmp, markWord::unlocked_value); // set unlocked bit
4735 }
4736 stdcx_(tmp, obj);
4737 bne(CR0, retry);
4738
4739 if (semantics & MemBarFenceAfter) {
4740 fence();
4741 } else if (semantics & MemBarAcq) {
4742 isync();
4743 }
4744 }
4745
4746 // Implements fast-locking.
4747 //
4748 // - obj: the object to be locked
4749 // - t1, t2: temporary register
4863 beq(CR0, not_unlocked);
4864 stop("fast_unlock already unlocked");
4865 bind(not_unlocked);
4866 #endif
4867
4868 // Try to unlock. Transition lock bits 0b00 => 0b01
4869 atomically_flip_locked_state(/* is_unlock */ true, obj, t, push_and_slow, MacroAssembler::MemBarRel);
4870 b(unlocked);
4871
4872 bind(push_and_slow);
4873
4874 // Restore lock-stack and handle the unlock in runtime.
4875 lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4876 DEBUG_ONLY(stdx(obj, R16_thread, top);)
4877 addi(top, top, oopSize);
4878 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
4879 b(slow);
4880
4881 bind(unlocked);
4882 }
|
3301
3302 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2,
3303 MacroAssembler::PreservationLevel preservation_level) {
3304 Label resolved;
3305
3306 // A null weak handle resolves to null.
3307 cmpdi(CR0, result, 0);
3308 beq(CR0, resolved);
3309
3310 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, result, noreg, result, tmp1, tmp2,
3311 preservation_level);
3312 bind(resolved);
3313 }
3314
3315 void MacroAssembler::load_method_holder(Register holder, Register method) {
3316 ld(holder, in_bytes(Method::const_offset()), method);
3317 ld(holder, in_bytes(ConstMethod::constants_offset()), holder);
3318 ld(holder, ConstantPool::pool_holder_offset(), holder);
3319 }
3320
3321 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
3322 assert_different_registers(markword, R0);
3323 andi(R0, markword, markWord::inline_type_pattern_mask);
3324 cmpwi(CR0, R0, markWord::inline_type_pattern);
3325 beq(CR0, is_inline_type);
3326 }
3327
3328 void MacroAssembler::test_oop_is_not_inline_type(Register object, Label& not_inline_type, bool can_be_null) {
3329 if (can_be_null) {
3330 cmpdi(CR0, object, 0);
3331 beq(CR0, not_inline_type);
3332 }
3333 ld(R0, oopDesc::mark_offset_in_bytes(), object);
3334 andi(R0, R0, markWord::inline_type_pattern_mask);
3335 cmpwi(CR0, R0, markWord::inline_type_pattern);
3336 bne(CR0, not_inline_type);
3337 }
3338
3339 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Label& is_null_free_inline_type) {
3340 testbitdi(CR0, R0, flags, ResolvedFieldEntry::is_null_free_inline_type_shift);
3341 bne(CR0, is_null_free_inline_type);
3342 }
3343
3344 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Label& not_null_free_inline_type) {
3345 testbitdi(CR0, R0, flags, ResolvedFieldEntry::is_null_free_inline_type_shift);
3346 beq(CR0, not_null_free_inline_type);
3347 }
3348
3349 void MacroAssembler::test_field_is_flat(Register flags, Label& is_flat) {
3350 testbitdi(CR0, R0, flags, ResolvedFieldEntry::is_flat_shift);
3351 bne(CR0, is_flat);
3352 }
3353
3354 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set,
3355 Label& jmp_label, bool maybe_far) {
3356 Label test_mark_word;
3357 // load mark word
3358 ld(temp_reg, oopDesc::mark_offset_in_bytes(), oop);
3359 // if unlocked bit is set we can directly use the mark word
3360 andi_(R0, temp_reg, markWord::unlocked_value);
3361 bne(CR0, test_mark_word);
3362 // slow path use klass prototype
3363 load_prototype_header(temp_reg, oop);
3364
3365 bind(test_mark_word);
3366 andi_(R0, temp_reg, test_bit);
3367 if (maybe_far) {
3368 bc_far_optimized(jmp_set ? Assembler::bcondCRbiIs0 : Assembler::bcondCRbiIs1,
3369 bi0(CR0, Assembler::equal), jmp_label);
3370 } else {
3371 if (jmp_set) {
3372 bne(CR0, jmp_label);
3373 } else {
3374 beq(CR0, jmp_label);
3375 }
3376 }
3377 }
3378
3379 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg, Label& is_flat_array, bool maybe_far) {
3380 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array, maybe_far);
3381 }
3382
3383 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg, Label& is_non_flat_array) {
3384 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array);
3385 }
3386
3387 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array, bool maybe_far) {
3388 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array, maybe_far);
3389 }
3390
3391 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label& is_non_null_free_array) {
3392 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
3393 }
3394
3395 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) {
3396 testbitdi(CR0, R0, lh, exact_log2(Klass::_lh_array_tag_flat_value_bit_inplace));
3397 bne(CR0, is_flat_array);
3398 }
3399
3400 void MacroAssembler::load_metadata(Register dst, Register src) {
3401 if (UseCompactObjectHeaders) {
3402 load_narrow_klass_compact(dst, src);
3403 } else {
3404 lwz(dst, oopDesc::klass_offset_in_bytes(), src);
3405 }
3406 }
3407
3408 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3409 load_klass(dst, src);
3410 ld(dst, Klass::prototype_header_offset(), dst);
3411 }
3412
3413 void MacroAssembler::flat_field_copy(DecoratorSet decorators, Register src, Register dst, Register inline_layout_info) {
3414 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3415 bs->flat_field_copy(this, decorators, src, dst, inline_layout_info);
3416 }
3417
3418 void MacroAssembler::payload_offset(Register inline_klass, Register offset) {
3419 ld(offset, in_bytes(InlineKlass::adr_members_offset()), inline_klass);
3420 lwz(offset, in_bytes(InlineKlass::payload_offset_offset()), offset);
3421 }
3422
3423 void MacroAssembler::payload_address(Register oop, Register data, Register inline_klass, Register t1) {
3424 // ((address) (void*) o) + vk->payload_offset();
3425 payload_offset(inline_klass, t1);
3426 add(data, oop, t1);
3427 }
3428
3429 void MacroAssembler::inline_layout_info(Register holder_klass, Register index, Register layout_info) {
3430 assert_different_registers(holder_klass, index, layout_info);
3431 InlineLayoutInfo array[2];
3432 int size = (char*)&array[1] - (char*)&array[0]; // computing size of array elements
3433 if (is_power_of_2(size)) {
3434 sldi(index, index, log2i_exact(size)); // Scale index by power of 2
3435 } else {
3436 mulld(index, index, size); // Scale the index to be the entry index * array_element_size
3437 }
3438 ld(layout_info, InstanceKlass::inline_layout_info_array_offset(), holder_klass);
3439 addi(layout_info, layout_info, Array<InlineLayoutInfo>::base_offset_in_bytes());
3440 add(layout_info, layout_info, index);
3441 }
3442
3443
3444 // Clear Array
3445 // For very short arrays. tmp == R0 is allowed.
3446 void MacroAssembler::clear_memory_unrolled(Register base_ptr, int cnt_dwords, Register tmp, int offset) {
3447 if (cnt_dwords > 0) { li(tmp, 0); }
3448 for (int i = 0; i < cnt_dwords; ++i) { std(tmp, offset + i * 8, base_ptr); }
3449 }
3450
3451 // Version for constant short array length. Kills base_ptr. tmp == R0 is allowed.
3452 void MacroAssembler::clear_memory_constlen(Register base_ptr, int cnt_dwords, Register tmp) {
3453 if (cnt_dwords < 8) {
3454 clear_memory_unrolled(base_ptr, cnt_dwords, tmp);
3455 return;
3456 }
3457
3458 Label loop;
3459 const long loopcnt = cnt_dwords >> 1,
3460 remainder = cnt_dwords & 1;
3461
3462 li(tmp, loopcnt);
3463 mtctr(tmp);
3512
3513 bind(fastloop);
3514 dcbz(base_ptr); // Clear 128byte aligned block.
3515 addi(base_ptr, base_ptr, cl_size);
3516 bdnz(fastloop);
3517
3518 bind(small_rest);
3519 cmpdi(CR0, cnt_dwords, 0); // size 0?
3520 beq(CR0, done); // rest == 0
3521 li(tmp, 0);
3522 mtctr(cnt_dwords); // Load counter.
3523
3524 bind(restloop); // Clear rest.
3525 std(tmp, 0, base_ptr); // Clear 8byte aligned block.
3526 addi(base_ptr, base_ptr, 8);
3527 bdnz(restloop);
3528
3529 bind(done);
3530 }
3531
3532 // base: Address of a buffer to be filled, 8 bytes aligned. Killed.
3533 // cnt: Count in 8-byte unit.
3534 // value: Value to be filled with.
3535 void MacroAssembler::fill_words(Register base, Register cnt, Register value) {
3536 Label loop, loop_end, done;
3537
3538 // 2x unrolled loop
3539 srdi_(R0, cnt, 1);
3540 beq(CR0, loop_end); // less than 2 elements
3541 mtctr(R0);
3542
3543 bind(loop);
3544 std(value, 0, base);
3545 std(value, 8, base);
3546 addi(base, base, 16);
3547 bdnz(loop);
3548
3549 bind(loop_end);
3550 andi_(R0, cnt, 1);
3551 beq(CR0, done);
3552 std(value, 0, base); // last element
3553
3554 bind(done);
3555 }
3556
3557 /////////////////////////////////////////// String intrinsics ////////////////////////////////////////////
3558
3559 // Helpers for Intrinsic Emitters
3560 //
3561 // Revert the byte order of a 32bit value in a register
3562 // src: 0x44556677
3563 // dst: 0x77665544
3564 // Three steps to obtain the result:
3565 // 1) Rotate src (as doubleword) left 5 bytes. That puts the leftmost byte of the src word
3566 // into the rightmost byte position. Afterwards, everything left of the rightmost byte is cleared.
3567 // This value initializes dst.
3568 // 2) Rotate src (as word) left 3 bytes. That puts the rightmost byte of the src word into the leftmost
3569 // byte position. Furthermore, byte 5 is rotated into byte 6 position where it is supposed to go.
3570 // This value is mask inserted into dst with a [0..23] mask of 1s.
3571 // 3) Rotate src (as word) left 1 byte. That puts byte 6 into byte 5 position.
3572 // This value is mask inserted into dst with a [8..15] mask of 1s.
3573 void MacroAssembler::load_reverse_32(Register dst, Register src) {
3574 assert_different_registers(dst, src);
3575
3576 rldicl(dst, src, (4+1)*8, 56); // Rotate byte 4 into position 7 (rightmost), clear all to the left.
4856 }
4857
4858 // Function to flip between unlocked and locked state (fast locking).
4859 // Branches to failed if the state is not as expected with CR0 NE.
4860 // Falls through upon success with CR0 EQ.
4861 // This requires fewer instructions and registers and is easier to use than the
4862 // cmpxchg based implementation.
4863 void MacroAssembler::atomically_flip_locked_state(bool is_unlock, Register obj, Register tmp, Label& failed, int semantics) {
4864 assert_different_registers(obj, tmp, R0);
4865 Label retry;
4866
4867 if (semantics & MemBarRel) {
4868 release();
4869 }
4870
4871 bind(retry);
4872 STATIC_ASSERT(markWord::locked_value == 0); // Or need to change this!
4873 if (!is_unlock) {
4874 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_acquire_lock());
4875 xori(tmp, tmp, markWord::unlocked_value); // flip unlocked bit
4876 andi_(R0, tmp, markWord::lock_mask_in_place | markWord::inline_type_bit_in_place);
4877 bne(CR0, failed); // failed if new header doesn't contain locked_value (which is 0) or belongs to an inline type
4878 } else {
4879 ldarx(tmp, obj, MacroAssembler::cmpxchgx_hint_release_lock());
4880 andi_(R0, tmp, markWord::lock_mask_in_place);
4881 bne(CR0, failed); // failed if old header doesn't contain locked_value (which is 0)
4882 ori(tmp, tmp, markWord::unlocked_value); // set unlocked bit
4883 }
4884 stdcx_(tmp, obj);
4885 bne(CR0, retry);
4886
4887 if (semantics & MemBarFenceAfter) {
4888 fence();
4889 } else if (semantics & MemBarAcq) {
4890 isync();
4891 }
4892 }
4893
4894 // Implements fast-locking.
4895 //
4896 // - obj: the object to be locked
4897 // - t1, t2: temporary register
5011 beq(CR0, not_unlocked);
5012 stop("fast_unlock already unlocked");
5013 bind(not_unlocked);
5014 #endif
5015
5016 // Try to unlock. Transition lock bits 0b00 => 0b01
5017 atomically_flip_locked_state(/* is_unlock */ true, obj, t, push_and_slow, MacroAssembler::MemBarRel);
5018 b(unlocked);
5019
5020 bind(push_and_slow);
5021
5022 // Restore lock-stack and handle the unlock in runtime.
5023 lwz(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
5024 DEBUG_ONLY(stdx(obj, R16_thread, top);)
5025 addi(top, top, oopSize);
5026 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
5027 b(slow);
5028
5029 bind(unlocked);
5030 }
5031
5032 // Unimplemented methods for inline types.
5033 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
5034 Unimplemented();
5035 }
5036
5037 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
5038 Unimplemented();
5039 }
5040
5041 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
5042 VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
5043 RegState reg_state[]) {
5044 Unimplemented();
5045 }
5046
5047 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
5048 VMRegPair* from, int from_count, int& from_index, VMReg to,
5049 RegState reg_state[], Register val_array) {
5050 Unimplemented();
5051 }
5052
5053 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
5054 Unimplemented();
5055 }
5056
5057 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
5058 Unimplemented();
5059 }
|