572 Label zf_bad_zero;
573 jcc(Assembler::zero, zf_correct);
574 jmp(zf_bad_zero);
575 #endif
576
577 bind(slow_path);
578 if (stub != nullptr) {
579 bind(stub->slow_path_continuation());
580 }
581 #ifdef ASSERT
582 // Check that stub->continuation() label is reached with ZF not set.
583 jcc(Assembler::notZero, zf_correct);
584 stop("Fast Unlock ZF != 0");
585 bind(zf_bad_zero);
586 stop("Fast Unlock ZF != 1");
587 bind(zf_correct);
588 #endif
589 // C2 uses the value of ZF to determine the continuation.
590 }
591
592 static void abort_verify_int_in_range(uint idx, jint val, jint lo, jint hi) {
593 fatal("Invalid CastII, idx: %u, val: %d, lo: %d, hi: %d", idx, val, lo, hi);
594 }
595
596 static void reconstruct_frame_pointer_helper(MacroAssembler* masm, Register dst) {
597 const int framesize = Compile::current()->output()->frame_size_in_bytes();
598 masm->movptr(dst, rsp);
599 if (framesize > 2 * wordSize) {
600 masm->addptr(dst, framesize - 2 * wordSize);
601 }
602 }
603
604 void C2_MacroAssembler::reconstruct_frame_pointer(Register rtmp) {
605 if (PreserveFramePointer) {
606 // frame pointer is valid
607 #ifdef ASSERT
608 // Verify frame pointer value in rbp.
609 reconstruct_frame_pointer_helper(this, rtmp);
610 Label L_success;
611 cmpq(rbp, rtmp);
612 jccb(Assembler::equal, L_success);
635 jccb(Assembler::less, fail);
636 }
637 if (hi != max_jint) {
638 cmpl(val, hi);
639 jccb(Assembler::greater, fail);
640 }
641 jmpb(succeed);
642
643 bind(fail);
644 movl(c_rarg0, idx);
645 movl(c_rarg1, val);
646 movl(c_rarg2, lo);
647 movl(c_rarg3, hi);
648 reconstruct_frame_pointer(rscratch1);
649 call(RuntimeAddress(CAST_FROM_FN_PTR(address, abort_verify_int_in_range)));
650 hlt();
651 bind(succeed);
652 BLOCK_COMMENT("} // CastII");
653 }
654
655 static void abort_verify_long_in_range(uint idx, jlong val, jlong lo, jlong hi) {
656 fatal("Invalid CastLL, idx: %u, val: " JLONG_FORMAT ", lo: " JLONG_FORMAT ", hi: " JLONG_FORMAT, idx, val, lo, hi);
657 }
658
659 void C2_MacroAssembler::verify_long_in_range(uint idx, const TypeLong* t, Register val, Register tmp) {
660 jlong lo = t->_lo;
661 jlong hi = t->_hi;
662 assert(lo < hi, "type should not be empty or constant, idx: %u, lo: " JLONG_FORMAT ", hi: " JLONG_FORMAT, idx, lo, hi);
663 if (t == TypeLong::LONG) {
664 return;
665 }
666
667 BLOCK_COMMENT("CastLL {");
668 Label fail;
669 Label succeed;
670
671 auto cmp_val = [&](jlong bound) {
672 if (is_simm32(bound)) {
673 cmpq(val, checked_cast<int>(bound));
674 } else {
675 mov64(tmp, bound);
3544 arrays_hashcode_elvload(vtmp[idx], Address(ary1, index, Address::times(elsize), 8 * idx * elsize), eltype);
3545 }
3546 // vresult = vresult * vnext + ary1[index+8*idx:index+8*idx+7];
3547 for (int idx = 0; idx < 4; idx++) {
3548 vpmulld(vresult[idx], vresult[idx], vnext, Assembler::AVX_256bit);
3549 arrays_hashcode_elvcast(vtmp[idx], eltype);
3550 vpaddd(vresult[idx], vresult[idx], vtmp[idx], Assembler::AVX_256bit);
3551 }
3552 // index += 32;
3553 addl(index, 32);
3554 // index < bound;
3555 cmpl(index, bound);
3556 jcc(Assembler::less, UNROLLED_VECTOR_LOOP_BEGIN);
3557 // }
3558
3559 lea(ary1, Address(ary1, bound, Address::times(elsize)));
3560 subl(cnt1, bound);
3561 // release bound
3562
3563 // vresult *= IntVector.fromArray(I256, power_of_31_backwards, 1);
3564 for (int idx = 0; idx < 4; idx++) {
3565 lea(tmp2, ExternalAddress(StubRoutines::x86::arrays_hashcode_powers_of_31() + ((8 * idx + 1) * sizeof(jint))));
3566 arrays_hashcode_elvload(vcoef[idx], Address(tmp2, 0), T_INT);
3567 vpmulld(vresult[idx], vresult[idx], vcoef[idx], Assembler::AVX_256bit);
3568 }
3569 // result += vresult.reduceLanes(ADD);
3570 for (int idx = 0; idx < 4; idx++) {
3571 reduceI(Op_AddReductionVI, 256/(sizeof(jint) * 8), result, result, vresult[idx], vtmp[(idx * 2 + 0) % 4], vtmp[(idx * 2 + 1) % 4]);
3572 }
3573
3574 // } else if (cnt1 < 32) {
3575
3576 bind(SHORT_UNROLLED_BEGIN);
3577 // int i = 1;
3578 movl(index, 1);
3579 cmpl(index, cnt1);
3580 jcc(Assembler::greaterEqual, SHORT_UNROLLED_LOOP_EXIT);
3581
3582 // for (; i < cnt1 ; i += 2) {
3583 bind(SHORT_UNROLLED_LOOP_BEGIN);
3584 movl(tmp3, 961);
3585 imull(result, tmp3);
3586 arrays_hashcode_elload(tmp2, Address(ary1, index, Address::times(elsize), -elsize), eltype);
|
572 Label zf_bad_zero;
573 jcc(Assembler::zero, zf_correct);
574 jmp(zf_bad_zero);
575 #endif
576
577 bind(slow_path);
578 if (stub != nullptr) {
579 bind(stub->slow_path_continuation());
580 }
581 #ifdef ASSERT
582 // Check that stub->continuation() label is reached with ZF not set.
583 jcc(Assembler::notZero, zf_correct);
584 stop("Fast Unlock ZF != 0");
585 bind(zf_bad_zero);
586 stop("Fast Unlock ZF != 1");
587 bind(zf_correct);
588 #endif
589 // C2 uses the value of ZF to determine the continuation.
590 }
591
592 void C2_MacroAssembler::abort_verify_int_in_range(uint idx, jint val, jint lo, jint hi) {
593 fatal("Invalid CastII, idx: %u, val: %d, lo: %d, hi: %d", idx, val, lo, hi);
594 }
595
596 static void reconstruct_frame_pointer_helper(MacroAssembler* masm, Register dst) {
597 const int framesize = Compile::current()->output()->frame_size_in_bytes();
598 masm->movptr(dst, rsp);
599 if (framesize > 2 * wordSize) {
600 masm->addptr(dst, framesize - 2 * wordSize);
601 }
602 }
603
604 void C2_MacroAssembler::reconstruct_frame_pointer(Register rtmp) {
605 if (PreserveFramePointer) {
606 // frame pointer is valid
607 #ifdef ASSERT
608 // Verify frame pointer value in rbp.
609 reconstruct_frame_pointer_helper(this, rtmp);
610 Label L_success;
611 cmpq(rbp, rtmp);
612 jccb(Assembler::equal, L_success);
635 jccb(Assembler::less, fail);
636 }
637 if (hi != max_jint) {
638 cmpl(val, hi);
639 jccb(Assembler::greater, fail);
640 }
641 jmpb(succeed);
642
643 bind(fail);
644 movl(c_rarg0, idx);
645 movl(c_rarg1, val);
646 movl(c_rarg2, lo);
647 movl(c_rarg3, hi);
648 reconstruct_frame_pointer(rscratch1);
649 call(RuntimeAddress(CAST_FROM_FN_PTR(address, abort_verify_int_in_range)));
650 hlt();
651 bind(succeed);
652 BLOCK_COMMENT("} // CastII");
653 }
654
655 void C2_MacroAssembler::abort_verify_long_in_range(uint idx, jlong val, jlong lo, jlong hi) {
656 fatal("Invalid CastLL, idx: %u, val: " JLONG_FORMAT ", lo: " JLONG_FORMAT ", hi: " JLONG_FORMAT, idx, val, lo, hi);
657 }
658
659 void C2_MacroAssembler::verify_long_in_range(uint idx, const TypeLong* t, Register val, Register tmp) {
660 jlong lo = t->_lo;
661 jlong hi = t->_hi;
662 assert(lo < hi, "type should not be empty or constant, idx: %u, lo: " JLONG_FORMAT ", hi: " JLONG_FORMAT, idx, lo, hi);
663 if (t == TypeLong::LONG) {
664 return;
665 }
666
667 BLOCK_COMMENT("CastLL {");
668 Label fail;
669 Label succeed;
670
671 auto cmp_val = [&](jlong bound) {
672 if (is_simm32(bound)) {
673 cmpq(val, checked_cast<int>(bound));
674 } else {
675 mov64(tmp, bound);
3544 arrays_hashcode_elvload(vtmp[idx], Address(ary1, index, Address::times(elsize), 8 * idx * elsize), eltype);
3545 }
3546 // vresult = vresult * vnext + ary1[index+8*idx:index+8*idx+7];
3547 for (int idx = 0; idx < 4; idx++) {
3548 vpmulld(vresult[idx], vresult[idx], vnext, Assembler::AVX_256bit);
3549 arrays_hashcode_elvcast(vtmp[idx], eltype);
3550 vpaddd(vresult[idx], vresult[idx], vtmp[idx], Assembler::AVX_256bit);
3551 }
3552 // index += 32;
3553 addl(index, 32);
3554 // index < bound;
3555 cmpl(index, bound);
3556 jcc(Assembler::less, UNROLLED_VECTOR_LOOP_BEGIN);
3557 // }
3558
3559 lea(ary1, Address(ary1, bound, Address::times(elsize)));
3560 subl(cnt1, bound);
3561 // release bound
3562
3563 // vresult *= IntVector.fromArray(I256, power_of_31_backwards, 1);
3564 lea(tmp2, ExternalAddress(StubRoutines::x86::arrays_hashcode_powers_of_31() + (0 * sizeof(jint))));
3565 for (int idx = 0; idx < 4; idx++) {
3566 arrays_hashcode_elvload(vcoef[idx], Address(tmp2, (int)((8 * idx + 1) * sizeof(jint))), T_INT);
3567 vpmulld(vresult[idx], vresult[idx], vcoef[idx], Assembler::AVX_256bit);
3568 }
3569 // result += vresult.reduceLanes(ADD);
3570 for (int idx = 0; idx < 4; idx++) {
3571 reduceI(Op_AddReductionVI, 256/(sizeof(jint) * 8), result, result, vresult[idx], vtmp[(idx * 2 + 0) % 4], vtmp[(idx * 2 + 1) % 4]);
3572 }
3573
3574 // } else if (cnt1 < 32) {
3575
3576 bind(SHORT_UNROLLED_BEGIN);
3577 // int i = 1;
3578 movl(index, 1);
3579 cmpl(index, cnt1);
3580 jcc(Assembler::greaterEqual, SHORT_UNROLLED_LOOP_EXIT);
3581
3582 // for (; i < cnt1 ; i += 2) {
3583 bind(SHORT_UNROLLED_LOOP_BEGIN);
3584 movl(tmp3, 961);
3585 imull(result, tmp3);
3586 arrays_hashcode_elload(tmp2, Address(ary1, index, Address::times(elsize), -elsize), eltype);
|