571 __ addptr(rsp, wordSize * 2);
572
573 __ ret(0);
574
575 return start;
576 }
577 //---------------------------------------------------------------------------------------------------
578
579 address generate_vector_mask(const char *stub_name, int32_t mask) {
580 __ align(CodeEntryAlignment);
581 StubCodeMark mark(this, "StubRoutines", stub_name);
582 address start = __ pc();
583
584 for (int i = 0; i < 16; i++) {
585 __ emit_data(mask, relocInfo::none, 0);
586 }
587
588 return start;
589 }
590
591 address generate_popcount_avx_lut(const char *stub_name) {
592 __ align64();
593 StubCodeMark mark(this, "StubRoutines", stub_name);
594 address start = __ pc();
595 __ emit_data(0x02010100, relocInfo::none, 0);
596 __ emit_data(0x03020201, relocInfo::none, 0);
597 __ emit_data(0x03020201, relocInfo::none, 0);
598 __ emit_data(0x04030302, relocInfo::none, 0);
599 __ emit_data(0x02010100, relocInfo::none, 0);
600 __ emit_data(0x03020201, relocInfo::none, 0);
601 __ emit_data(0x03020201, relocInfo::none, 0);
602 __ emit_data(0x04030302, relocInfo::none, 0);
603 __ emit_data(0x02010100, relocInfo::none, 0);
604 __ emit_data(0x03020201, relocInfo::none, 0);
605 __ emit_data(0x03020201, relocInfo::none, 0);
606 __ emit_data(0x04030302, relocInfo::none, 0);
607 __ emit_data(0x02010100, relocInfo::none, 0);
608 __ emit_data(0x03020201, relocInfo::none, 0);
609 __ emit_data(0x03020201, relocInfo::none, 0);
610 __ emit_data(0x04030302, relocInfo::none, 0);
618 address start = __ pc();
619 __ emit_data(0x03020100, relocInfo::none, 0);
620 __ emit_data(0x07060504, relocInfo::none, 0);
621 __ emit_data(0x0B0A0908, relocInfo::none, 0);
622 __ emit_data(0x0F0E0D0C, relocInfo::none, 0);
623 __ emit_data(0x13121110, relocInfo::none, 0);
624 __ emit_data(0x17161514, relocInfo::none, 0);
625 __ emit_data(0x1B1A1918, relocInfo::none, 0);
626 __ emit_data(0x1F1E1D1C, relocInfo::none, 0);
627 __ emit_data(0x23222120, relocInfo::none, 0);
628 __ emit_data(0x27262524, relocInfo::none, 0);
629 __ emit_data(0x2B2A2928, relocInfo::none, 0);
630 __ emit_data(0x2F2E2D2C, relocInfo::none, 0);
631 __ emit_data(0x33323130, relocInfo::none, 0);
632 __ emit_data(0x37363534, relocInfo::none, 0);
633 __ emit_data(0x3B3A3938, relocInfo::none, 0);
634 __ emit_data(0x3F3E3D3C, relocInfo::none, 0);
635 return start;
636 }
637
638 address generate_vector_byte_shuffle_mask(const char *stub_name) {
639 __ align(CodeEntryAlignment);
640 StubCodeMark mark(this, "StubRoutines", stub_name);
641 address start = __ pc();
642 __ emit_data(0x70707070, relocInfo::none, 0);
643 __ emit_data(0x70707070, relocInfo::none, 0);
644 __ emit_data(0x70707070, relocInfo::none, 0);
645 __ emit_data(0x70707070, relocInfo::none, 0);
646 __ emit_data(0xF0F0F0F0, relocInfo::none, 0);
647 __ emit_data(0xF0F0F0F0, relocInfo::none, 0);
648 __ emit_data(0xF0F0F0F0, relocInfo::none, 0);
649 __ emit_data(0xF0F0F0F0, relocInfo::none, 0);
650 return start;
651 }
652
653 address generate_vector_mask_long_double(const char *stub_name, int32_t maskhi, int32_t masklo) {
654 __ align(CodeEntryAlignment);
655 StubCodeMark mark(this, "StubRoutines", stub_name);
656 address start = __ pc();
657
3968 StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF);
3969 StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x80000000);
3970 StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask_long_double("vector_double_sign_mask", 0x7FFFFFFF, 0xFFFFFFFF);
3971 StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask_long_double("vector_double_sign_flip", 0x80000000, 0x00000000);
3972 StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff);
3973 StubRoutines::x86::_vector_int_to_byte_mask = generate_vector_mask("vector_int_to_byte_mask", 0x000000ff);
3974 StubRoutines::x86::_vector_int_to_short_mask = generate_vector_mask("vector_int_to_short_mask", 0x0000ffff);
3975 StubRoutines::x86::_vector_32_bit_mask = generate_vector_custom_i32("vector_32_bit_mask", Assembler::AVX_512bit,
3976 0xFFFFFFFF, 0, 0, 0);
3977 StubRoutines::x86::_vector_64_bit_mask = generate_vector_custom_i32("vector_64_bit_mask", Assembler::AVX_512bit,
3978 0xFFFFFFFF, 0xFFFFFFFF, 0, 0);
3979 StubRoutines::x86::_vector_int_shuffle_mask = generate_vector_mask("vector_int_shuffle_mask", 0x03020100);
3980 StubRoutines::x86::_vector_byte_shuffle_mask = generate_vector_byte_shuffle_mask("vector_byte_shuffle_mask");
3981 StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask("vector_short_shuffle_mask", 0x01000100);
3982 StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask_long_double("vector_long_shuffle_mask", 0x00000001, 0x0);
3983 StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
3984 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask_long_double("vector_long_sign_mask", 0x80000000, 0x00000000);
3985 StubRoutines::x86::_vector_all_bits_set = generate_vector_mask("vector_all_bits_set", 0xFFFFFFFF);
3986 StubRoutines::x86::_vector_int_mask_cmp_bits = generate_vector_mask("vector_int_mask_cmp_bits", 0x00000001);
3987 StubRoutines::x86::_vector_iota_indices = generate_iota_indices("iota_indices");
3988
3989 if (UsePopCountInstruction && VM_Version::supports_avx2() && !VM_Version::supports_avx512_vpopcntdq()) {
3990 // lut implementation influenced by counting 1s algorithm from section 5-1 of Hackers' Delight.
3991 StubRoutines::x86::_vector_popcount_lut = generate_popcount_avx_lut("popcount_lut");
3992 }
3993
3994 // support for verify_oop (must happen after universe_init)
3995 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
3996
3997 // arraycopy stubs used by compilers
3998 generate_arraycopy_stubs();
3999
4000 // don't bother generating these AES intrinsic stubs unless global flag is set
4001 if (UseAESIntrinsics) {
4002 StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // might be needed by the others
4003
4004 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
4005 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
4006 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
4007 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
4008 }
4009
|
571 __ addptr(rsp, wordSize * 2);
572
573 __ ret(0);
574
575 return start;
576 }
577 //---------------------------------------------------------------------------------------------------
578
579 address generate_vector_mask(const char *stub_name, int32_t mask) {
580 __ align(CodeEntryAlignment);
581 StubCodeMark mark(this, "StubRoutines", stub_name);
582 address start = __ pc();
583
584 for (int i = 0; i < 16; i++) {
585 __ emit_data(mask, relocInfo::none, 0);
586 }
587
588 return start;
589 }
590
591 address generate_count_leading_zeros_lut(const char *stub_name) {
592 __ align64();
593 StubCodeMark mark(this, "StubRoutines", stub_name);
594 address start = __ pc();
595 __ emit_data(0x02020304, relocInfo::none, 0);
596 __ emit_data(0x01010101, relocInfo::none, 0);
597 __ emit_data(0x00000000, relocInfo::none, 0);
598 __ emit_data(0x00000000, relocInfo::none, 0);
599 __ emit_data(0x02020304, relocInfo::none, 0);
600 __ emit_data(0x01010101, relocInfo::none, 0);
601 __ emit_data(0x00000000, relocInfo::none, 0);
602 __ emit_data(0x00000000, relocInfo::none, 0);
603 __ emit_data(0x02020304, relocInfo::none, 0);
604 __ emit_data(0x01010101, relocInfo::none, 0);
605 __ emit_data(0x00000000, relocInfo::none, 0);
606 __ emit_data(0x00000000, relocInfo::none, 0);
607 __ emit_data(0x02020304, relocInfo::none, 0);
608 __ emit_data(0x01010101, relocInfo::none, 0);
609 __ emit_data(0x00000000, relocInfo::none, 0);
610 __ emit_data(0x00000000, relocInfo::none, 0);
611 return start;
612 }
613
614
615 address generate_popcount_avx_lut(const char *stub_name) {
616 __ align64();
617 StubCodeMark mark(this, "StubRoutines", stub_name);
618 address start = __ pc();
619 __ emit_data(0x02010100, relocInfo::none, 0);
620 __ emit_data(0x03020201, relocInfo::none, 0);
621 __ emit_data(0x03020201, relocInfo::none, 0);
622 __ emit_data(0x04030302, relocInfo::none, 0);
623 __ emit_data(0x02010100, relocInfo::none, 0);
624 __ emit_data(0x03020201, relocInfo::none, 0);
625 __ emit_data(0x03020201, relocInfo::none, 0);
626 __ emit_data(0x04030302, relocInfo::none, 0);
627 __ emit_data(0x02010100, relocInfo::none, 0);
628 __ emit_data(0x03020201, relocInfo::none, 0);
629 __ emit_data(0x03020201, relocInfo::none, 0);
630 __ emit_data(0x04030302, relocInfo::none, 0);
631 __ emit_data(0x02010100, relocInfo::none, 0);
632 __ emit_data(0x03020201, relocInfo::none, 0);
633 __ emit_data(0x03020201, relocInfo::none, 0);
634 __ emit_data(0x04030302, relocInfo::none, 0);
642 address start = __ pc();
643 __ emit_data(0x03020100, relocInfo::none, 0);
644 __ emit_data(0x07060504, relocInfo::none, 0);
645 __ emit_data(0x0B0A0908, relocInfo::none, 0);
646 __ emit_data(0x0F0E0D0C, relocInfo::none, 0);
647 __ emit_data(0x13121110, relocInfo::none, 0);
648 __ emit_data(0x17161514, relocInfo::none, 0);
649 __ emit_data(0x1B1A1918, relocInfo::none, 0);
650 __ emit_data(0x1F1E1D1C, relocInfo::none, 0);
651 __ emit_data(0x23222120, relocInfo::none, 0);
652 __ emit_data(0x27262524, relocInfo::none, 0);
653 __ emit_data(0x2B2A2928, relocInfo::none, 0);
654 __ emit_data(0x2F2E2D2C, relocInfo::none, 0);
655 __ emit_data(0x33323130, relocInfo::none, 0);
656 __ emit_data(0x37363534, relocInfo::none, 0);
657 __ emit_data(0x3B3A3938, relocInfo::none, 0);
658 __ emit_data(0x3F3E3D3C, relocInfo::none, 0);
659 return start;
660 }
661
662 address generate_vector_reverse_bit_lut(const char *stub_name) {
663 __ align(CodeEntryAlignment);
664 StubCodeMark mark(this, "StubRoutines", stub_name);
665 address start = __ pc();
666 __ emit_data(0x0C040800, relocInfo::none, 0);
667 __ emit_data(0x0E060A02, relocInfo::none, 0);
668 __ emit_data(0x0D050901, relocInfo::none, 0);
669 __ emit_data(0x0F070B03, relocInfo::none, 0);
670 __ emit_data(0x0C040800, relocInfo::none, 0);
671 __ emit_data(0x0E060A02, relocInfo::none, 0);
672 __ emit_data(0x0D050901, relocInfo::none, 0);
673 __ emit_data(0x0F070B03, relocInfo::none, 0);
674 __ emit_data(0x0C040800, relocInfo::none, 0);
675 __ emit_data(0x0E060A02, relocInfo::none, 0);
676 __ emit_data(0x0D050901, relocInfo::none, 0);
677 __ emit_data(0x0F070B03, relocInfo::none, 0);
678 __ emit_data(0x0C040800, relocInfo::none, 0);
679 __ emit_data(0x0E060A02, relocInfo::none, 0);
680 __ emit_data(0x0D050901, relocInfo::none, 0);
681 __ emit_data(0x0F070B03, relocInfo::none, 0);
682 return start;
683 }
684
685 address generate_vector_reverse_byte_perm_mask_long(const char *stub_name) {
686 __ align(CodeEntryAlignment);
687 StubCodeMark mark(this, "StubRoutines", stub_name);
688 address start = __ pc();
689 __ emit_data(0x04050607, relocInfo::none, 0);
690 __ emit_data(0x00010203, relocInfo::none, 0);
691 __ emit_data(0x0C0D0E0F, relocInfo::none, 0);
692 __ emit_data(0x08090A0B, relocInfo::none, 0);
693 __ emit_data(0x04050607, relocInfo::none, 0);
694 __ emit_data(0x00010203, relocInfo::none, 0);
695 __ emit_data(0x0C0D0E0F, relocInfo::none, 0);
696 __ emit_data(0x08090A0B, relocInfo::none, 0);
697 __ emit_data(0x04050607, relocInfo::none, 0);
698 __ emit_data(0x00010203, relocInfo::none, 0);
699 __ emit_data(0x0C0D0E0F, relocInfo::none, 0);
700 __ emit_data(0x08090A0B, relocInfo::none, 0);
701 __ emit_data(0x04050607, relocInfo::none, 0);
702 __ emit_data(0x00010203, relocInfo::none, 0);
703 __ emit_data(0x0C0D0E0F, relocInfo::none, 0);
704 __ emit_data(0x08090A0B, relocInfo::none, 0);
705 return start;
706 }
707
708 address generate_vector_reverse_byte_perm_mask_int(const char *stub_name) {
709 __ align(CodeEntryAlignment);
710 StubCodeMark mark(this, "StubRoutines", stub_name);
711 address start = __ pc();
712 __ emit_data(0x00010203, relocInfo::none, 0);
713 __ emit_data(0x04050607, relocInfo::none, 0);
714 __ emit_data(0x08090A0B, relocInfo::none, 0);
715 __ emit_data(0x0C0D0E0F, relocInfo::none, 0);
716 __ emit_data(0x00010203, relocInfo::none, 0);
717 __ emit_data(0x04050607, relocInfo::none, 0);
718 __ emit_data(0x08090A0B, relocInfo::none, 0);
719 __ emit_data(0x0C0D0E0F, relocInfo::none, 0);
720 __ emit_data(0x00010203, relocInfo::none, 0);
721 __ emit_data(0x04050607, relocInfo::none, 0);
722 __ emit_data(0x08090A0B, relocInfo::none, 0);
723 __ emit_data(0x0C0D0E0F, relocInfo::none, 0);
724 __ emit_data(0x00010203, relocInfo::none, 0);
725 __ emit_data(0x04050607, relocInfo::none, 0);
726 __ emit_data(0x08090A0B, relocInfo::none, 0);
727 __ emit_data(0x0C0D0E0F, relocInfo::none, 0);
728 return start;
729 }
730
731 address generate_vector_reverse_byte_perm_mask_short(const char *stub_name) {
732 __ align(CodeEntryAlignment);
733 StubCodeMark mark(this, "StubRoutines", stub_name);
734 address start = __ pc();
735 __ emit_data(0x02030001, relocInfo::none, 0);
736 __ emit_data(0x06070405, relocInfo::none, 0);
737 __ emit_data(0x0A0B0809, relocInfo::none, 0);
738 __ emit_data(0x0E0F0C0D, relocInfo::none, 0);
739 __ emit_data(0x02030001, relocInfo::none, 0);
740 __ emit_data(0x06070405, relocInfo::none, 0);
741 __ emit_data(0x0A0B0809, relocInfo::none, 0);
742 __ emit_data(0x0E0F0C0D, relocInfo::none, 0);
743 __ emit_data(0x02030001, relocInfo::none, 0);
744 __ emit_data(0x06070405, relocInfo::none, 0);
745 __ emit_data(0x0A0B0809, relocInfo::none, 0);
746 __ emit_data(0x0E0F0C0D, relocInfo::none, 0);
747 __ emit_data(0x02030001, relocInfo::none, 0);
748 __ emit_data(0x06070405, relocInfo::none, 0);
749 __ emit_data(0x0A0B0809, relocInfo::none, 0);
750 __ emit_data(0x0E0F0C0D, relocInfo::none, 0);
751 return start;
752 }
753
754 address generate_vector_byte_shuffle_mask(const char *stub_name) {
755 __ align(CodeEntryAlignment);
756 StubCodeMark mark(this, "StubRoutines", stub_name);
757 address start = __ pc();
758 __ emit_data(0x70707070, relocInfo::none, 0);
759 __ emit_data(0x70707070, relocInfo::none, 0);
760 __ emit_data(0x70707070, relocInfo::none, 0);
761 __ emit_data(0x70707070, relocInfo::none, 0);
762 __ emit_data(0xF0F0F0F0, relocInfo::none, 0);
763 __ emit_data(0xF0F0F0F0, relocInfo::none, 0);
764 __ emit_data(0xF0F0F0F0, relocInfo::none, 0);
765 __ emit_data(0xF0F0F0F0, relocInfo::none, 0);
766 return start;
767 }
768
769 address generate_vector_mask_long_double(const char *stub_name, int32_t maskhi, int32_t masklo) {
770 __ align(CodeEntryAlignment);
771 StubCodeMark mark(this, "StubRoutines", stub_name);
772 address start = __ pc();
773
4084 StubRoutines::x86::_vector_float_sign_mask = generate_vector_mask("vector_float_sign_mask", 0x7FFFFFFF);
4085 StubRoutines::x86::_vector_float_sign_flip = generate_vector_mask("vector_float_sign_flip", 0x80000000);
4086 StubRoutines::x86::_vector_double_sign_mask = generate_vector_mask_long_double("vector_double_sign_mask", 0x7FFFFFFF, 0xFFFFFFFF);
4087 StubRoutines::x86::_vector_double_sign_flip = generate_vector_mask_long_double("vector_double_sign_flip", 0x80000000, 0x00000000);
4088 StubRoutines::x86::_vector_short_to_byte_mask = generate_vector_mask("vector_short_to_byte_mask", 0x00ff00ff);
4089 StubRoutines::x86::_vector_int_to_byte_mask = generate_vector_mask("vector_int_to_byte_mask", 0x000000ff);
4090 StubRoutines::x86::_vector_int_to_short_mask = generate_vector_mask("vector_int_to_short_mask", 0x0000ffff);
4091 StubRoutines::x86::_vector_32_bit_mask = generate_vector_custom_i32("vector_32_bit_mask", Assembler::AVX_512bit,
4092 0xFFFFFFFF, 0, 0, 0);
4093 StubRoutines::x86::_vector_64_bit_mask = generate_vector_custom_i32("vector_64_bit_mask", Assembler::AVX_512bit,
4094 0xFFFFFFFF, 0xFFFFFFFF, 0, 0);
4095 StubRoutines::x86::_vector_int_shuffle_mask = generate_vector_mask("vector_int_shuffle_mask", 0x03020100);
4096 StubRoutines::x86::_vector_byte_shuffle_mask = generate_vector_byte_shuffle_mask("vector_byte_shuffle_mask");
4097 StubRoutines::x86::_vector_short_shuffle_mask = generate_vector_mask("vector_short_shuffle_mask", 0x01000100);
4098 StubRoutines::x86::_vector_long_shuffle_mask = generate_vector_mask_long_double("vector_long_shuffle_mask", 0x00000001, 0x0);
4099 StubRoutines::x86::_vector_byte_perm_mask = generate_vector_byte_perm_mask("vector_byte_perm_mask");
4100 StubRoutines::x86::_vector_long_sign_mask = generate_vector_mask_long_double("vector_long_sign_mask", 0x80000000, 0x00000000);
4101 StubRoutines::x86::_vector_all_bits_set = generate_vector_mask("vector_all_bits_set", 0xFFFFFFFF);
4102 StubRoutines::x86::_vector_int_mask_cmp_bits = generate_vector_mask("vector_int_mask_cmp_bits", 0x00000001);
4103 StubRoutines::x86::_vector_iota_indices = generate_iota_indices("iota_indices");
4104 StubRoutines::x86::_vector_count_leading_zeros_lut = generate_count_leading_zeros_lut("count_leading_zeros_lut");
4105 StubRoutines::x86::_vector_reverse_bit_lut = generate_vector_reverse_bit_lut("reverse_bit_lut");
4106 StubRoutines::x86::_vector_reverse_byte_perm_mask_long = generate_vector_reverse_byte_perm_mask_long("perm_mask_long");
4107 StubRoutines::x86::_vector_reverse_byte_perm_mask_int = generate_vector_reverse_byte_perm_mask_int("perm_mask_int");
4108 StubRoutines::x86::_vector_reverse_byte_perm_mask_short = generate_vector_reverse_byte_perm_mask_short("perm_mask_short");
4109
4110 if (VM_Version::supports_avx2() && !VM_Version::supports_avx512_vpopcntdq()) {
4111 // lut implementation influenced by counting 1s algorithm from section 5-1 of Hackers' Delight.
4112 StubRoutines::x86::_vector_popcount_lut = generate_popcount_avx_lut("popcount_lut");
4113 }
4114
4115 // support for verify_oop (must happen after universe_init)
4116 StubRoutines::_verify_oop_subroutine_entry = generate_verify_oop();
4117
4118 // arraycopy stubs used by compilers
4119 generate_arraycopy_stubs();
4120
4121 // don't bother generating these AES intrinsic stubs unless global flag is set
4122 if (UseAESIntrinsics) {
4123 StubRoutines::x86::_key_shuffle_mask_addr = generate_key_shuffle_mask(); // might be needed by the others
4124
4125 StubRoutines::_aescrypt_encryptBlock = generate_aescrypt_encryptBlock();
4126 StubRoutines::_aescrypt_decryptBlock = generate_aescrypt_decryptBlock();
4127 StubRoutines::_cipherBlockChaining_encryptAESCrypt = generate_cipherBlockChaining_encryptAESCrypt();
4128 StubRoutines::_cipherBlockChaining_decryptAESCrypt = generate_cipherBlockChaining_decryptAESCrypt_Parallel();
4129 }
4130
|