crypto: x86/aes-gcm - Use new AES library API

+17 -16

arch/x86/crypto/aes-gcm-aesni-x86_64.S

··· 143 143 .octa 0 144 144 145 145 // Offsets in struct aes_gcm_key_aesni 146 - #define OFFSETOF_AESKEYLEN 480 147 - #define OFFSETOF_H_POWERS 496 148 - #define OFFSETOF_H_POWERS_XORED 624 149 - #define OFFSETOF_H_TIMES_X64 688 146 + #define OFFSETOF_AESKEYLEN 0 147 + #define OFFSETOF_AESROUNDKEYS 16 148 + #define OFFSETOF_H_POWERS 272 149 + #define OFFSETOF_H_POWERS_XORED 400 150 + #define OFFSETOF_H_TIMES_X64 464 150 151 151 152 .text 152 153 ··· 506 505 507 506 // Encrypt an all-zeroes block to get the raw hash subkey. 508 507 movl OFFSETOF_AESKEYLEN(KEY), %eax 509 - lea 6*16(KEY,%rax,4), RNDKEYLAST_PTR 510 - movdqa (KEY), H_POW1 // Zero-th round key XOR all-zeroes block 511 - lea 16(KEY), %rax 508 + lea OFFSETOF_AESROUNDKEYS+6*16(KEY,%rax,4), RNDKEYLAST_PTR 509 + movdqa OFFSETOF_AESROUNDKEYS(KEY), H_POW1 510 + lea OFFSETOF_AESROUNDKEYS+16(KEY), %rax 512 511 1: 513 512 aesenc (%rax), H_POW1 514 513 add $16, %rax ··· 625 624 // the zero-th AES round key. Clobbers TMP0 and TMP1. 626 625 .macro _ctr_begin_8x 627 626 movq .Lone(%rip), TMP0 628 - movdqa (KEY), TMP1 // zero-th round key 627 + movdqa OFFSETOF_AESROUNDKEYS(KEY), TMP1 // zero-th round key 629 628 .irp i, 0,1,2,3,4,5,6,7 630 629 _vpshufb BSWAP_MASK, LE_CTR, AESDATA\i 631 630 pxor TMP1, AESDATA\i ··· 727 726 movdqu (LE_CTR_PTR), LE_CTR 728 727 729 728 movl OFFSETOF_AESKEYLEN(KEY), AESKEYLEN 730 - lea 6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR 729 + lea OFFSETOF_AESROUNDKEYS+6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR 731 730 732 731 // If there are at least 8*16 bytes of data, then continue into the main 733 732 // loop, which processes 8*16 bytes of data per iteration. ··· 746 745 .if \enc 747 746 // Encrypt the first 8 plaintext blocks. 748 747 _ctr_begin_8x 749 - lea 16(KEY), %rsi 748 + lea OFFSETOF_AESROUNDKEYS+16(KEY), %rsi 750 749 .p2align 4 751 750 1: 752 751 movdqa (%rsi), TMP0 ··· 768 767 769 768 // Generate the next set of 8 counter blocks and start encrypting them. 770 769 _ctr_begin_8x 771 - lea 16(KEY), %rsi 770 + lea OFFSETOF_AESROUNDKEYS+16(KEY), %rsi 772 771 773 772 // Do a round of AES, and start the GHASH update of 8 ciphertext blocks 774 773 // by doing the unreduced multiplication for the first ciphertext block. ··· 870 869 // Encrypt the next counter block. 871 870 _vpshufb BSWAP_MASK, LE_CTR, TMP0 872 871 paddd ONE, LE_CTR 873 - pxor (KEY), TMP0 872 + pxor OFFSETOF_AESROUNDKEYS(KEY), TMP0 874 873 lea -6*16(RNDKEYLAST_PTR), %rsi // Reduce code size 875 874 cmp $24, AESKEYLEN 876 875 jl 128f // AES-128? ··· 927 926 928 927 // Encrypt a counter block for the last time. 929 928 pshufb BSWAP_MASK, LE_CTR 930 - pxor (KEY), LE_CTR 931 - lea 16(KEY), %rsi 929 + pxor OFFSETOF_AESROUNDKEYS(KEY), LE_CTR 930 + lea OFFSETOF_AESROUNDKEYS+16(KEY), %rsi 932 931 1: 933 932 aesenc (%rsi), LE_CTR 934 933 add $16, %rsi ··· 1039 1038 1040 1039 // Make %rax point to the 6th from last AES round key. (Using signed 1041 1040 // byte offsets -7*16 through 6*16 decreases code size.) 1042 - lea (KEY,AESKEYLEN64,4), %rax 1041 + lea OFFSETOF_AESROUNDKEYS(KEY,AESKEYLEN64,4), %rax 1043 1042 1044 1043 // AES-encrypt the counter block and also multiply GHASH_ACC by H^1. 1045 1044 // Interleave the AES and GHASH instructions to improve performance. 1046 1045 pshufb BSWAP_MASK, %xmm0 1047 - pxor (KEY), %xmm0 1046 + pxor OFFSETOF_AESROUNDKEYS(KEY), %xmm0 1048 1047 cmp $24, AESKEYLEN 1049 1048 jl 128f // AES-128? 1050 1049 je 192f // AES-192?

+11 -10

arch/x86/crypto/aes-gcm-vaes-avx2.S

··· 122 122 .octa 2 123 123 124 124 // Offsets in struct aes_gcm_key_vaes_avx2 125 - #define OFFSETOF_AESKEYLEN 480 126 - #define OFFSETOF_H_POWERS 512 125 + #define OFFSETOF_AESKEYLEN 0 126 + #define OFFSETOF_AESROUNDKEYS 16 127 + #define OFFSETOF_H_POWERS 288 127 128 #define NUM_H_POWERS 8 128 129 #define OFFSETOFEND_H_POWERS (OFFSETOF_H_POWERS + (NUM_H_POWERS * 16)) 129 130 #define OFFSETOF_H_POWERS_XORED OFFSETOFEND_H_POWERS ··· 241 240 242 241 // Encrypt an all-zeroes block to get the raw hash subkey. 243 242 movl OFFSETOF_AESKEYLEN(KEY), %eax 244 - lea 6*16(KEY,%rax,4), RNDKEYLAST_PTR 245 - vmovdqu (KEY), H_CUR_XMM // Zero-th round key XOR all-zeroes block 246 - lea 16(KEY), %rax 243 + lea OFFSETOF_AESROUNDKEYS+6*16(KEY,%rax,4), RNDKEYLAST_PTR 244 + vmovdqu OFFSETOF_AESROUNDKEYS(KEY), H_CUR_XMM 245 + lea OFFSETOF_AESROUNDKEYS+16(KEY), %rax 247 246 1: 248 247 vaesenc (%rax), H_CUR_XMM, H_CUR_XMM 249 248 add $16, %rax ··· 636 635 // the last AES round. Clobbers %rax and TMP0. 637 636 .macro _aesenc_loop vecs:vararg 638 637 _ctr_begin \vecs 639 - lea 16(KEY), %rax 638 + lea OFFSETOF_AESROUNDKEYS+16(KEY), %rax 640 639 .Laesenc_loop\@: 641 640 vbroadcasti128 (%rax), TMP0 642 641 _vaesenc TMP0, \vecs ··· 769 768 // Make RNDKEYLAST_PTR point to the last AES round key. This is the 770 769 // round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 771 770 // respectively. Then load the zero-th and last round keys. 772 - lea 6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR 773 - vbroadcasti128 (KEY), RNDKEY0 771 + lea OFFSETOF_AESROUNDKEYS+6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR 772 + vbroadcasti128 OFFSETOF_AESROUNDKEYS(KEY), RNDKEY0 774 773 vbroadcasti128 (RNDKEYLAST_PTR), RNDKEYLAST 775 774 776 775 // Finish initializing LE_CTR by adding 1 to the second block. ··· 1070 1069 .endif 1071 1070 1072 1071 // Make %rax point to the last AES round key for the chosen AES variant. 1073 - lea 6*16(KEY,AESKEYLEN64,4), %rax 1072 + lea OFFSETOF_AESROUNDKEYS+6*16(KEY,AESKEYLEN64,4), %rax 1074 1073 1075 1074 // Start the AES encryption of the counter block by swapping the counter 1076 1075 // block to big-endian and XOR-ing it with the zero-th AES round key. 1077 1076 vpshufb BSWAP_MASK, LE_CTR, %xmm0 1078 - vpxor (KEY), %xmm0, %xmm0 1077 + vpxor OFFSETOF_AESROUNDKEYS(KEY), %xmm0, %xmm0 1079 1078 1080 1079 // Complete the AES encryption and multiply GHASH_ACC by H^1. 1081 1080 // Interleave the AES and GHASH instructions to improve performance.

+14 -11

arch/x86/crypto/aes-gcm-vaes-avx512.S

··· 86 86 #define NUM_H_POWERS 16 87 87 88 88 // Offset to AES key length (in bytes) in the key struct 89 - #define OFFSETOF_AESKEYLEN 480 89 + #define OFFSETOF_AESKEYLEN 0 90 + 91 + // Offset to AES round keys in the key struct 92 + #define OFFSETOF_AESROUNDKEYS 16 90 93 91 94 // Offset to start of hash key powers array in the key struct 92 - #define OFFSETOF_H_POWERS 512 95 + #define OFFSETOF_H_POWERS 320 93 96 94 97 // Offset to end of hash key powers array in the key struct. 95 98 // ··· 304 301 305 302 // Encrypt an all-zeroes block to get the raw hash subkey. 306 303 movl OFFSETOF_AESKEYLEN(KEY), %eax 307 - lea 6*16(KEY,%rax,4), RNDKEYLAST_PTR 308 - vmovdqu (KEY), %xmm0 // Zero-th round key XOR all-zeroes block 309 - add $16, KEY 304 + lea OFFSETOF_AESROUNDKEYS+6*16(KEY,%rax,4), RNDKEYLAST_PTR 305 + vmovdqu OFFSETOF_AESROUNDKEYS(KEY), %xmm0 306 + add $OFFSETOF_AESROUNDKEYS+16, KEY 310 307 1: 311 308 vaesenc (KEY), %xmm0, %xmm0 312 309 add $16, KEY ··· 793 790 // Make RNDKEYLAST_PTR point to the last AES round key. This is the 794 791 // round key with index 10, 12, or 14 for AES-128, AES-192, or AES-256 795 792 // respectively. Then load the zero-th and last round keys. 796 - lea 6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR 797 - vbroadcasti32x4 (KEY), RNDKEY0 793 + lea OFFSETOF_AESROUNDKEYS+6*16(KEY,AESKEYLEN64,4), RNDKEYLAST_PTR 794 + vbroadcasti32x4 OFFSETOF_AESROUNDKEYS(KEY), RNDKEY0 798 795 vbroadcasti32x4 (RNDKEYLAST_PTR), RNDKEYLAST 799 796 800 797 // Finish initializing LE_CTR by adding [0, 1, ...] to its low words. ··· 837 834 // Encrypt the first 4 vectors of plaintext blocks. Leave the resulting 838 835 // ciphertext in GHASHDATA[0-3] for GHASH. 839 836 _ctr_begin_4x 840 - lea 16(KEY), %rax 837 + lea OFFSETOF_AESROUNDKEYS+16(KEY), %rax 841 838 1: 842 839 vbroadcasti32x4 (%rax), RNDKEY 843 840 _vaesenc_4x RNDKEY ··· 960 957 vpshufb BSWAP_MASK, LE_CTR, %zmm0 961 958 vpaddd LE_CTR_INC, LE_CTR, LE_CTR 962 959 vpxord RNDKEY0, %zmm0, %zmm0 963 - lea 16(KEY), %rax 960 + lea OFFSETOF_AESROUNDKEYS+16(KEY), %rax 964 961 1: 965 962 vbroadcasti32x4 (%rax), RNDKEY 966 963 vaesenc RNDKEY, %zmm0, %zmm0 ··· 1090 1087 .endif 1091 1088 1092 1089 // Make %rax point to the last AES round key for the chosen AES variant. 1093 - lea 6*16(KEY,AESKEYLEN64,4), %rax 1090 + lea OFFSETOF_AESROUNDKEYS+6*16(KEY,AESKEYLEN64,4), %rax 1094 1091 1095 1092 // Start the AES encryption of the counter block by swapping the counter 1096 1093 // block to big-endian and XOR-ing it with the zero-th AES round key. 1097 1094 vpshufb BSWAP_MASK, LE_CTR, %xmm0 1098 - vpxor (KEY), %xmm0, %xmm0 1095 + vpxor OFFSETOF_AESROUNDKEYS(KEY), %xmm0, %xmm0 1099 1096 1100 1097 // Complete the AES encryption and multiply GHASH_ACC by H^1. 1101 1098 // Interleave the AES and GHASH instructions to improve performance.

+25 -32

arch/x86/crypto/aesni-intel_glue.c

··· 780 780 /* The common part of the x86_64 AES-GCM key struct */ 781 781 struct aes_gcm_key { 782 782 /* Expanded AES key and the AES key length in bytes */ 783 - struct crypto_aes_ctx aes_key; 783 + struct aes_enckey aes_key; 784 784 785 785 /* RFC4106 nonce (used only by the rfc4106 algorithms) */ 786 786 u32 rfc4106_nonce; ··· 789 789 /* Key struct used by the AES-NI implementations of AES-GCM */ 790 790 struct aes_gcm_key_aesni { 791 791 /* 792 - * Common part of the key. The assembly code requires 16-byte alignment 793 - * for the round keys; we get this by them being located at the start of 794 - * the struct and the whole struct being 16-byte aligned. 792 + * Common part of the key. 16-byte alignment is required by the 793 + * assembly code. 795 794 */ 796 - struct aes_gcm_key base; 795 + struct aes_gcm_key base __aligned(16); 797 796 798 797 /* 799 798 * Powers of the hash key H^8 through H^1. These are 128-bit values. ··· 823 824 struct aes_gcm_key_vaes_avx2 { 824 825 /* 825 826 * Common part of the key. The assembly code prefers 16-byte alignment 826 - * for the round keys; we get this by them being located at the start of 827 - * the struct and the whole struct being 32-byte aligned. 827 + * for this. 828 828 */ 829 - struct aes_gcm_key base; 829 + struct aes_gcm_key base __aligned(16); 830 830 831 831 /* 832 832 * Powers of the hash key H^8 through H^1. These are 128-bit values. ··· 852 854 struct aes_gcm_key_vaes_avx512 { 853 855 /* 854 856 * Common part of the key. The assembly code prefers 16-byte alignment 855 - * for the round keys; we get this by them being located at the start of 856 - * the struct and the whole struct being 64-byte aligned. 857 + * for this. 857 858 */ 858 - struct aes_gcm_key base; 859 + struct aes_gcm_key base __aligned(16); 859 860 860 861 /* 861 862 * Powers of the hash key H^16 through H^1. These are 128-bit values. ··· 1179 1182 } 1180 1183 1181 1184 /* The assembly code assumes the following offsets. */ 1182 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_enc) != 0); 1183 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_length) != 480); 1184 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers) != 496); 1185 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers_xored) != 624); 1186 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_times_x64) != 688); 1187 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.key_enc) != 0); 1188 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.key_length) != 480); 1189 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, h_powers) != 512); 1190 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, h_powers_xored) != 640); 1191 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.key_enc) != 0); 1192 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.key_length) != 480); 1193 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, h_powers) != 512); 1194 - BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, padding) != 768); 1185 + static_assert(offsetof(struct aes_gcm_key_aesni, base.aes_key.len) == 0); 1186 + static_assert(offsetof(struct aes_gcm_key_aesni, base.aes_key.k.rndkeys) == 16); 1187 + static_assert(offsetof(struct aes_gcm_key_aesni, h_powers) == 272); 1188 + static_assert(offsetof(struct aes_gcm_key_aesni, h_powers_xored) == 400); 1189 + static_assert(offsetof(struct aes_gcm_key_aesni, h_times_x64) == 464); 1190 + static_assert(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.len) == 0); 1191 + static_assert(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.k.rndkeys) == 16); 1192 + static_assert(offsetof(struct aes_gcm_key_vaes_avx2, h_powers) == 288); 1193 + static_assert(offsetof(struct aes_gcm_key_vaes_avx2, h_powers_xored) == 416); 1194 + static_assert(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.len) == 0); 1195 + static_assert(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.k.rndkeys) == 16); 1196 + static_assert(offsetof(struct aes_gcm_key_vaes_avx512, h_powers) == 320); 1197 + static_assert(offsetof(struct aes_gcm_key_vaes_avx512, padding) == 576); 1198 + 1199 + err = aes_prepareenckey(&key->aes_key, raw_key, keylen); 1200 + if (err) 1201 + return err; 1195 1202 1196 1203 if (likely(crypto_simd_usable())) { 1197 - err = aes_check_keylen(keylen); 1198 - if (err) 1199 - return err; 1200 1204 kernel_fpu_begin(); 1201 - aesni_set_key(&key->aes_key, raw_key, keylen); 1202 1205 aes_gcm_precompute(key, flags); 1203 1206 kernel_fpu_end(); 1204 1207 } else { ··· 1211 1214 be128 h1 = {}; 1212 1215 be128 h; 1213 1216 int i; 1214 - 1215 - err = aes_expandkey(&key->aes_key, raw_key, keylen); 1216 - if (err) 1217 - return err; 1218 1217 1219 1218 /* Encrypt the all-zeroes block to get the hash key H^1 */ 1220 1219 aes_encrypt(&key->aes_key, (u8 *)&h1, (u8 *)&h1);

Configure Feed

Configure Feed