Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: aes: Introduce improved AES library
The kernel's AES library currently has the following issues:
- It doesn't take advantage of the architecture-optimized AES code,
including the implementations using AES instructions.
- It's much slower than even the other software AES implementations: 2-4
times slower than "aes-generic", "aes-arm", and "aes-arm64".
- It requires that both the encryption and decryption round keys be
computed and cached. This is wasteful for users that need only the
forward (encryption) direction of the cipher: the key struct is 484
bytes when only 244 are actually needed. This missed optimization is
very common, as many AES modes (e.g. GCM, CFB, CTR, CMAC, and even the
tweak key in XTS) use the cipher only in the forward (encryption)
direction even when doing decryption.
- It doesn't provide the flexibility to customize the prepared key
format. The API is defined to do key expansion, and several callers
in drivers/crypto/ use it specifically to expand the key. This is an
issue when integrating the existing powerpc, s390, and sparc code,
which is necessary to provide full parity with the traditional API.
To resolve these issues, I'm proposing the following changes:
1. New structs 'aes_key' and 'aes_enckey' are introduced, with
corresponding functions aes_preparekey() and aes_prepareenckey().
Generally these structs will include the encryption+decryption round
keys and the encryption round keys, respectively. However, the exact
format will be under control of the architecture-specific AES code.
(The verb "prepare" is chosen over "expand" since key expansion isn't
necessarily done. It's also consistent with hmac*_preparekey().)
2. aes_encrypt() and aes_decrypt() will be changed to operate on the new
structs instead of struct crypto_aes_ctx.
3. aes_encrypt() and aes_decrypt() will use architecture-optimized code
when available, or else fall back to a new generic AES implementation
that unifies the existing two fragmented generic AES implementations.
The new generic AES implementation uses tables for both SubBytes and
MixColumns, making it almost as fast as "aes-generic". However,
instead of aes-generic's huge 8192-byte tables per direction, it uses
only 1024 bytes for encryption and 1280 bytes for decryption (similar
to "aes-arm"). The cost is just some extra rotations.
The new generic AES implementation also includes table prefetching,
making it have some "constant-time hardening". That's an improvement
from aes-generic which has no constant-time hardening.
It does slightly regress in constant-time hardening vs. the old
lib/crypto/aes.c which had smaller tables, and from aes-fixed-time
which disabled IRQs on top of that. But I think this is tolerable.
The real solutions for constant-time AES are AES instructions or
bit-slicing. The table-based code remains a best-effort fallback for
the increasingly-rare case where a real solution is unavailable.
4. crypto_aes_ctx and aes_expandkey() will remain for now, but only for
callers that are using them specifically for the AES key expansion
(as opposed to en/decrypting data with the AES library).
This commit begins the migration process by introducing the new structs
and functions, backed by the new generic AES implementation.
To allow callers to be incrementally converted, aes_encrypt() and
aes_decrypt() are temporarily changed into macros that use a _Generic
expression to call either the old functions (which take crypto_aes_ctx)
or the new functions (which take the new types). Once all callers have
been updated, these macros will go away, the old functions will be
removed, and the "_new" suffix will be dropped from the new functions.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260112192035.10427-3-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+500
-69
+146
-11
include/crypto/aes.h
+146
-11
include/crypto/aes.h
···
18
18
#define AES_MAX_KEYLENGTH (15 * 16)
19
19
#define AES_MAX_KEYLENGTH_U32 (AES_MAX_KEYLENGTH / sizeof(u32))
20
20
21
+
union aes_enckey_arch {
22
+
u32 rndkeys[AES_MAX_KEYLENGTH_U32];
23
+
};
24
+
25
+
union aes_invkey_arch {
26
+
u32 inv_rndkeys[AES_MAX_KEYLENGTH_U32];
27
+
};
28
+
29
+
/**
30
+
* struct aes_enckey - An AES key prepared for encryption
31
+
* @len: Key length in bytes: 16 for AES-128, 24 for AES-192, 32 for AES-256.
32
+
* @nrounds: Number of rounds: 10 for AES-128, 12 for AES-192, 14 for AES-256.
33
+
* This is '6 + @len / 4' and is cached so that AES implementations
34
+
* that need it don't have to recompute it for each en/decryption.
35
+
* @padding: Padding to make offsetof(@k) be a multiple of 16, so that aligning
36
+
* this struct to a 16-byte boundary results in @k also being 16-byte
37
+
* aligned. Users aren't required to align this struct to 16 bytes,
38
+
* but it may slightly improve performance.
39
+
* @k: This typically contains the AES round keys as an array of '@nrounds + 1'
40
+
* groups of four u32 words. However, architecture-specific implementations
41
+
* of AES may store something else here, e.g. just the raw key if it's all
42
+
* they need.
43
+
*
44
+
* Note that this struct is about half the size of struct aes_key. This is
45
+
* separate from struct aes_key so that modes that need only AES encryption
46
+
* (e.g. AES-GCM, AES-CTR, AES-CMAC, tweak key in AES-XTS) don't incur the time
47
+
* and space overhead of computing and caching the decryption round keys.
48
+
*
49
+
* Note that there's no decryption-only equivalent (i.e. "struct aes_deckey"),
50
+
* since (a) it's rare that modes need decryption-only, and (b) some AES
51
+
* implementations use the same @k for both encryption and decryption, either
52
+
* always or conditionally; in the latter case both @k and @inv_k are needed.
53
+
*/
54
+
struct aes_enckey {
55
+
u32 len;
56
+
u32 nrounds;
57
+
u32 padding[2];
58
+
union aes_enckey_arch k;
59
+
};
60
+
61
+
/**
62
+
* struct aes_key - An AES key prepared for encryption and decryption
63
+
* @aes_enckey: Common fields and the key prepared for encryption
64
+
* @inv_k: This generally contains the round keys for the AES Equivalent
65
+
* Inverse Cipher, as an array of '@nrounds + 1' groups of four u32
66
+
* words. However, architecture-specific implementations of AES may
67
+
* store something else here. For example, they may leave this field
68
+
* uninitialized if they use @k for both encryption and decryption.
69
+
*/
70
+
struct aes_key {
71
+
struct aes_enckey; /* Include all fields of aes_enckey. */
72
+
union aes_invkey_arch inv_k;
73
+
};
74
+
21
75
/*
22
76
* Please ensure that the first two fields are 16-byte aligned
23
77
* relative to the start of the structure, i.e., don't move them!
···
88
34
/*
89
35
* validate key length for AES algorithms
90
36
*/
91
-
static inline int aes_check_keylen(unsigned int keylen)
37
+
static inline int aes_check_keylen(size_t keylen)
92
38
{
93
39
switch (keylen) {
94
40
case AES_KEYSIZE_128:
···
123
69
unsigned int key_len);
124
70
125
71
/**
126
-
* aes_encrypt - Encrypt a single AES block
127
-
* @ctx: Context struct containing the key schedule
128
-
* @out: Buffer to store the ciphertext
129
-
* @in: Buffer containing the plaintext
72
+
* aes_preparekey() - Prepare an AES key for encryption and decryption
73
+
* @key: (output) The key structure to initialize
74
+
* @in_key: The raw AES key
75
+
* @key_len: Length of the raw key in bytes. Should be either AES_KEYSIZE_128,
76
+
* AES_KEYSIZE_192, or AES_KEYSIZE_256.
77
+
*
78
+
* This prepares an AES key for both the encryption and decryption directions of
79
+
* the block cipher. Typically this involves expanding the raw key into both
80
+
* the standard round keys and the Equivalent Inverse Cipher round keys, but
81
+
* some architecture-specific implementations don't do the full expansion here.
82
+
*
83
+
* The caller is responsible for zeroizing both the struct aes_key and the raw
84
+
* key once they are no longer needed.
85
+
*
86
+
* If you don't need decryption support, use aes_prepareenckey() instead.
87
+
*
88
+
* Return: 0 on success or -EINVAL if the given key length is invalid. No other
89
+
* errors are possible, so callers that always pass a valid key length
90
+
* don't need to check for errors.
91
+
*
92
+
* Context: Any context.
130
93
*/
131
-
void aes_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);
94
+
int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len);
132
95
133
96
/**
134
-
* aes_decrypt - Decrypt a single AES block
135
-
* @ctx: Context struct containing the key schedule
136
-
* @out: Buffer to store the plaintext
137
-
* @in: Buffer containing the ciphertext
97
+
* aes_prepareenckey() - Prepare an AES key for encryption-only
98
+
* @key: (output) The key structure to initialize
99
+
* @in_key: The raw AES key
100
+
* @key_len: Length of the raw key in bytes. Should be either AES_KEYSIZE_128,
101
+
* AES_KEYSIZE_192, or AES_KEYSIZE_256.
102
+
*
103
+
* This prepares an AES key for only the encryption direction of the block
104
+
* cipher. Typically this involves expanding the raw key into only the standard
105
+
* round keys, resulting in a struct about half the size of struct aes_key.
106
+
*
107
+
* The caller is responsible for zeroizing both the struct aes_enckey and the
108
+
* raw key once they are no longer needed.
109
+
*
110
+
* Note that while the resulting prepared key supports only AES encryption, it
111
+
* can still be used for decrypting in a mode of operation that uses AES in only
112
+
* the encryption (forward) direction, for example counter mode.
113
+
*
114
+
* Return: 0 on success or -EINVAL if the given key length is invalid. No other
115
+
* errors are possible, so callers that always pass a valid key length
116
+
* don't need to check for errors.
117
+
*
118
+
* Context: Any context.
138
119
*/
139
-
void aes_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);
120
+
int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len);
121
+
122
+
typedef union {
123
+
const struct aes_enckey *enc_key;
124
+
const struct aes_key *full_key;
125
+
} aes_encrypt_arg __attribute__ ((__transparent_union__));
126
+
127
+
/**
128
+
* aes_encrypt() - Encrypt a single AES block
129
+
* @key: The AES key, as a pointer to either an encryption-only key
130
+
* (struct aes_enckey) or a full, bidirectional key (struct aes_key).
131
+
* @out: Buffer to store the ciphertext block
132
+
* @in: Buffer containing the plaintext block
133
+
*
134
+
* Context: Any context.
135
+
*/
136
+
#define aes_encrypt(key, out, in) \
137
+
_Generic((key), \
138
+
struct crypto_aes_ctx *: aes_encrypt_old((const struct crypto_aes_ctx *)(key), (out), (in)), \
139
+
const struct crypto_aes_ctx *: aes_encrypt_old((const struct crypto_aes_ctx *)(key), (out), (in)), \
140
+
struct aes_enckey *: aes_encrypt_new((const struct aes_enckey *)(key), (out), (in)), \
141
+
const struct aes_enckey *: aes_encrypt_new((const struct aes_enckey *)(key), (out), (in)), \
142
+
struct aes_key *: aes_encrypt_new((const struct aes_key *)(key), (out), (in)), \
143
+
const struct aes_key *: aes_encrypt_new((const struct aes_key *)(key), (out), (in)))
144
+
void aes_encrypt_old(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);
145
+
void aes_encrypt_new(aes_encrypt_arg key, u8 out[at_least AES_BLOCK_SIZE],
146
+
const u8 in[at_least AES_BLOCK_SIZE]);
147
+
148
+
/**
149
+
* aes_decrypt() - Decrypt a single AES block
150
+
* @key: The AES key, previously initialized by aes_preparekey()
151
+
* @out: Buffer to store the plaintext block
152
+
* @in: Buffer containing the ciphertext block
153
+
*
154
+
* Context: Any context.
155
+
*/
156
+
#define aes_decrypt(key, out, in) \
157
+
_Generic((key), \
158
+
struct crypto_aes_ctx *: aes_decrypt_old((const struct crypto_aes_ctx *)(key), (out), (in)), \
159
+
const struct crypto_aes_ctx *: aes_decrypt_old((const struct crypto_aes_ctx *)(key), (out), (in)), \
160
+
struct aes_key *: aes_decrypt_new((const struct aes_key *)(key), (out), (in)), \
161
+
const struct aes_key *: aes_decrypt_new((const struct aes_key *)(key), (out), (in)))
162
+
void aes_decrypt_old(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);
163
+
void aes_decrypt_new(const struct aes_key *key, u8 out[at_least AES_BLOCK_SIZE],
164
+
const u8 in[at_least AES_BLOCK_SIZE]);
140
165
141
166
extern const u8 crypto_aes_sbox[];
142
167
extern const u8 crypto_aes_inv_sbox[];
168
+
extern const u32 aes_enc_tab[256];
169
+
extern const u32 aes_dec_tab[256];
143
170
144
171
void aescfb_encrypt(const struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src,
145
172
int len, const u8 iv[AES_BLOCK_SIZE]);
+4
lib/crypto/Kconfig
+4
lib/crypto/Kconfig
+9
-2
lib/crypto/Makefile
+9
-2
lib/crypto/Makefile
···
15
15
obj-$(CONFIG_CRYPTO_LIB_UTILS) += libcryptoutils.o
16
16
libcryptoutils-y := memneq.o utils.o
17
17
18
-
obj-$(CONFIG_CRYPTO_LIB_AES) += libaes.o
19
-
libaes-y := aes.o
18
+
################################################################################
19
+
20
+
obj-$(CONFIG_CRYPTO_LIB_AES) += libaes.o
21
+
libaes-y := aes.o
22
+
ifeq ($(CONFIG_CRYPTO_LIB_AES_ARCH),y)
23
+
CFLAGS_aes.o += -I$(src)/$(SRCARCH)
24
+
endif # CONFIG_CRYPTO_LIB_AES_ARCH
25
+
26
+
################################################################################
20
27
21
28
obj-$(CONFIG_CRYPTO_LIB_AESCFB) += libaescfb.o
22
29
libaescfb-y := aescfb.o
+341
-56
lib/crypto/aes.c
+341
-56
lib/crypto/aes.c
···
1
1
// SPDX-License-Identifier: GPL-2.0
2
2
/*
3
3
* Copyright (C) 2017-2019 Linaro Ltd <ard.biesheuvel@linaro.org>
4
+
* Copyright 2026 Google LLC
4
5
*/
5
6
6
7
#include <crypto/aes.h>
8
+
#include <linux/cache.h>
7
9
#include <linux/crypto.h>
8
10
#include <linux/export.h>
9
11
#include <linux/module.h>
···
91
89
EXPORT_SYMBOL(crypto_aes_sbox);
92
90
EXPORT_SYMBOL(crypto_aes_inv_sbox);
93
91
92
+
/* aes_enc_tab[i] contains MixColumn([SubByte(i), 0, 0, 0]). */
93
+
const u32 ____cacheline_aligned aes_enc_tab[256] = {
94
+
0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6,
95
+
0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56,
96
+
0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f,
97
+
0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb,
98
+
0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453,
99
+
0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c,
100
+
0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551,
101
+
0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a,
102
+
0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637,
103
+
0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df,
104
+
0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d,
105
+
0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b,
106
+
0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd,
107
+
0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1,
108
+
0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d,
109
+
0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85,
110
+
0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a,
111
+
0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe,
112
+
0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d,
113
+
0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1,
114
+
0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5,
115
+
0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3,
116
+
0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755,
117
+
0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6,
118
+
0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54,
119
+
0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428,
120
+
0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264,
121
+
0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8,
122
+
0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531,
123
+
0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda,
124
+
0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac,
125
+
0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810,
126
+
0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657,
127
+
0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e,
128
+
0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c,
129
+
0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c,
130
+
0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199,
131
+
0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122,
132
+
0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c,
133
+
0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5,
134
+
0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7,
135
+
0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e,
136
+
0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c,
137
+
};
138
+
EXPORT_SYMBOL(aes_enc_tab);
139
+
140
+
/* aes_dec_tab[i] contains InvMixColumn([InvSubByte(i), 0, 0, 0]). */
141
+
const u32 ____cacheline_aligned aes_dec_tab[256] = {
142
+
0x50a7f451, 0x5365417e, 0xc3a4171a, 0x965e273a, 0xcb6bab3b, 0xf1459d1f,
143
+
0xab58faac, 0x9303e34b, 0x55fa3020, 0xf66d76ad, 0x9176cc88, 0x254c02f5,
144
+
0xfcd7e54f, 0xd7cb2ac5, 0x80443526, 0x8fa362b5, 0x495ab1de, 0x671bba25,
145
+
0x980eea45, 0xe1c0fe5d, 0x02752fc3, 0x12f04c81, 0xa397468d, 0xc6f9d36b,
146
+
0xe75f8f03, 0x959c9215, 0xeb7a6dbf, 0xda595295, 0x2d83bed4, 0xd3217458,
147
+
0x2969e049, 0x44c8c98e, 0x6a89c275, 0x78798ef4, 0x6b3e5899, 0xdd71b927,
148
+
0xb64fe1be, 0x17ad88f0, 0x66ac20c9, 0xb43ace7d, 0x184adf63, 0x82311ae5,
149
+
0x60335197, 0x457f5362, 0xe07764b1, 0x84ae6bbb, 0x1ca081fe, 0x942b08f9,
150
+
0x58684870, 0x19fd458f, 0x876cde94, 0xb7f87b52, 0x23d373ab, 0xe2024b72,
151
+
0x578f1fe3, 0x2aab5566, 0x0728ebb2, 0x03c2b52f, 0x9a7bc586, 0xa50837d3,
152
+
0xf2872830, 0xb2a5bf23, 0xba6a0302, 0x5c8216ed, 0x2b1ccf8a, 0x92b479a7,
153
+
0xf0f207f3, 0xa1e2694e, 0xcdf4da65, 0xd5be0506, 0x1f6234d1, 0x8afea6c4,
154
+
0x9d532e34, 0xa055f3a2, 0x32e18a05, 0x75ebf6a4, 0x39ec830b, 0xaaef6040,
155
+
0x069f715e, 0x51106ebd, 0xf98a213e, 0x3d06dd96, 0xae053edd, 0x46bde64d,
156
+
0xb58d5491, 0x055dc471, 0x6fd40604, 0xff155060, 0x24fb9819, 0x97e9bdd6,
157
+
0xcc434089, 0x779ed967, 0xbd42e8b0, 0x888b8907, 0x385b19e7, 0xdbeec879,
158
+
0x470a7ca1, 0xe90f427c, 0xc91e84f8, 0x00000000, 0x83868009, 0x48ed2b32,
159
+
0xac70111e, 0x4e725a6c, 0xfbff0efd, 0x5638850f, 0x1ed5ae3d, 0x27392d36,
160
+
0x64d90f0a, 0x21a65c68, 0xd1545b9b, 0x3a2e3624, 0xb1670a0c, 0x0fe75793,
161
+
0xd296eeb4, 0x9e919b1b, 0x4fc5c080, 0xa220dc61, 0x694b775a, 0x161a121c,
162
+
0x0aba93e2, 0xe52aa0c0, 0x43e0223c, 0x1d171b12, 0x0b0d090e, 0xadc78bf2,
163
+
0xb9a8b62d, 0xc8a91e14, 0x8519f157, 0x4c0775af, 0xbbdd99ee, 0xfd607fa3,
164
+
0x9f2601f7, 0xbcf5725c, 0xc53b6644, 0x347efb5b, 0x7629438b, 0xdcc623cb,
165
+
0x68fcedb6, 0x63f1e4b8, 0xcadc31d7, 0x10856342, 0x40229713, 0x2011c684,
166
+
0x7d244a85, 0xf83dbbd2, 0x1132f9ae, 0x6da129c7, 0x4b2f9e1d, 0xf330b2dc,
167
+
0xec52860d, 0xd0e3c177, 0x6c16b32b, 0x99b970a9, 0xfa489411, 0x2264e947,
168
+
0xc48cfca8, 0x1a3ff0a0, 0xd82c7d56, 0xef903322, 0xc74e4987, 0xc1d138d9,
169
+
0xfea2ca8c, 0x360bd498, 0xcf81f5a6, 0x28de7aa5, 0x268eb7da, 0xa4bfad3f,
170
+
0xe49d3a2c, 0x0d927850, 0x9bcc5f6a, 0x62467e54, 0xc2138df6, 0xe8b8d890,
171
+
0x5ef7392e, 0xf5afc382, 0xbe805d9f, 0x7c93d069, 0xa92dd56f, 0xb31225cf,
172
+
0x3b99acc8, 0xa77d1810, 0x6e639ce8, 0x7bbb3bdb, 0x097826cd, 0xf418596e,
173
+
0x01b79aec, 0xa89a4f83, 0x656e95e6, 0x7ee6ffaa, 0x08cfbc21, 0xe6e815ef,
174
+
0xd99be7ba, 0xce366f4a, 0xd4099fea, 0xd67cb029, 0xafb2a431, 0x31233f2a,
175
+
0x3094a5c6, 0xc066a235, 0x37bc4e74, 0xa6ca82fc, 0xb0d090e0, 0x15d8a733,
176
+
0x4a9804f1, 0xf7daec41, 0x0e50cd7f, 0x2ff69117, 0x8dd64d76, 0x4db0ef43,
177
+
0x544daacc, 0xdf0496e4, 0xe3b5d19e, 0x1b886a4c, 0xb81f2cc1, 0x7f516546,
178
+
0x04ea5e9d, 0x5d358c01, 0x737487fa, 0x2e410bfb, 0x5a1d67b3, 0x52d2db92,
179
+
0x335610e9, 0x1347d66d, 0x8c61d79a, 0x7a0ca137, 0x8e14f859, 0x893c13eb,
180
+
0xee27a9ce, 0x35c961b7, 0xede51ce1, 0x3cb1477a, 0x59dfd29c, 0x3f73f255,
181
+
0x79ce1418, 0xbf37c773, 0xeacdf753, 0x5baafd5f, 0x146f3ddf, 0x86db4478,
182
+
0x81f3afca, 0x3ec468b9, 0x2c342438, 0x5f40a3c2, 0x72c31d16, 0x0c25e2bc,
183
+
0x8b493c28, 0x41950dff, 0x7101a839, 0xdeb30c08, 0x9ce4b4d8, 0x90c15664,
184
+
0x6184cb7b, 0x70b632d5, 0x745c6c48, 0x4257b8d0,
185
+
};
186
+
EXPORT_SYMBOL(aes_dec_tab);
187
+
188
+
/* Prefetch data into L1 cache. @mem should be cacheline-aligned. */
189
+
static __always_inline void aes_prefetch(const void *mem, size_t len)
190
+
{
191
+
for (size_t i = 0; i < len; i += L1_CACHE_BYTES)
192
+
*(volatile const u8 *)(mem + i);
193
+
barrier();
194
+
}
195
+
94
196
static u32 mul_by_x(u32 w)
95
197
{
96
198
u32 x = w & 0x7f7f7f7f;
···
275
169
(aes_sbox[(in >> 24) & 0xff] << 24);
276
170
}
277
171
278
-
/**
279
-
* aes_expandkey - Expands the AES key as described in FIPS-197
280
-
* @ctx: The location where the computed key will be stored.
281
-
* @in_key: The supplied key.
282
-
* @key_len: The length of the supplied key.
283
-
*
284
-
* Returns 0 on success. The function fails only if an invalid key size (or
285
-
* pointer) is supplied.
286
-
* The expanded key size is 240 bytes (max of 14 rounds with a unique 16 bytes
287
-
* key schedule plus a 16 bytes key which is used before the first round).
288
-
* The decryption key is prepared for the "Equivalent Inverse Cipher" as
289
-
* described in FIPS-197. The first slot (16 bytes) of each key (enc or dec) is
290
-
* for the initial combination, the second slot for the first round and so on.
291
-
*/
292
-
int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
293
-
unsigned int key_len)
172
+
static void aes_expandkey_generic(u32 rndkeys[], u32 *inv_rndkeys,
173
+
const u8 *in_key, int key_len)
294
174
{
295
175
u32 kwords = key_len / sizeof(u32);
296
176
u32 rc, i, j;
297
-
int err;
298
-
299
-
err = aes_check_keylen(key_len);
300
-
if (err)
301
-
return err;
302
-
303
-
ctx->key_length = key_len;
304
177
305
178
for (i = 0; i < kwords; i++)
306
-
ctx->key_enc[i] = get_unaligned_le32(in_key + i * sizeof(u32));
179
+
rndkeys[i] = get_unaligned_le32(&in_key[i * sizeof(u32)]);
307
180
308
181
for (i = 0, rc = 1; i < 10; i++, rc = mul_by_x(rc)) {
309
-
u32 *rki = ctx->key_enc + (i * kwords);
182
+
u32 *rki = &rndkeys[i * kwords];
310
183
u32 *rko = rki + kwords;
311
184
312
185
rko[0] = ror32(subw(rki[kwords - 1]), 8) ^ rc ^ rki[0];
···
314
229
* the Inverse Mix Columns transformation to all but the first and
315
230
* the last one.
316
231
*/
317
-
ctx->key_dec[0] = ctx->key_enc[key_len + 24];
318
-
ctx->key_dec[1] = ctx->key_enc[key_len + 25];
319
-
ctx->key_dec[2] = ctx->key_enc[key_len + 26];
320
-
ctx->key_dec[3] = ctx->key_enc[key_len + 27];
232
+
if (inv_rndkeys) {
233
+
inv_rndkeys[0] = rndkeys[key_len + 24];
234
+
inv_rndkeys[1] = rndkeys[key_len + 25];
235
+
inv_rndkeys[2] = rndkeys[key_len + 26];
236
+
inv_rndkeys[3] = rndkeys[key_len + 27];
321
237
322
-
for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) {
323
-
ctx->key_dec[i] = inv_mix_columns(ctx->key_enc[j]);
324
-
ctx->key_dec[i + 1] = inv_mix_columns(ctx->key_enc[j + 1]);
325
-
ctx->key_dec[i + 2] = inv_mix_columns(ctx->key_enc[j + 2]);
326
-
ctx->key_dec[i + 3] = inv_mix_columns(ctx->key_enc[j + 3]);
238
+
for (i = 4, j = key_len + 20; j > 0; i += 4, j -= 4) {
239
+
inv_rndkeys[i] = inv_mix_columns(rndkeys[j]);
240
+
inv_rndkeys[i + 1] = inv_mix_columns(rndkeys[j + 1]);
241
+
inv_rndkeys[i + 2] = inv_mix_columns(rndkeys[j + 2]);
242
+
inv_rndkeys[i + 3] = inv_mix_columns(rndkeys[j + 3]);
243
+
}
244
+
245
+
inv_rndkeys[i] = rndkeys[0];
246
+
inv_rndkeys[i + 1] = rndkeys[1];
247
+
inv_rndkeys[i + 2] = rndkeys[2];
248
+
inv_rndkeys[i + 3] = rndkeys[3];
327
249
}
250
+
}
328
251
329
-
ctx->key_dec[i] = ctx->key_enc[0];
330
-
ctx->key_dec[i + 1] = ctx->key_enc[1];
331
-
ctx->key_dec[i + 2] = ctx->key_enc[2];
332
-
ctx->key_dec[i + 3] = ctx->key_enc[3];
333
-
252
+
int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
253
+
unsigned int key_len)
254
+
{
255
+
if (aes_check_keylen(key_len) != 0)
256
+
return -EINVAL;
257
+
ctx->key_length = key_len;
258
+
aes_expandkey_generic(ctx->key_enc, ctx->key_dec, in_key, key_len);
334
259
return 0;
335
260
}
336
261
EXPORT_SYMBOL(aes_expandkey);
337
262
338
-
/**
339
-
* aes_encrypt - Encrypt a single AES block
340
-
* @ctx: Context struct containing the key schedule
341
-
* @out: Buffer to store the ciphertext
342
-
* @in: Buffer containing the plaintext
343
-
*/
344
-
void aes_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)
263
+
void aes_encrypt_old(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)
345
264
{
346
265
const u32 *rkp = ctx->key_enc + 4;
347
266
int rounds = 6 + ctx->key_length / 4;
···
388
299
put_unaligned_le32(subshift(st1, 2) ^ rkp[6], out + 8);
389
300
put_unaligned_le32(subshift(st1, 3) ^ rkp[7], out + 12);
390
301
}
391
-
EXPORT_SYMBOL(aes_encrypt);
302
+
EXPORT_SYMBOL(aes_encrypt_old);
392
303
393
-
/**
394
-
* aes_decrypt - Decrypt a single AES block
395
-
* @ctx: Context struct containing the key schedule
396
-
* @out: Buffer to store the plaintext
397
-
* @in: Buffer containing the ciphertext
398
-
*/
399
-
void aes_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)
304
+
static __always_inline u32 enc_quarterround(const u32 w[4], int i, u32 rk)
305
+
{
306
+
return rk ^ aes_enc_tab[(u8)w[i]] ^
307
+
rol32(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)], 8) ^
308
+
rol32(aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^
309
+
rol32(aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)], 24);
310
+
}
311
+
312
+
static __always_inline u32 enclast_quarterround(const u32 w[4], int i, u32 rk)
313
+
{
314
+
return rk ^ ((aes_enc_tab[(u8)w[i]] & 0x0000ff00) >> 8) ^
315
+
(aes_enc_tab[(u8)(w[(i + 1) % 4] >> 8)] & 0x0000ff00) ^
316
+
((aes_enc_tab[(u8)(w[(i + 2) % 4] >> 16)] & 0x0000ff00) << 8) ^
317
+
((aes_enc_tab[(u8)(w[(i + 3) % 4] >> 24)] & 0x0000ff00) << 16);
318
+
}
319
+
320
+
static void __maybe_unused aes_encrypt_generic(const u32 rndkeys[], int nrounds,
321
+
u8 out[AES_BLOCK_SIZE],
322
+
const u8 in[AES_BLOCK_SIZE])
323
+
{
324
+
const u32 *rkp = rndkeys;
325
+
int n = nrounds - 1;
326
+
u32 w[4];
327
+
328
+
w[0] = get_unaligned_le32(&in[0]) ^ *rkp++;
329
+
w[1] = get_unaligned_le32(&in[4]) ^ *rkp++;
330
+
w[2] = get_unaligned_le32(&in[8]) ^ *rkp++;
331
+
w[3] = get_unaligned_le32(&in[12]) ^ *rkp++;
332
+
333
+
/*
334
+
* Prefetch the table before doing data and key-dependent loads from it.
335
+
*
336
+
* This is intended only as a basic constant-time hardening measure that
337
+
* avoids interfering with performance too much. Its effectiveness is
338
+
* not guaranteed. For proper constant-time AES, a CPU that supports
339
+
* AES instructions should be used instead.
340
+
*/
341
+
aes_prefetch(aes_enc_tab, sizeof(aes_enc_tab));
342
+
343
+
do {
344
+
u32 w0 = enc_quarterround(w, 0, *rkp++);
345
+
u32 w1 = enc_quarterround(w, 1, *rkp++);
346
+
u32 w2 = enc_quarterround(w, 2, *rkp++);
347
+
u32 w3 = enc_quarterround(w, 3, *rkp++);
348
+
349
+
w[0] = w0;
350
+
w[1] = w1;
351
+
w[2] = w2;
352
+
w[3] = w3;
353
+
} while (--n);
354
+
355
+
put_unaligned_le32(enclast_quarterround(w, 0, *rkp++), &out[0]);
356
+
put_unaligned_le32(enclast_quarterround(w, 1, *rkp++), &out[4]);
357
+
put_unaligned_le32(enclast_quarterround(w, 2, *rkp++), &out[8]);
358
+
put_unaligned_le32(enclast_quarterround(w, 3, *rkp++), &out[12]);
359
+
}
360
+
361
+
static __always_inline u32 dec_quarterround(const u32 w[4], int i, u32 rk)
362
+
{
363
+
return rk ^ aes_dec_tab[(u8)w[i]] ^
364
+
rol32(aes_dec_tab[(u8)(w[(i + 3) % 4] >> 8)], 8) ^
365
+
rol32(aes_dec_tab[(u8)(w[(i + 2) % 4] >> 16)], 16) ^
366
+
rol32(aes_dec_tab[(u8)(w[(i + 1) % 4] >> 24)], 24);
367
+
}
368
+
369
+
static __always_inline u32 declast_quarterround(const u32 w[4], int i, u32 rk)
370
+
{
371
+
return rk ^ aes_inv_sbox[(u8)w[i]] ^
372
+
((u32)aes_inv_sbox[(u8)(w[(i + 3) % 4] >> 8)] << 8) ^
373
+
((u32)aes_inv_sbox[(u8)(w[(i + 2) % 4] >> 16)] << 16) ^
374
+
((u32)aes_inv_sbox[(u8)(w[(i + 1) % 4] >> 24)] << 24);
375
+
}
376
+
377
+
static void __maybe_unused aes_decrypt_generic(const u32 inv_rndkeys[],
378
+
int nrounds,
379
+
u8 out[AES_BLOCK_SIZE],
380
+
const u8 in[AES_BLOCK_SIZE])
381
+
{
382
+
const u32 *rkp = inv_rndkeys;
383
+
int n = nrounds - 1;
384
+
u32 w[4];
385
+
386
+
w[0] = get_unaligned_le32(&in[0]) ^ *rkp++;
387
+
w[1] = get_unaligned_le32(&in[4]) ^ *rkp++;
388
+
w[2] = get_unaligned_le32(&in[8]) ^ *rkp++;
389
+
w[3] = get_unaligned_le32(&in[12]) ^ *rkp++;
390
+
391
+
aes_prefetch(aes_dec_tab, sizeof(aes_dec_tab));
392
+
393
+
do {
394
+
u32 w0 = dec_quarterround(w, 0, *rkp++);
395
+
u32 w1 = dec_quarterround(w, 1, *rkp++);
396
+
u32 w2 = dec_quarterround(w, 2, *rkp++);
397
+
u32 w3 = dec_quarterround(w, 3, *rkp++);
398
+
399
+
w[0] = w0;
400
+
w[1] = w1;
401
+
w[2] = w2;
402
+
w[3] = w3;
403
+
} while (--n);
404
+
405
+
aes_prefetch((const void *)aes_inv_sbox, sizeof(aes_inv_sbox));
406
+
put_unaligned_le32(declast_quarterround(w, 0, *rkp++), &out[0]);
407
+
put_unaligned_le32(declast_quarterround(w, 1, *rkp++), &out[4]);
408
+
put_unaligned_le32(declast_quarterround(w, 2, *rkp++), &out[8]);
409
+
put_unaligned_le32(declast_quarterround(w, 3, *rkp++), &out[12]);
410
+
}
411
+
412
+
void aes_decrypt_old(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in)
400
413
{
401
414
const u32 *rkp = ctx->key_dec + 4;
402
415
int rounds = 6 + ctx->key_length / 4;
···
541
350
put_unaligned_le32(inv_subshift(st1, 2) ^ rkp[6], out + 8);
542
351
put_unaligned_le32(inv_subshift(st1, 3) ^ rkp[7], out + 12);
543
352
}
544
-
EXPORT_SYMBOL(aes_decrypt);
353
+
EXPORT_SYMBOL(aes_decrypt_old);
545
354
546
-
MODULE_DESCRIPTION("Generic AES library");
355
+
/*
356
+
* Note: the aes_prepare*key_* names reflect the fact that the implementation
357
+
* might not actually expand the key. (The s390 code for example doesn't.)
358
+
* Where the key is expanded we use the more specific names aes_expandkey_*.
359
+
*
360
+
* aes_preparekey_arch() is passed an optional pointer 'inv_k' which points to
361
+
* the area to store the prepared decryption key. It will be NULL if the user
362
+
* is requesting encryption-only. aes_preparekey_arch() is also passed a valid
363
+
* 'key_len' and 'nrounds', corresponding to AES-128, AES-192, or AES-256.
364
+
*/
365
+
#ifdef CONFIG_CRYPTO_LIB_AES_ARCH
366
+
/* An arch-specific implementation of AES is available. Include it. */
367
+
#include "aes.h" /* $(SRCARCH)/aes.h */
368
+
#else
369
+
/* No arch-specific implementation of AES is available. Use generic code. */
370
+
371
+
static void aes_preparekey_arch(union aes_enckey_arch *k,
372
+
union aes_invkey_arch *inv_k,
373
+
const u8 *in_key, int key_len, int nrounds)
374
+
{
375
+
aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL,
376
+
in_key, key_len);
377
+
}
378
+
379
+
static void aes_encrypt_arch(const struct aes_enckey *key,
380
+
u8 out[AES_BLOCK_SIZE],
381
+
const u8 in[AES_BLOCK_SIZE])
382
+
{
383
+
aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
384
+
}
385
+
386
+
static void aes_decrypt_arch(const struct aes_key *key,
387
+
u8 out[AES_BLOCK_SIZE],
388
+
const u8 in[AES_BLOCK_SIZE])
389
+
{
390
+
aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in);
391
+
}
392
+
#endif
393
+
394
+
static int __aes_preparekey(struct aes_enckey *enc_key,
395
+
union aes_invkey_arch *inv_k,
396
+
const u8 *in_key, size_t key_len)
397
+
{
398
+
if (aes_check_keylen(key_len) != 0)
399
+
return -EINVAL;
400
+
enc_key->len = key_len;
401
+
enc_key->nrounds = 6 + key_len / 4;
402
+
aes_preparekey_arch(&enc_key->k, inv_k, in_key, key_len,
403
+
enc_key->nrounds);
404
+
return 0;
405
+
}
406
+
407
+
int aes_preparekey(struct aes_key *key, const u8 *in_key, size_t key_len)
408
+
{
409
+
return __aes_preparekey((struct aes_enckey *)key, &key->inv_k,
410
+
in_key, key_len);
411
+
}
412
+
EXPORT_SYMBOL(aes_preparekey);
413
+
414
+
int aes_prepareenckey(struct aes_enckey *key, const u8 *in_key, size_t key_len)
415
+
{
416
+
return __aes_preparekey(key, NULL, in_key, key_len);
417
+
}
418
+
EXPORT_SYMBOL(aes_prepareenckey);
419
+
420
+
void aes_encrypt_new(aes_encrypt_arg key, u8 out[AES_BLOCK_SIZE],
421
+
const u8 in[AES_BLOCK_SIZE])
422
+
{
423
+
aes_encrypt_arch(key.enc_key, out, in);
424
+
}
425
+
EXPORT_SYMBOL(aes_encrypt_new);
426
+
427
+
void aes_decrypt_new(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],
428
+
const u8 in[AES_BLOCK_SIZE])
429
+
{
430
+
aes_decrypt_arch(key, out, in);
431
+
}
432
+
EXPORT_SYMBOL(aes_decrypt_new);
433
+
434
+
#ifdef aes_mod_init_arch
435
+
static int __init aes_mod_init(void)
436
+
{
437
+
aes_mod_init_arch();
438
+
return 0;
439
+
}
440
+
subsys_initcall(aes_mod_init);
441
+
442
+
static void __exit aes_mod_exit(void)
443
+
{
444
+
}
445
+
module_exit(aes_mod_exit);
446
+
#endif
447
+
448
+
MODULE_DESCRIPTION("AES block cipher");
547
449
MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
450
+
MODULE_AUTHOR("Eric Biggers <ebiggers@kernel.org>");
548
451
MODULE_LICENSE("GPL v2");