Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

crypto: arm/aes-neonbs - go back to using aes-arm directly

In aes-neonbs, instead of going through the crypto API for the parts
that the bit-sliced AES code doesn't handle, namely AES-CBC encryption
and single-block AES, just call the ARM scalar AES cipher directly.

This basically goes back to the original approach that was used before
commit b56f5cbc7e08 ("crypto: arm/aes-neonbs - resolve fallback cipher
at runtime"). Calling the ARM scalar AES cipher directly is faster,
simpler, and avoids any chance of bugs specific to the use of fallback
ciphers such as module loading deadlocks which have happened twice. The
deadlocks turned out to be fixable in other ways, but there's no need to
rely on anything so fragile in the first place.

The rationale for the above-mentioned commit was to allow people to
choose to use a time-invariant AES implementation for the fallback
cipher. There are a couple problems with that rationale, though:

- In practice the ARM scalar AES cipher (aes-arm) was used anyway, since
it has a higher priority than aes-fixed-time. Users *could* go out of
their way to disable or blacklist aes-arm, or to lower its priority
using NETLINK_CRYPTO, but very few users customize the crypto API to
this extent. Systems with the ARMv8 Crypto Extensions used aes-ce,
but the bit-sliced algorithms are irrelevant on such systems anyway.

- Since commit 913a3aa07d16 ("crypto: arm/aes - add some hardening
against cache-timing attacks"), the ARM scalar AES cipher is partially
hardened against cache-timing attacks. It actually works like
aes-fixed-time, in that it disables interrupts and prefetches its
lookup table. It does use a larger table than aes-fixed-time, but
even so, it is not clear that aes-fixed-time is meaningfully more
time-invariant than aes-arm. And of course, the real solution for
time-invariant AES is to use a CPU that supports AES instructions.

Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

authored by

Eric Biggers and committed by
Herbert Xu
f235bc11 9a22b281

+66 -95
+10 -4
arch/arm/crypto/Kconfig
··· 166 166 config CRYPTO_AES_ARM_BS 167 167 tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (bit-sliced NEON)" 168 168 depends on KERNEL_MODE_NEON 169 + select CRYPTO_AES_ARM 169 170 select CRYPTO_SKCIPHER 170 171 select CRYPTO_LIB_AES 171 - select CRYPTO_AES 172 - select CRYPTO_CBC 173 172 select CRYPTO_SIMD 174 173 help 175 174 Length-preserving ciphers: AES cipher algorithms (FIPS-197) ··· 182 183 Bit sliced AES gives around 45% speedup on Cortex-A15 for CTR mode 183 184 and for XTS mode encryption, CBC and XTS mode decryption speedup is 184 185 around 25%. (CBC encryption speed is not affected by this driver.) 185 - This implementation does not rely on any lookup tables so it is 186 - believed to be invulnerable to cache timing attacks. 186 + 187 + The bit sliced AES code does not use lookup tables, so it is believed 188 + to be invulnerable to cache timing attacks. However, since the bit 189 + sliced AES code cannot process single blocks efficiently, in certain 190 + cases table-based code with some countermeasures against cache timing 191 + attacks will still be used as a fallback method; specifically CBC 192 + encryption (not CBC decryption), the encryption of XTS tweaks, XTS 193 + ciphertext stealing when the message isn't a multiple of 16 bytes, and 194 + CTR when invoked in a context in which NEON instructions are unusable. 187 195 188 196 config CRYPTO_AES_ARM_CE 189 197 tristate "Ciphers: AES, modes: ECB/CBC/CTS/CTR/XTS (ARMv8 Crypto Extensions)"
+3 -2
arch/arm/crypto/aes-cipher-glue.c
··· 9 9 #include <crypto/aes.h> 10 10 #include <crypto/algapi.h> 11 11 #include <linux/module.h> 12 + #include "aes-cipher.h" 12 13 13 - asmlinkage void __aes_arm_encrypt(u32 *rk, int rounds, const u8 *in, u8 *out); 14 - asmlinkage void __aes_arm_decrypt(u32 *rk, int rounds, const u8 *in, u8 *out); 14 + EXPORT_SYMBOL_GPL(__aes_arm_encrypt); 15 + EXPORT_SYMBOL_GPL(__aes_arm_decrypt); 15 16 16 17 static void aes_arm_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 17 18 {
+13
arch/arm/crypto/aes-cipher.h
··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + #ifndef ARM_CRYPTO_AES_CIPHER_H 3 + #define ARM_CRYPTO_AES_CIPHER_H 4 + 5 + #include <linux/linkage.h> 6 + #include <linux/types.h> 7 + 8 + asmlinkage void __aes_arm_encrypt(const u32 rk[], int rounds, 9 + const u8 *in, u8 *out); 10 + asmlinkage void __aes_arm_decrypt(const u32 rk[], int rounds, 11 + const u8 *in, u8 *out); 12 + 13 + #endif /* ARM_CRYPTO_AES_CIPHER_H */
+40 -89
arch/arm/crypto/aes-neonbs-glue.c
··· 9 9 #include <asm/simd.h> 10 10 #include <crypto/aes.h> 11 11 #include <crypto/ctr.h> 12 - #include <crypto/internal/cipher.h> 13 12 #include <crypto/internal/simd.h> 14 13 #include <crypto/internal/skcipher.h> 15 14 #include <crypto/scatterwalk.h> 16 15 #include <crypto/xts.h> 17 16 #include <linux/module.h> 17 + #include "aes-cipher.h" 18 18 19 19 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 20 20 MODULE_DESCRIPTION("Bit sliced AES using NEON instructions"); 21 21 MODULE_LICENSE("GPL v2"); 22 22 23 23 MODULE_ALIAS_CRYPTO("ecb(aes)"); 24 - MODULE_ALIAS_CRYPTO("cbc(aes)-all"); 24 + MODULE_ALIAS_CRYPTO("cbc(aes)"); 25 25 MODULE_ALIAS_CRYPTO("ctr(aes)"); 26 26 MODULE_ALIAS_CRYPTO("xts(aes)"); 27 - 28 - MODULE_IMPORT_NS(CRYPTO_INTERNAL); 29 27 30 28 asmlinkage void aesbs_convert_key(u8 out[], u32 const rk[], int rounds); 31 29 ··· 50 52 51 53 struct aesbs_cbc_ctx { 52 54 struct aesbs_ctx key; 53 - struct crypto_skcipher *enc_tfm; 55 + struct crypto_aes_ctx fallback; 54 56 }; 55 57 56 58 struct aesbs_xts_ctx { 57 59 struct aesbs_ctx key; 58 - struct crypto_cipher *cts_tfm; 59 - struct crypto_cipher *tweak_tfm; 60 + struct crypto_aes_ctx fallback; 61 + struct crypto_aes_ctx tweak_key; 60 62 }; 61 63 62 64 struct aesbs_ctr_ctx { ··· 127 129 unsigned int key_len) 128 130 { 129 131 struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 130 - struct crypto_aes_ctx rk; 131 132 int err; 132 133 133 - err = aes_expandkey(&rk, in_key, key_len); 134 + err = aes_expandkey(&ctx->fallback, in_key, key_len); 134 135 if (err) 135 136 return err; 136 137 137 138 ctx->key.rounds = 6 + key_len / 4; 138 139 139 140 kernel_neon_begin(); 140 - aesbs_convert_key(ctx->key.rk, rk.key_enc, ctx->key.rounds); 141 + aesbs_convert_key(ctx->key.rk, ctx->fallback.key_enc, ctx->key.rounds); 141 142 kernel_neon_end(); 142 - memzero_explicit(&rk, sizeof(rk)); 143 143 144 - return crypto_skcipher_setkey(ctx->enc_tfm, in_key, key_len); 144 + return 0; 145 145 } 146 146 147 147 static int cbc_encrypt(struct skcipher_request *req) 148 148 { 149 - struct skcipher_request *subreq = skcipher_request_ctx(req); 150 149 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 151 - struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 150 + const struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 151 + struct skcipher_walk walk; 152 + unsigned int nbytes; 153 + int err; 152 154 153 - skcipher_request_set_tfm(subreq, ctx->enc_tfm); 154 - skcipher_request_set_callback(subreq, 155 - skcipher_request_flags(req), 156 - NULL, NULL); 157 - skcipher_request_set_crypt(subreq, req->src, req->dst, 158 - req->cryptlen, req->iv); 155 + err = skcipher_walk_virt(&walk, req, false); 159 156 160 - return crypto_skcipher_encrypt(subreq); 157 + while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { 158 + const u8 *src = walk.src.virt.addr; 159 + u8 *dst = walk.dst.virt.addr; 160 + u8 *prev = walk.iv; 161 + 162 + do { 163 + crypto_xor_cpy(dst, src, prev, AES_BLOCK_SIZE); 164 + __aes_arm_encrypt(ctx->fallback.key_enc, 165 + ctx->key.rounds, dst, dst); 166 + prev = dst; 167 + src += AES_BLOCK_SIZE; 168 + dst += AES_BLOCK_SIZE; 169 + nbytes -= AES_BLOCK_SIZE; 170 + } while (nbytes >= AES_BLOCK_SIZE); 171 + memcpy(walk.iv, prev, AES_BLOCK_SIZE); 172 + err = skcipher_walk_done(&walk, nbytes); 173 + } 174 + return err; 161 175 } 162 176 163 177 static int cbc_decrypt(struct skcipher_request *req) ··· 198 188 } 199 189 200 190 return err; 201 - } 202 - 203 - static int cbc_init(struct crypto_skcipher *tfm) 204 - { 205 - struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 206 - unsigned int reqsize; 207 - 208 - ctx->enc_tfm = crypto_alloc_skcipher("cbc(aes)", 0, CRYPTO_ALG_ASYNC | 209 - CRYPTO_ALG_NEED_FALLBACK); 210 - if (IS_ERR(ctx->enc_tfm)) 211 - return PTR_ERR(ctx->enc_tfm); 212 - 213 - reqsize = sizeof(struct skcipher_request); 214 - reqsize += crypto_skcipher_reqsize(ctx->enc_tfm); 215 - crypto_skcipher_set_reqsize(tfm, reqsize); 216 - 217 - return 0; 218 - } 219 - 220 - static void cbc_exit(struct crypto_skcipher *tfm) 221 - { 222 - struct aesbs_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); 223 - 224 - crypto_free_skcipher(ctx->enc_tfm); 225 191 } 226 192 227 193 static int aesbs_ctr_setkey_sync(struct crypto_skcipher *tfm, const u8 *in_key, ··· 257 271 static void ctr_encrypt_one(struct crypto_skcipher *tfm, const u8 *src, u8 *dst) 258 272 { 259 273 struct aesbs_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); 260 - unsigned long flags; 261 274 262 - /* 263 - * Temporarily disable interrupts to avoid races where 264 - * cachelines are evicted when the CPU is interrupted 265 - * to do something else. 266 - */ 267 - local_irq_save(flags); 268 - aes_encrypt(&ctx->fallback, dst, src); 269 - local_irq_restore(flags); 275 + __aes_arm_encrypt(ctx->fallback.key_enc, ctx->key.rounds, src, dst); 270 276 } 271 277 272 278 static int ctr_encrypt_sync(struct skcipher_request *req) ··· 280 302 return err; 281 303 282 304 key_len /= 2; 283 - err = crypto_cipher_setkey(ctx->cts_tfm, in_key, key_len); 305 + err = aes_expandkey(&ctx->fallback, in_key, key_len); 284 306 if (err) 285 307 return err; 286 - err = crypto_cipher_setkey(ctx->tweak_tfm, in_key + key_len, key_len); 308 + err = aes_expandkey(&ctx->tweak_key, in_key + key_len, key_len); 287 309 if (err) 288 310 return err; 289 311 290 312 return aesbs_setkey(tfm, in_key, key_len); 291 - } 292 - 293 - static int xts_init(struct crypto_skcipher *tfm) 294 - { 295 - struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 296 - 297 - ctx->cts_tfm = crypto_alloc_cipher("aes", 0, 0); 298 - if (IS_ERR(ctx->cts_tfm)) 299 - return PTR_ERR(ctx->cts_tfm); 300 - 301 - ctx->tweak_tfm = crypto_alloc_cipher("aes", 0, 0); 302 - if (IS_ERR(ctx->tweak_tfm)) 303 - crypto_free_cipher(ctx->cts_tfm); 304 - 305 - return PTR_ERR_OR_ZERO(ctx->tweak_tfm); 306 - } 307 - 308 - static void xts_exit(struct crypto_skcipher *tfm) 309 - { 310 - struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 311 - 312 - crypto_free_cipher(ctx->tweak_tfm); 313 - crypto_free_cipher(ctx->cts_tfm); 314 313 } 315 314 316 315 static int __xts_crypt(struct skcipher_request *req, bool encrypt, ··· 296 341 { 297 342 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 298 343 struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm); 344 + const int rounds = ctx->key.rounds; 299 345 int tail = req->cryptlen % AES_BLOCK_SIZE; 300 346 struct skcipher_request subreq; 301 347 u8 buf[2 * AES_BLOCK_SIZE]; ··· 320 364 if (err) 321 365 return err; 322 366 323 - crypto_cipher_encrypt_one(ctx->tweak_tfm, walk.iv, walk.iv); 367 + __aes_arm_encrypt(ctx->tweak_key.key_enc, rounds, walk.iv, walk.iv); 324 368 325 369 while (walk.nbytes >= AES_BLOCK_SIZE) { 326 370 unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE; ··· 334 378 335 379 kernel_neon_begin(); 336 380 fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->key.rk, 337 - ctx->key.rounds, blocks, walk.iv, reorder_last_tweak); 381 + rounds, blocks, walk.iv, reorder_last_tweak); 338 382 kernel_neon_end(); 339 383 err = skcipher_walk_done(&walk, 340 384 walk.nbytes - blocks * AES_BLOCK_SIZE); ··· 352 396 crypto_xor(buf, req->iv, AES_BLOCK_SIZE); 353 397 354 398 if (encrypt) 355 - crypto_cipher_encrypt_one(ctx->cts_tfm, buf, buf); 399 + __aes_arm_encrypt(ctx->fallback.key_enc, rounds, buf, buf); 356 400 else 357 - crypto_cipher_decrypt_one(ctx->cts_tfm, buf, buf); 401 + __aes_arm_decrypt(ctx->fallback.key_dec, rounds, buf, buf); 358 402 359 403 crypto_xor(buf, req->iv, AES_BLOCK_SIZE); 360 404 ··· 395 439 .base.cra_blocksize = AES_BLOCK_SIZE, 396 440 .base.cra_ctxsize = sizeof(struct aesbs_cbc_ctx), 397 441 .base.cra_module = THIS_MODULE, 398 - .base.cra_flags = CRYPTO_ALG_INTERNAL | 399 - CRYPTO_ALG_NEED_FALLBACK, 442 + .base.cra_flags = CRYPTO_ALG_INTERNAL, 400 443 401 444 .min_keysize = AES_MIN_KEY_SIZE, 402 445 .max_keysize = AES_MAX_KEY_SIZE, ··· 404 449 .setkey = aesbs_cbc_setkey, 405 450 .encrypt = cbc_encrypt, 406 451 .decrypt = cbc_decrypt, 407 - .init = cbc_init, 408 - .exit = cbc_exit, 409 452 }, { 410 453 .base.cra_name = "__ctr(aes)", 411 454 .base.cra_driver_name = "__ctr-aes-neonbs", ··· 453 500 .setkey = aesbs_xts_setkey, 454 501 .encrypt = xts_encrypt, 455 502 .decrypt = xts_decrypt, 456 - .init = xts_init, 457 - .exit = xts_exit, 458 503 } }; 459 504 460 505 static struct simd_skcipher_alg *aes_simd_algs[ARRAY_SIZE(aes_algs)];