Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: powerpc/aes: Migrate SPE optimized code into library
Move the PowerPC SPE AES assembly code into lib/crypto/, wire the key
expansion and single-block en/decryption functions up to the AES library
API, and remove the superseded "aes-ppc-spe" crypto_cipher algorithm.
The result is that both the AES library and crypto_cipher APIs are now
optimized with SPE, whereas previously only crypto_cipher was (and
optimizations weren't enabled by default, which this commit fixes too).
Note that many of the functions in the PowerPC SPE assembly code are
still used by the AES mode implementations in arch/powerpc/crypto/. For
now, just export these functions. These exports will go away once the
AES modes are migrated to the library as well. (Trying to split up the
assembly files seemed like much more trouble than it would be worth.)
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260112192035.10427-13-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+122
-85
+1
-1
arch/powerpc/crypto/Kconfig
+1
-1
arch/powerpc/crypto/Kconfig
···
5
5
config CRYPTO_AES_PPC_SPE
6
6
tristate "Ciphers: AES, modes: ECB/CBC/CTR/XTS (SPE)"
7
7
depends on SPE
8
+
select CRYPTO_LIB_AES
8
9
select CRYPTO_SKCIPHER
9
10
help
10
-
Block ciphers: AES cipher algorithms (FIPS-197)
11
11
Length-preserving ciphers: AES with ECB, CBC, CTR, and XTS modes
12
12
13
13
Architecture: powerpc using:
+1
-1
arch/powerpc/crypto/Makefile
+1
-1
arch/powerpc/crypto/Makefile
···
9
9
obj-$(CONFIG_CRYPTO_AES_GCM_P10) += aes-gcm-p10-crypto.o
10
10
obj-$(CONFIG_CRYPTO_DEV_VMX_ENCRYPT) += vmx-crypto.o
11
11
12
-
aes-ppc-spe-y := aes-spe-core.o aes-spe-keys.o aes-tab-4k.o aes-spe-modes.o aes-spe-glue.o
12
+
aes-ppc-spe-y := aes-spe-glue.o
13
13
aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o
14
14
vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o
15
15
arch/powerpc/crypto/aes-spe-core.S
lib/crypto/powerpc/aes-spe-core.S
arch/powerpc/crypto/aes-spe-core.S
lib/crypto/powerpc/aes-spe-core.S
+5
-83
arch/powerpc/crypto/aes-spe-glue.c
+5
-83
arch/powerpc/crypto/aes-spe-glue.c
···
51
51
u32 rounds;
52
52
};
53
53
54
-
extern void ppc_encrypt_aes(u8 *out, const u8 *in, u32 *key_enc, u32 rounds);
55
-
extern void ppc_decrypt_aes(u8 *out, const u8 *in, u32 *key_dec, u32 rounds);
56
-
extern void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
57
-
u32 bytes);
58
-
extern void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
59
-
u32 bytes);
60
-
extern void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
61
-
u32 bytes, u8 *iv);
62
-
extern void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
63
-
u32 bytes, u8 *iv);
64
-
extern void ppc_crypt_ctr (u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
65
-
u32 bytes, u8 *iv);
66
-
extern void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
67
-
u32 bytes, u8 *iv, u32 *key_twk);
68
-
extern void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
69
-
u32 bytes, u8 *iv, u32 *key_twk);
70
-
71
-
extern void ppc_expand_key_128(u32 *key_enc, const u8 *key);
72
-
extern void ppc_expand_key_192(u32 *key_enc, const u8 *key);
73
-
extern void ppc_expand_key_256(u32 *key_enc, const u8 *key);
74
-
75
-
extern void ppc_generate_decrypt_key(u32 *key_dec,u32 *key_enc,
76
-
unsigned int key_len);
77
-
78
54
static void spe_begin(void)
79
55
{
80
56
/* disable preemption and save users SPE registers if required */
···
65
89
preempt_enable();
66
90
}
67
91
68
-
static int ppc_aes_setkey(struct crypto_tfm *tfm, const u8 *in_key,
69
-
unsigned int key_len)
92
+
static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,
93
+
const u8 *in_key, unsigned int key_len)
70
94
{
71
-
struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
95
+
struct ppc_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
72
96
73
97
switch (key_len) {
74
98
case AES_KEYSIZE_128:
···
90
114
ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len);
91
115
92
116
return 0;
93
-
}
94
-
95
-
static int ppc_aes_setkey_skcipher(struct crypto_skcipher *tfm,
96
-
const u8 *in_key, unsigned int key_len)
97
-
{
98
-
return ppc_aes_setkey(crypto_skcipher_tfm(tfm), in_key, key_len);
99
117
}
100
118
101
119
static int ppc_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
···
127
157
ppc_generate_decrypt_key(ctx->key_dec, ctx->key_enc, key_len);
128
158
129
159
return 0;
130
-
}
131
-
132
-
static void ppc_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
133
-
{
134
-
struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
135
-
136
-
spe_begin();
137
-
ppc_encrypt_aes(out, in, ctx->key_enc, ctx->rounds);
138
-
spe_end();
139
-
}
140
-
141
-
static void ppc_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
142
-
{
143
-
struct ppc_aes_ctx *ctx = crypto_tfm_ctx(tfm);
144
-
145
-
spe_begin();
146
-
ppc_decrypt_aes(out, in, ctx->key_dec, ctx->rounds);
147
-
spe_end();
148
160
}
149
161
150
162
static int ppc_ecb_crypt(struct skcipher_request *req, bool enc)
···
362
410
* with kmalloc() in the crypto infrastructure
363
411
*/
364
412
365
-
static struct crypto_alg aes_cipher_alg = {
366
-
.cra_name = "aes",
367
-
.cra_driver_name = "aes-ppc-spe",
368
-
.cra_priority = 300,
369
-
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
370
-
.cra_blocksize = AES_BLOCK_SIZE,
371
-
.cra_ctxsize = sizeof(struct ppc_aes_ctx),
372
-
.cra_alignmask = 0,
373
-
.cra_module = THIS_MODULE,
374
-
.cra_u = {
375
-
.cipher = {
376
-
.cia_min_keysize = AES_MIN_KEY_SIZE,
377
-
.cia_max_keysize = AES_MAX_KEY_SIZE,
378
-
.cia_setkey = ppc_aes_setkey,
379
-
.cia_encrypt = ppc_aes_encrypt,
380
-
.cia_decrypt = ppc_aes_decrypt
381
-
}
382
-
}
383
-
};
384
-
385
413
static struct skcipher_alg aes_skcipher_algs[] = {
386
414
{
387
415
.base.cra_name = "ecb(aes)",
···
420
488
421
489
static int __init ppc_aes_mod_init(void)
422
490
{
423
-
int err;
424
-
425
-
err = crypto_register_alg(&aes_cipher_alg);
426
-
if (err)
427
-
return err;
428
-
429
-
err = crypto_register_skciphers(aes_skcipher_algs,
430
-
ARRAY_SIZE(aes_skcipher_algs));
431
-
if (err)
432
-
crypto_unregister_alg(&aes_cipher_alg);
433
-
return err;
491
+
return crypto_register_skciphers(aes_skcipher_algs,
492
+
ARRAY_SIZE(aes_skcipher_algs));
434
493
}
435
494
436
495
static void __exit ppc_aes_mod_fini(void)
437
496
{
438
-
crypto_unregister_alg(&aes_cipher_alg);
439
497
crypto_unregister_skciphers(aes_skcipher_algs,
440
498
ARRAY_SIZE(aes_skcipher_algs));
441
499
}
arch/powerpc/crypto/aes-spe-keys.S
lib/crypto/powerpc/aes-spe-keys.S
arch/powerpc/crypto/aes-spe-keys.S
lib/crypto/powerpc/aes-spe-keys.S
arch/powerpc/crypto/aes-spe-modes.S
lib/crypto/powerpc/aes-spe-modes.S
arch/powerpc/crypto/aes-spe-modes.S
lib/crypto/powerpc/aes-spe-modes.S
arch/powerpc/crypto/aes-spe-regs.h
lib/crypto/powerpc/aes-spe-regs.h
arch/powerpc/crypto/aes-spe-regs.h
lib/crypto/powerpc/aes-spe-regs.h
arch/powerpc/crypto/aes-tab-4k.S
lib/crypto/powerpc/aes-tab-4k.S
arch/powerpc/crypto/aes-tab-4k.S
lib/crypto/powerpc/aes-tab-4k.S
+31
include/crypto/aes.h
+31
include/crypto/aes.h
···
20
20
21
21
union aes_enckey_arch {
22
22
u32 rndkeys[AES_MAX_KEYLENGTH_U32];
23
+
#ifdef CONFIG_CRYPTO_LIB_AES_ARCH
24
+
#if defined(CONFIG_PPC) && defined(CONFIG_SPE)
25
+
/* Used unconditionally (when SPE AES code is enabled in kconfig) */
26
+
u32 spe_enc_key[AES_MAX_KEYLENGTH_U32] __aligned(8);
27
+
#endif
28
+
#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */
23
29
};
24
30
25
31
union aes_invkey_arch {
26
32
u32 inv_rndkeys[AES_MAX_KEYLENGTH_U32];
33
+
#ifdef CONFIG_CRYPTO_LIB_AES_ARCH
34
+
#if defined(CONFIG_PPC) && defined(CONFIG_SPE)
35
+
/* Used unconditionally (when SPE AES code is enabled in kconfig) */
36
+
u32 spe_dec_key[AES_MAX_KEYLENGTH_U32] __aligned(8);
37
+
#endif
38
+
#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */
27
39
};
28
40
29
41
/**
···
136
124
#ifdef CONFIG_ARM64
137
125
int ce_aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key,
138
126
unsigned int key_len);
127
+
#elif defined(CONFIG_PPC)
128
+
void ppc_expand_key_128(u32 *key_enc, const u8 *key);
129
+
void ppc_expand_key_192(u32 *key_enc, const u8 *key);
130
+
void ppc_expand_key_256(u32 *key_enc, const u8 *key);
131
+
void ppc_generate_decrypt_key(u32 *key_dec, u32 *key_enc, unsigned int key_len);
132
+
void ppc_encrypt_ecb(u8 *out, const u8 *in, u32 *key_enc, u32 rounds,
133
+
u32 bytes);
134
+
void ppc_decrypt_ecb(u8 *out, const u8 *in, u32 *key_dec, u32 rounds,
135
+
u32 bytes);
136
+
void ppc_encrypt_cbc(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes,
137
+
u8 *iv);
138
+
void ppc_decrypt_cbc(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, u32 bytes,
139
+
u8 *iv);
140
+
void ppc_crypt_ctr(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes,
141
+
u8 *iv);
142
+
void ppc_encrypt_xts(u8 *out, const u8 *in, u32 *key_enc, u32 rounds, u32 bytes,
143
+
u8 *iv, u32 *key_twk);
144
+
void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, u32 bytes,
145
+
u8 *iv, u32 *key_twk);
139
146
#endif
140
147
141
148
/**
+1
lib/crypto/Kconfig
+1
lib/crypto/Kconfig
+9
lib/crypto/Makefile
+9
lib/crypto/Makefile
···
29
29
libaes-$(CONFIG_KERNEL_MODE_NEON) += arm64/aes-ce-core.o
30
30
endif
31
31
32
+
ifeq ($(CONFIG_PPC),y)
33
+
ifeq ($(CONFIG_SPE),y)
34
+
libaes-y += powerpc/aes-spe-core.o \
35
+
powerpc/aes-spe-keys.o \
36
+
powerpc/aes-spe-modes.o \
37
+
powerpc/aes-tab-4k.o
38
+
endif
39
+
endif # CONFIG_PPC
40
+
32
41
endif # CONFIG_CRYPTO_LIB_AES_ARCH
33
42
34
43
################################################################################
+74
lib/crypto/powerpc/aes.h
+74
lib/crypto/powerpc/aes.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-only */
2
+
/*
3
+
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
4
+
* Copyright 2026 Google LLC
5
+
*/
6
+
#include <asm/simd.h>
7
+
#include <asm/switch_to.h>
8
+
#include <linux/cpufeature.h>
9
+
#include <linux/jump_label.h>
10
+
#include <linux/preempt.h>
11
+
#include <linux/uaccess.h>
12
+
13
+
EXPORT_SYMBOL_GPL(ppc_expand_key_128);
14
+
EXPORT_SYMBOL_GPL(ppc_expand_key_192);
15
+
EXPORT_SYMBOL_GPL(ppc_expand_key_256);
16
+
EXPORT_SYMBOL_GPL(ppc_generate_decrypt_key);
17
+
EXPORT_SYMBOL_GPL(ppc_encrypt_ecb);
18
+
EXPORT_SYMBOL_GPL(ppc_decrypt_ecb);
19
+
EXPORT_SYMBOL_GPL(ppc_encrypt_cbc);
20
+
EXPORT_SYMBOL_GPL(ppc_decrypt_cbc);
21
+
EXPORT_SYMBOL_GPL(ppc_crypt_ctr);
22
+
EXPORT_SYMBOL_GPL(ppc_encrypt_xts);
23
+
EXPORT_SYMBOL_GPL(ppc_decrypt_xts);
24
+
25
+
void ppc_encrypt_aes(u8 *out, const u8 *in, const u32 *key_enc, u32 rounds);
26
+
void ppc_decrypt_aes(u8 *out, const u8 *in, const u32 *key_dec, u32 rounds);
27
+
28
+
static void spe_begin(void)
29
+
{
30
+
/* disable preemption and save users SPE registers if required */
31
+
preempt_disable();
32
+
enable_kernel_spe();
33
+
}
34
+
35
+
static void spe_end(void)
36
+
{
37
+
disable_kernel_spe();
38
+
/* reenable preemption */
39
+
preempt_enable();
40
+
}
41
+
42
+
static void aes_preparekey_arch(union aes_enckey_arch *k,
43
+
union aes_invkey_arch *inv_k,
44
+
const u8 *in_key, int key_len, int nrounds)
45
+
{
46
+
if (key_len == AES_KEYSIZE_128)
47
+
ppc_expand_key_128(k->spe_enc_key, in_key);
48
+
else if (key_len == AES_KEYSIZE_192)
49
+
ppc_expand_key_192(k->spe_enc_key, in_key);
50
+
else
51
+
ppc_expand_key_256(k->spe_enc_key, in_key);
52
+
53
+
if (inv_k)
54
+
ppc_generate_decrypt_key(inv_k->spe_dec_key, k->spe_enc_key,
55
+
key_len);
56
+
}
57
+
58
+
static void aes_encrypt_arch(const struct aes_enckey *key,
59
+
u8 out[AES_BLOCK_SIZE],
60
+
const u8 in[AES_BLOCK_SIZE])
61
+
{
62
+
spe_begin();
63
+
ppc_encrypt_aes(out, in, key->k.spe_enc_key, key->nrounds / 2 - 1);
64
+
spe_end();
65
+
}
66
+
67
+
static void aes_decrypt_arch(const struct aes_key *key,
68
+
u8 out[AES_BLOCK_SIZE],
69
+
const u8 in[AES_BLOCK_SIZE])
70
+
{
71
+
spe_begin();
72
+
ppc_decrypt_aes(out, in, key->inv_k.spe_dec_key, key->nrounds / 2 - 1);
73
+
spe_end();
74
+
}