Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: s390/aes: Migrate optimized code into library
Implement aes_preparekey_arch(), aes_encrypt_arch(), and
aes_decrypt_arch() using the CPACF AES instructions.
Then, remove the superseded "aes-s390" crypto_cipher.
The result is that both the AES library and crypto_cipher APIs use the
CPACF AES instructions, whereas previously only crypto_cipher did (and
it wasn't enabled by default, which this commit fixes as well).
Note that this preserves the optimization where the AES key is stored in
raw form rather than expanded form. CPACF just takes the raw key.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Tested-by: Holger Dengler <dengler@linux.ibm.com>
Reviewed-by: Holger Dengler <dengler@linux.ibm.com>
Link: https://lore.kernel.org/r/20260112192035.10427-16-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+110
-115
-2
arch/s390/crypto/Kconfig
-2
arch/s390/crypto/Kconfig
···
14
14
15
15
config CRYPTO_AES_S390
16
16
tristate "Ciphers: AES, modes: ECB, CBC, CTR, XTS, GCM"
17
-
select CRYPTO_ALGAPI
18
17
select CRYPTO_SKCIPHER
19
18
help
20
-
Block cipher: AES cipher algorithms (FIPS 197)
21
19
AEAD cipher: AES with GCM
22
20
Length-preserving ciphers: AES with ECB, CBC, XTS, and CTR modes
23
21
-113
arch/s390/crypto/aes_s390.c
-113
arch/s390/crypto/aes_s390.c
···
20
20
#include <crypto/algapi.h>
21
21
#include <crypto/ghash.h>
22
22
#include <crypto/internal/aead.h>
23
-
#include <crypto/internal/cipher.h>
24
23
#include <crypto/internal/skcipher.h>
25
24
#include <crypto/scatterwalk.h>
26
25
#include <linux/err.h>
···
44
45
unsigned long fc;
45
46
union {
46
47
struct crypto_skcipher *skcipher;
47
-
struct crypto_cipher *cip;
48
48
} fallback;
49
49
};
50
50
···
68
70
unsigned int buf_bytes;
69
71
u8 *ptr;
70
72
unsigned int nbytes;
71
-
};
72
-
73
-
static int setkey_fallback_cip(struct crypto_tfm *tfm, const u8 *in_key,
74
-
unsigned int key_len)
75
-
{
76
-
struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
77
-
78
-
sctx->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
79
-
sctx->fallback.cip->base.crt_flags |= (tfm->crt_flags &
80
-
CRYPTO_TFM_REQ_MASK);
81
-
82
-
return crypto_cipher_setkey(sctx->fallback.cip, in_key, key_len);
83
-
}
84
-
85
-
static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
86
-
unsigned int key_len)
87
-
{
88
-
struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
89
-
unsigned long fc;
90
-
91
-
/* Pick the correct function code based on the key length */
92
-
fc = (key_len == 16) ? CPACF_KM_AES_128 :
93
-
(key_len == 24) ? CPACF_KM_AES_192 :
94
-
(key_len == 32) ? CPACF_KM_AES_256 : 0;
95
-
96
-
/* Check if the function code is available */
97
-
sctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
98
-
if (!sctx->fc)
99
-
return setkey_fallback_cip(tfm, in_key, key_len);
100
-
101
-
sctx->key_len = key_len;
102
-
memcpy(sctx->key, in_key, key_len);
103
-
return 0;
104
-
}
105
-
106
-
static void crypto_aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
107
-
{
108
-
struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
109
-
110
-
if (unlikely(!sctx->fc)) {
111
-
crypto_cipher_encrypt_one(sctx->fallback.cip, out, in);
112
-
return;
113
-
}
114
-
cpacf_km(sctx->fc, &sctx->key, out, in, AES_BLOCK_SIZE);
115
-
}
116
-
117
-
static void crypto_aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
118
-
{
119
-
struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
120
-
121
-
if (unlikely(!sctx->fc)) {
122
-
crypto_cipher_decrypt_one(sctx->fallback.cip, out, in);
123
-
return;
124
-
}
125
-
cpacf_km(sctx->fc | CPACF_DECRYPT,
126
-
&sctx->key, out, in, AES_BLOCK_SIZE);
127
-
}
128
-
129
-
static int fallback_init_cip(struct crypto_tfm *tfm)
130
-
{
131
-
const char *name = tfm->__crt_alg->cra_name;
132
-
struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
133
-
134
-
sctx->fallback.cip = crypto_alloc_cipher(name, 0,
135
-
CRYPTO_ALG_NEED_FALLBACK);
136
-
137
-
if (IS_ERR(sctx->fallback.cip)) {
138
-
pr_err("Allocating AES fallback algorithm %s failed\n",
139
-
name);
140
-
return PTR_ERR(sctx->fallback.cip);
141
-
}
142
-
143
-
return 0;
144
-
}
145
-
146
-
static void fallback_exit_cip(struct crypto_tfm *tfm)
147
-
{
148
-
struct s390_aes_ctx *sctx = crypto_tfm_ctx(tfm);
149
-
150
-
crypto_free_cipher(sctx->fallback.cip);
151
-
sctx->fallback.cip = NULL;
152
-
}
153
-
154
-
static struct crypto_alg aes_alg = {
155
-
.cra_name = "aes",
156
-
.cra_driver_name = "aes-s390",
157
-
.cra_priority = 300,
158
-
.cra_flags = CRYPTO_ALG_TYPE_CIPHER |
159
-
CRYPTO_ALG_NEED_FALLBACK,
160
-
.cra_blocksize = AES_BLOCK_SIZE,
161
-
.cra_ctxsize = sizeof(struct s390_aes_ctx),
162
-
.cra_module = THIS_MODULE,
163
-
.cra_init = fallback_init_cip,
164
-
.cra_exit = fallback_exit_cip,
165
-
.cra_u = {
166
-
.cipher = {
167
-
.cia_min_keysize = AES_MIN_KEY_SIZE,
168
-
.cia_max_keysize = AES_MAX_KEY_SIZE,
169
-
.cia_setkey = aes_set_key,
170
-
.cia_encrypt = crypto_aes_encrypt,
171
-
.cia_decrypt = crypto_aes_decrypt,
172
-
}
173
-
}
174
73
};
175
74
176
75
static int setkey_fallback_skcipher(struct crypto_skcipher *tfm, const u8 *key,
···
944
1049
},
945
1050
};
946
1051
947
-
static struct crypto_alg *aes_s390_alg;
948
1052
static struct skcipher_alg *aes_s390_skcipher_algs[5];
949
1053
static int aes_s390_skciphers_num;
950
1054
static struct aead_alg *aes_s390_aead_alg;
···
960
1066
961
1067
static void aes_s390_fini(void)
962
1068
{
963
-
if (aes_s390_alg)
964
-
crypto_unregister_alg(aes_s390_alg);
965
1069
while (aes_s390_skciphers_num--)
966
1070
crypto_unregister_skcipher(aes_s390_skcipher_algs[aes_s390_skciphers_num]);
967
1071
if (ctrblk)
···
982
1090
if (cpacf_test_func(&km_functions, CPACF_KM_AES_128) ||
983
1091
cpacf_test_func(&km_functions, CPACF_KM_AES_192) ||
984
1092
cpacf_test_func(&km_functions, CPACF_KM_AES_256)) {
985
-
ret = crypto_register_alg(&aes_alg);
986
-
if (ret)
987
-
goto out_err;
988
-
aes_s390_alg = &aes_alg;
989
1093
ret = aes_s390_register_skcipher(&ecb_aes_alg);
990
1094
if (ret)
991
1095
goto out_err;
···
1044
1156
1045
1157
MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm");
1046
1158
MODULE_LICENSE("GPL");
1047
-
MODULE_IMPORT_NS("CRYPTO_INTERNAL");
+3
include/crypto/aes.h
+3
include/crypto/aes.h
···
46
46
* overlap rndkeys) is set to 0 to differentiate the two formats.
47
47
*/
48
48
struct p8_aes_key p8;
49
+
#elif defined(CONFIG_S390)
50
+
/* Used when the CPU supports CPACF AES for this key's length */
51
+
u8 raw_key[AES_MAX_KEY_SIZE];
49
52
#endif
50
53
#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */
51
54
};
+1
lib/crypto/Kconfig
+1
lib/crypto/Kconfig
+106
lib/crypto/s390/aes.h
+106
lib/crypto/s390/aes.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0-or-later */
2
+
/*
3
+
* AES optimized using the CP Assist for Cryptographic Functions (CPACF)
4
+
*
5
+
* Copyright 2026 Google LLC
6
+
*/
7
+
#include <asm/cpacf.h>
8
+
#include <linux/cpufeature.h>
9
+
10
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_aes128);
11
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_aes192);
12
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_cpacf_aes256);
13
+
14
+
/*
15
+
* When the CPU supports CPACF AES for the requested key length, we need only
16
+
* save a copy of the raw AES key, as that's what the CPACF instructions need.
17
+
*
18
+
* When unsupported, fall back to the generic key expansion and en/decryption.
19
+
*/
20
+
static void aes_preparekey_arch(union aes_enckey_arch *k,
21
+
union aes_invkey_arch *inv_k,
22
+
const u8 *in_key, int key_len, int nrounds)
23
+
{
24
+
if (key_len == AES_KEYSIZE_128) {
25
+
if (static_branch_likely(&have_cpacf_aes128)) {
26
+
memcpy(k->raw_key, in_key, AES_KEYSIZE_128);
27
+
return;
28
+
}
29
+
} else if (key_len == AES_KEYSIZE_192) {
30
+
if (static_branch_likely(&have_cpacf_aes192)) {
31
+
memcpy(k->raw_key, in_key, AES_KEYSIZE_192);
32
+
return;
33
+
}
34
+
} else {
35
+
if (static_branch_likely(&have_cpacf_aes256)) {
36
+
memcpy(k->raw_key, in_key, AES_KEYSIZE_256);
37
+
return;
38
+
}
39
+
}
40
+
aes_expandkey_generic(k->rndkeys, inv_k ? inv_k->inv_rndkeys : NULL,
41
+
in_key, key_len);
42
+
}
43
+
44
+
static inline bool aes_crypt_s390(const struct aes_enckey *key,
45
+
u8 out[AES_BLOCK_SIZE],
46
+
const u8 in[AES_BLOCK_SIZE], int decrypt)
47
+
{
48
+
if (key->len == AES_KEYSIZE_128) {
49
+
if (static_branch_likely(&have_cpacf_aes128)) {
50
+
cpacf_km(CPACF_KM_AES_128 | decrypt,
51
+
(void *)key->k.raw_key, out, in,
52
+
AES_BLOCK_SIZE);
53
+
return true;
54
+
}
55
+
} else if (key->len == AES_KEYSIZE_192) {
56
+
if (static_branch_likely(&have_cpacf_aes192)) {
57
+
cpacf_km(CPACF_KM_AES_192 | decrypt,
58
+
(void *)key->k.raw_key, out, in,
59
+
AES_BLOCK_SIZE);
60
+
return true;
61
+
}
62
+
} else {
63
+
if (static_branch_likely(&have_cpacf_aes256)) {
64
+
cpacf_km(CPACF_KM_AES_256 | decrypt,
65
+
(void *)key->k.raw_key, out, in,
66
+
AES_BLOCK_SIZE);
67
+
return true;
68
+
}
69
+
}
70
+
return false;
71
+
}
72
+
73
+
static void aes_encrypt_arch(const struct aes_enckey *key,
74
+
u8 out[AES_BLOCK_SIZE],
75
+
const u8 in[AES_BLOCK_SIZE])
76
+
{
77
+
if (likely(aes_crypt_s390(key, out, in, 0)))
78
+
return;
79
+
aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
80
+
}
81
+
82
+
static void aes_decrypt_arch(const struct aes_key *key,
83
+
u8 out[AES_BLOCK_SIZE],
84
+
const u8 in[AES_BLOCK_SIZE])
85
+
{
86
+
if (likely(aes_crypt_s390((const struct aes_enckey *)key, out, in,
87
+
CPACF_DECRYPT)))
88
+
return;
89
+
aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds, out, in);
90
+
}
91
+
92
+
#define aes_mod_init_arch aes_mod_init_arch
93
+
static void aes_mod_init_arch(void)
94
+
{
95
+
if (cpu_have_feature(S390_CPU_FEATURE_MSA)) {
96
+
cpacf_mask_t km_functions;
97
+
98
+
cpacf_query(CPACF_KM, &km_functions);
99
+
if (cpacf_test_func(&km_functions, CPACF_KM_AES_128))
100
+
static_branch_enable(&have_cpacf_aes128);
101
+
if (cpacf_test_func(&km_functions, CPACF_KM_AES_192))
102
+
static_branch_enable(&have_cpacf_aes192);
103
+
if (cpacf_test_func(&km_functions, CPACF_KM_AES_256))
104
+
static_branch_enable(&have_cpacf_aes256);
105
+
}
106
+
}