Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
lib/crypto: powerpc/aes: Migrate POWER8 optimized code into library
Move the POWER8 AES assembly code into lib/crypto/, wire the key
expansion and single-block en/decryption functions up to the AES library
API, and remove the superseded "p8_aes" crypto_cipher algorithm.
The result is that both the AES library and crypto_cipher APIs are now
optimized for POWER8, whereas previously only crypto_cipher was (and
optimizations weren't enabled by default, which this commit fixes too).
Note that many of the functions in the POWER8 assembly code are still
used by the AES mode implementations in arch/powerpc/crypto/. For now,
just export these functions. These exports will go away once the AES
modes are migrated to the library as well. (Trying to split up the
assembly file seemed like much more trouble than it would be worth.)
Another challenge with this code is that the POWER8 assembly code uses a
custom format for the expanded AES key. Since that code is imported
from OpenSSL and is also targeted to POWER8 (rather than POWER9 which
has better data movement and byteswap instructions), that is not easily
changed. For now I've just kept the custom format. To maintain full
correctness, this requires executing some slow fallback code in the case
where the usability of VSX changes between key expansion and use. This
should be tolerable, as this case shouldn't happen in practice.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20260112192035.10427-14-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
+226
-172
+3
-4
arch/powerpc/crypto/Makefile
+3
-4
arch/powerpc/crypto/Makefile
···
11
11
12
12
aes-ppc-spe-y := aes-spe-glue.o
13
13
aes-gcm-p10-crypto-y := aes-gcm-p10-glue.o aes-gcm-p10.o ghashp10-ppc.o aesp10-ppc.o
14
-
vmx-crypto-objs := vmx.o aesp8-ppc.o ghashp8-ppc.o aes.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o
14
+
vmx-crypto-objs := vmx.o ghashp8-ppc.o aes_cbc.o aes_ctr.o aes_xts.o ghash.o
15
15
16
16
ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y)
17
17
override flavour := linux-ppc64le
···
26
26
quiet_cmd_perl = PERL $@
27
27
cmd_perl = $(PERL) $< $(flavour) > $@
28
28
29
-
targets += aesp10-ppc.S ghashp10-ppc.S aesp8-ppc.S ghashp8-ppc.S
29
+
targets += aesp10-ppc.S ghashp10-ppc.S ghashp8-ppc.S
30
30
31
31
$(obj)/aesp10-ppc.S $(obj)/ghashp10-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE
32
32
$(call if_changed,perl)
33
33
34
-
$(obj)/aesp8-ppc.S $(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE
34
+
$(obj)/ghashp8-ppc.S: $(obj)/%.S: $(src)/%.pl FORCE
35
35
$(call if_changed,perl)
36
36
37
37
OBJECT_FILES_NON_STANDARD_aesp10-ppc.o := y
38
38
OBJECT_FILES_NON_STANDARD_ghashp10-ppc.o := y
39
-
OBJECT_FILES_NON_STANDARD_aesp8-ppc.o := y
40
39
OBJECT_FILES_NON_STANDARD_ghashp8-ppc.o := y
-134
arch/powerpc/crypto/aes.c
-134
arch/powerpc/crypto/aes.c
···
1
-
// SPDX-License-Identifier: GPL-2.0-only
2
-
/*
3
-
* AES routines supporting VMX instructions on the Power 8
4
-
*
5
-
* Copyright (C) 2015 International Business Machines Inc.
6
-
*
7
-
* Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
8
-
*/
9
-
10
-
#include <asm/simd.h>
11
-
#include <asm/switch_to.h>
12
-
#include <crypto/aes.h>
13
-
#include <crypto/internal/cipher.h>
14
-
#include <crypto/internal/simd.h>
15
-
#include <linux/err.h>
16
-
#include <linux/kernel.h>
17
-
#include <linux/module.h>
18
-
#include <linux/uaccess.h>
19
-
20
-
#include "aesp8-ppc.h"
21
-
22
-
struct p8_aes_ctx {
23
-
struct crypto_cipher *fallback;
24
-
struct p8_aes_key enc_key;
25
-
struct p8_aes_key dec_key;
26
-
};
27
-
28
-
static int p8_aes_init(struct crypto_tfm *tfm)
29
-
{
30
-
const char *alg = crypto_tfm_alg_name(tfm);
31
-
struct crypto_cipher *fallback;
32
-
struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
33
-
34
-
fallback = crypto_alloc_cipher(alg, 0, CRYPTO_ALG_NEED_FALLBACK);
35
-
if (IS_ERR(fallback)) {
36
-
printk(KERN_ERR
37
-
"Failed to allocate transformation for '%s': %ld\n",
38
-
alg, PTR_ERR(fallback));
39
-
return PTR_ERR(fallback);
40
-
}
41
-
42
-
crypto_cipher_set_flags(fallback,
43
-
crypto_cipher_get_flags((struct
44
-
crypto_cipher *)
45
-
tfm));
46
-
ctx->fallback = fallback;
47
-
48
-
return 0;
49
-
}
50
-
51
-
static void p8_aes_exit(struct crypto_tfm *tfm)
52
-
{
53
-
struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
54
-
55
-
if (ctx->fallback) {
56
-
crypto_free_cipher(ctx->fallback);
57
-
ctx->fallback = NULL;
58
-
}
59
-
}
60
-
61
-
static int p8_aes_setkey(struct crypto_tfm *tfm, const u8 *key,
62
-
unsigned int keylen)
63
-
{
64
-
int ret;
65
-
struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
66
-
67
-
preempt_disable();
68
-
pagefault_disable();
69
-
enable_kernel_vsx();
70
-
ret = aes_p8_set_encrypt_key(key, keylen * 8, &ctx->enc_key);
71
-
ret |= aes_p8_set_decrypt_key(key, keylen * 8, &ctx->dec_key);
72
-
disable_kernel_vsx();
73
-
pagefault_enable();
74
-
preempt_enable();
75
-
76
-
ret |= crypto_cipher_setkey(ctx->fallback, key, keylen);
77
-
78
-
return ret ? -EINVAL : 0;
79
-
}
80
-
81
-
static void p8_aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
82
-
{
83
-
struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
84
-
85
-
if (!crypto_simd_usable()) {
86
-
crypto_cipher_encrypt_one(ctx->fallback, dst, src);
87
-
} else {
88
-
preempt_disable();
89
-
pagefault_disable();
90
-
enable_kernel_vsx();
91
-
aes_p8_encrypt(src, dst, &ctx->enc_key);
92
-
disable_kernel_vsx();
93
-
pagefault_enable();
94
-
preempt_enable();
95
-
}
96
-
}
97
-
98
-
static void p8_aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
99
-
{
100
-
struct p8_aes_ctx *ctx = crypto_tfm_ctx(tfm);
101
-
102
-
if (!crypto_simd_usable()) {
103
-
crypto_cipher_decrypt_one(ctx->fallback, dst, src);
104
-
} else {
105
-
preempt_disable();
106
-
pagefault_disable();
107
-
enable_kernel_vsx();
108
-
aes_p8_decrypt(src, dst, &ctx->dec_key);
109
-
disable_kernel_vsx();
110
-
pagefault_enable();
111
-
preempt_enable();
112
-
}
113
-
}
114
-
115
-
struct crypto_alg p8_aes_alg = {
116
-
.cra_name = "aes",
117
-
.cra_driver_name = "p8_aes",
118
-
.cra_module = THIS_MODULE,
119
-
.cra_priority = 1000,
120
-
.cra_type = NULL,
121
-
.cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_NEED_FALLBACK,
122
-
.cra_alignmask = 0,
123
-
.cra_blocksize = AES_BLOCK_SIZE,
124
-
.cra_ctxsize = sizeof(struct p8_aes_ctx),
125
-
.cra_init = p8_aes_init,
126
-
.cra_exit = p8_aes_exit,
127
-
.cra_cipher = {
128
-
.cia_min_keysize = AES_MIN_KEY_SIZE,
129
-
.cia_max_keysize = AES_MAX_KEY_SIZE,
130
-
.cia_setkey = p8_aes_setkey,
131
-
.cia_encrypt = p8_aes_encrypt,
132
-
.cia_decrypt = p8_aes_decrypt,
133
-
},
134
-
};
-23
arch/powerpc/crypto/aesp8-ppc.h
-23
arch/powerpc/crypto/aesp8-ppc.h
···
2
2
#include <linux/types.h>
3
3
#include <crypto/aes.h>
4
4
5
-
struct p8_aes_key {
6
-
u8 key[AES_MAX_KEYLENGTH];
7
-
int rounds;
8
-
};
9
-
10
5
extern struct shash_alg p8_ghash_alg;
11
-
extern struct crypto_alg p8_aes_alg;
12
6
extern struct skcipher_alg p8_aes_cbc_alg;
13
7
extern struct skcipher_alg p8_aes_ctr_alg;
14
8
extern struct skcipher_alg p8_aes_xts_alg;
15
-
16
-
int aes_p8_set_encrypt_key(const u8 *userKey, const int bits,
17
-
struct p8_aes_key *key);
18
-
int aes_p8_set_decrypt_key(const u8 *userKey, const int bits,
19
-
struct p8_aes_key *key);
20
-
void aes_p8_encrypt(const u8 *in, u8 *out, const struct p8_aes_key *key);
21
-
void aes_p8_decrypt(const u8 *in, u8 *out, const struct p8_aes_key *key);
22
-
void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len,
23
-
const struct p8_aes_key *key, u8 *iv, const int enc);
24
-
void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out, size_t len,
25
-
const struct p8_aes_key *key, const u8 *iv);
26
-
void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len,
27
-
const struct p8_aes_key *key1,
28
-
const struct p8_aes_key *key2, u8 *iv);
29
-
void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len,
30
-
const struct p8_aes_key *key1,
31
-
const struct p8_aes_key *key2, u8 *iv);
+1
arch/powerpc/crypto/aesp8-ppc.pl
lib/crypto/powerpc/aesp8-ppc.pl
+1
arch/powerpc/crypto/aesp8-ppc.pl
lib/crypto/powerpc/aesp8-ppc.pl
···
105
105
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
106
106
( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
107
107
( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
108
+
( $xlate="${dir}../../../arch/powerpc/crypto/ppc-xlate.pl" and -f $xlate) or
108
109
die "can't locate ppc-xlate.pl";
109
110
110
111
open STDOUT,"| $^X $xlate $flavour ".shift || die "can't call $xlate: $!";
+1
-9
arch/powerpc/crypto/vmx.c
+1
-9
arch/powerpc/crypto/vmx.c
···
27
27
if (ret)
28
28
goto err;
29
29
30
-
ret = crypto_register_alg(&p8_aes_alg);
31
-
if (ret)
32
-
goto err_unregister_ghash;
33
-
34
30
ret = crypto_register_skcipher(&p8_aes_cbc_alg);
35
31
if (ret)
36
-
goto err_unregister_aes;
32
+
goto err_unregister_ghash;
37
33
38
34
ret = crypto_register_skcipher(&p8_aes_ctr_alg);
39
35
if (ret)
···
45
49
crypto_unregister_skcipher(&p8_aes_ctr_alg);
46
50
err_unregister_aes_cbc:
47
51
crypto_unregister_skcipher(&p8_aes_cbc_alg);
48
-
err_unregister_aes:
49
-
crypto_unregister_alg(&p8_aes_alg);
50
52
err_unregister_ghash:
51
53
crypto_unregister_shash(&p8_ghash_alg);
52
54
err:
···
56
62
crypto_unregister_skcipher(&p8_aes_xts_alg);
57
63
crypto_unregister_skcipher(&p8_aes_ctr_alg);
58
64
crypto_unregister_skcipher(&p8_aes_cbc_alg);
59
-
crypto_unregister_alg(&p8_aes_alg);
60
65
crypto_unregister_shash(&p8_ghash_alg);
61
66
}
62
67
···
67
74
"support on Power 8");
68
75
MODULE_LICENSE("GPL");
69
76
MODULE_VERSION("1.0.0");
70
-
MODULE_IMPORT_NS("CRYPTO_INTERNAL");
+41
include/crypto/aes.h
+41
include/crypto/aes.h
···
18
18
#define AES_MAX_KEYLENGTH (15 * 16)
19
19
#define AES_MAX_KEYLENGTH_U32 (AES_MAX_KEYLENGTH / sizeof(u32))
20
20
21
+
/*
22
+
* The POWER8 VSX optimized AES assembly code is borrowed from OpenSSL and
23
+
* inherits OpenSSL's AES_KEY format, which stores the number of rounds after
24
+
* the round keys. That assembly code is difficult to change. So for
25
+
* compatibility purposes we reserve space for the extra nrounds field on PPC64.
26
+
*
27
+
* Note: when prepared for decryption, the round keys are just the reversed
28
+
* standard round keys, not the round keys for the Equivalent Inverse Cipher.
29
+
*/
30
+
struct p8_aes_key {
31
+
u32 rndkeys[AES_MAX_KEYLENGTH_U32];
32
+
int nrounds;
33
+
};
34
+
21
35
union aes_enckey_arch {
22
36
u32 rndkeys[AES_MAX_KEYLENGTH_U32];
23
37
#ifdef CONFIG_CRYPTO_LIB_AES_ARCH
24
38
#if defined(CONFIG_PPC) && defined(CONFIG_SPE)
25
39
/* Used unconditionally (when SPE AES code is enabled in kconfig) */
26
40
u32 spe_enc_key[AES_MAX_KEYLENGTH_U32] __aligned(8);
41
+
#elif defined(CONFIG_PPC)
42
+
/*
43
+
* Kernels that include the POWER8 VSX optimized AES code use this field
44
+
* when that code is usable at key preparation time. Otherwise they
45
+
* fall back to rndkeys. In the latter case, p8.nrounds (which doesn't
46
+
* overlap rndkeys) is set to 0 to differentiate the two formats.
47
+
*/
48
+
struct p8_aes_key p8;
27
49
#endif
28
50
#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */
29
51
};
···
56
34
#if defined(CONFIG_PPC) && defined(CONFIG_SPE)
57
35
/* Used unconditionally (when SPE AES code is enabled in kconfig) */
58
36
u32 spe_dec_key[AES_MAX_KEYLENGTH_U32] __aligned(8);
37
+
#elif defined(CONFIG_PPC)
38
+
/* Used conditionally, analogous to aes_enckey_arch::p8 */
39
+
struct p8_aes_key p8;
59
40
#endif
60
41
#endif /* CONFIG_CRYPTO_LIB_AES_ARCH */
61
42
};
···
180
155
u8 *iv, u32 *key_twk);
181
156
void ppc_decrypt_xts(u8 *out, const u8 *in, u32 *key_dec, u32 rounds, u32 bytes,
182
157
u8 *iv, u32 *key_twk);
158
+
int aes_p8_set_encrypt_key(const u8 *userKey, const int bits,
159
+
struct p8_aes_key *key);
160
+
int aes_p8_set_decrypt_key(const u8 *userKey, const int bits,
161
+
struct p8_aes_key *key);
162
+
void aes_p8_encrypt(const u8 *in, u8 *out, const struct p8_aes_key *key);
163
+
void aes_p8_decrypt(const u8 *in, u8 *out, const struct p8_aes_key *key);
164
+
void aes_p8_cbc_encrypt(const u8 *in, u8 *out, size_t len,
165
+
const struct p8_aes_key *key, u8 *iv, const int enc);
166
+
void aes_p8_ctr32_encrypt_blocks(const u8 *in, u8 *out, size_t len,
167
+
const struct p8_aes_key *key, const u8 *iv);
168
+
void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len,
169
+
const struct p8_aes_key *key1,
170
+
const struct p8_aes_key *key2, u8 *iv);
171
+
void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len,
172
+
const struct p8_aes_key *key1,
173
+
const struct p8_aes_key *key2, u8 *iv);
183
174
#endif
184
175
185
176
/**
+1
-1
lib/crypto/Kconfig
+1
-1
lib/crypto/Kconfig
+13
-1
lib/crypto/Makefile
+13
-1
lib/crypto/Makefile
···
35
35
powerpc/aes-spe-keys.o \
36
36
powerpc/aes-spe-modes.o \
37
37
powerpc/aes-tab-4k.o
38
-
endif
38
+
else
39
+
libaes-y += powerpc/aesp8-ppc.o
40
+
aes-perlasm-flavour-y := linux-ppc64
41
+
aes-perlasm-flavour-$(CONFIG_PPC64_ELF_ABI_V2) := linux-ppc64-elfv2
42
+
aes-perlasm-flavour-$(CONFIG_CPU_LITTLE_ENDIAN) := linux-ppc64le
43
+
quiet_cmd_perlasm_aes = PERLASM $@
44
+
cmd_perlasm_aes = $(PERL) $< $(aes-perlasm-flavour-y) $@
45
+
# Use if_changed instead of cmd, in case the flavour changed.
46
+
$(obj)/powerpc/aesp8-ppc.S: $(src)/powerpc/aesp8-ppc.pl FORCE
47
+
$(call if_changed,perlasm_aes)
48
+
targets += powerpc/aesp8-ppc.S
49
+
OBJECT_FILES_NON_STANDARD_powerpc/aesp8-ppc.o := y
50
+
endif # !CONFIG_SPE
39
51
endif # CONFIG_PPC
40
52
41
53
endif # CONFIG_CRYPTO_LIB_AES_ARCH
+164
lib/crypto/powerpc/aes.h
+164
lib/crypto/powerpc/aes.h
···
1
1
/* SPDX-License-Identifier: GPL-2.0-only */
2
2
/*
3
3
* Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
4
+
* Copyright (C) 2015 International Business Machines Inc.
4
5
* Copyright 2026 Google LLC
5
6
*/
6
7
#include <asm/simd.h>
···
10
9
#include <linux/jump_label.h>
11
10
#include <linux/preempt.h>
12
11
#include <linux/uaccess.h>
12
+
13
+
#ifdef CONFIG_SPE
13
14
14
15
EXPORT_SYMBOL_GPL(ppc_expand_key_128);
15
16
EXPORT_SYMBOL_GPL(ppc_expand_key_192);
···
75
72
ppc_decrypt_aes(out, in, key->inv_k.spe_dec_key, key->nrounds / 2 - 1);
76
73
spe_end();
77
74
}
75
+
76
+
#else /* CONFIG_SPE */
77
+
78
+
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_vec_crypto);
79
+
80
+
EXPORT_SYMBOL_GPL(aes_p8_set_encrypt_key);
81
+
EXPORT_SYMBOL_GPL(aes_p8_set_decrypt_key);
82
+
EXPORT_SYMBOL_GPL(aes_p8_encrypt);
83
+
EXPORT_SYMBOL_GPL(aes_p8_decrypt);
84
+
EXPORT_SYMBOL_GPL(aes_p8_cbc_encrypt);
85
+
EXPORT_SYMBOL_GPL(aes_p8_ctr32_encrypt_blocks);
86
+
EXPORT_SYMBOL_GPL(aes_p8_xts_encrypt);
87
+
EXPORT_SYMBOL_GPL(aes_p8_xts_decrypt);
88
+
89
+
static inline bool is_vsx_format(const struct p8_aes_key *key)
90
+
{
91
+
return key->nrounds != 0;
92
+
}
93
+
94
+
/*
95
+
* Convert a round key from VSX to generic format by reflecting the 16 bytes,
96
+
* and (if apply_inv_mix=true) applying InvMixColumn to each column.
97
+
*
98
+
* It would be nice if the VSX and generic key formats would be compatible. But
99
+
* that's very difficult to do, with the assembly code having been borrowed from
100
+
* OpenSSL and also targeted to POWER8 rather than POWER9.
101
+
*
102
+
* Fortunately, this conversion should only be needed in extremely rare cases,
103
+
* possibly not at all in practice. It's just included for full correctness.
104
+
*/
105
+
static void rndkey_from_vsx(u32 out[4], const u32 in[4], bool apply_inv_mix)
106
+
{
107
+
u32 k0 = swab32(in[0]);
108
+
u32 k1 = swab32(in[1]);
109
+
u32 k2 = swab32(in[2]);
110
+
u32 k3 = swab32(in[3]);
111
+
112
+
if (apply_inv_mix) {
113
+
k0 = inv_mix_columns(k0);
114
+
k1 = inv_mix_columns(k1);
115
+
k2 = inv_mix_columns(k2);
116
+
k3 = inv_mix_columns(k3);
117
+
}
118
+
out[0] = k3;
119
+
out[1] = k2;
120
+
out[2] = k1;
121
+
out[3] = k0;
122
+
}
123
+
124
+
static void aes_preparekey_arch(union aes_enckey_arch *k,
125
+
union aes_invkey_arch *inv_k,
126
+
const u8 *in_key, int key_len, int nrounds)
127
+
{
128
+
const int keybits = 8 * key_len;
129
+
int ret;
130
+
131
+
if (static_branch_likely(&have_vec_crypto) && likely(may_use_simd())) {
132
+
preempt_disable();
133
+
pagefault_disable();
134
+
enable_kernel_vsx();
135
+
ret = aes_p8_set_encrypt_key(in_key, keybits, &k->p8);
136
+
/*
137
+
* aes_p8_set_encrypt_key() should never fail here, since the
138
+
* key length was already validated.
139
+
*/
140
+
WARN_ON_ONCE(ret);
141
+
if (inv_k) {
142
+
ret = aes_p8_set_decrypt_key(in_key, keybits,
143
+
&inv_k->p8);
144
+
/* ... and likewise for aes_p8_set_decrypt_key(). */
145
+
WARN_ON_ONCE(ret);
146
+
}
147
+
disable_kernel_vsx();
148
+
pagefault_enable();
149
+
preempt_enable();
150
+
} else {
151
+
aes_expandkey_generic(k->rndkeys,
152
+
inv_k ? inv_k->inv_rndkeys : NULL,
153
+
in_key, key_len);
154
+
/* Mark the key as using the generic format. */
155
+
k->p8.nrounds = 0;
156
+
if (inv_k)
157
+
inv_k->p8.nrounds = 0;
158
+
}
159
+
}
160
+
161
+
static void aes_encrypt_arch(const struct aes_enckey *key,
162
+
u8 out[AES_BLOCK_SIZE],
163
+
const u8 in[AES_BLOCK_SIZE])
164
+
{
165
+
if (static_branch_likely(&have_vec_crypto) &&
166
+
likely(is_vsx_format(&key->k.p8) && may_use_simd())) {
167
+
preempt_disable();
168
+
pagefault_disable();
169
+
enable_kernel_vsx();
170
+
aes_p8_encrypt(in, out, &key->k.p8);
171
+
disable_kernel_vsx();
172
+
pagefault_enable();
173
+
preempt_enable();
174
+
} else if (unlikely(is_vsx_format(&key->k.p8))) {
175
+
/*
176
+
* This handles (the hopefully extremely rare) case where a key
177
+
* was prepared using the VSX optimized format, then encryption
178
+
* is done in a context that cannot use VSX instructions.
179
+
*/
180
+
u32 rndkeys[AES_MAX_KEYLENGTH_U32];
181
+
182
+
for (int i = 0; i < 4 * (key->nrounds + 1); i += 4)
183
+
rndkey_from_vsx(&rndkeys[i],
184
+
&key->k.p8.rndkeys[i], false);
185
+
aes_encrypt_generic(rndkeys, key->nrounds, out, in);
186
+
} else {
187
+
aes_encrypt_generic(key->k.rndkeys, key->nrounds, out, in);
188
+
}
189
+
}
190
+
191
+
static void aes_decrypt_arch(const struct aes_key *key, u8 out[AES_BLOCK_SIZE],
192
+
const u8 in[AES_BLOCK_SIZE])
193
+
{
194
+
if (static_branch_likely(&have_vec_crypto) &&
195
+
likely(is_vsx_format(&key->inv_k.p8) && may_use_simd())) {
196
+
preempt_disable();
197
+
pagefault_disable();
198
+
enable_kernel_vsx();
199
+
aes_p8_decrypt(in, out, &key->inv_k.p8);
200
+
disable_kernel_vsx();
201
+
pagefault_enable();
202
+
preempt_enable();
203
+
} else if (unlikely(is_vsx_format(&key->inv_k.p8))) {
204
+
/*
205
+
* This handles (the hopefully extremely rare) case where a key
206
+
* was prepared using the VSX optimized format, then decryption
207
+
* is done in a context that cannot use VSX instructions.
208
+
*/
209
+
u32 inv_rndkeys[AES_MAX_KEYLENGTH_U32];
210
+
int i;
211
+
212
+
rndkey_from_vsx(&inv_rndkeys[0],
213
+
&key->inv_k.p8.rndkeys[0], false);
214
+
for (i = 4; i < 4 * key->nrounds; i += 4) {
215
+
rndkey_from_vsx(&inv_rndkeys[i],
216
+
&key->inv_k.p8.rndkeys[i], true);
217
+
}
218
+
rndkey_from_vsx(&inv_rndkeys[i],
219
+
&key->inv_k.p8.rndkeys[i], false);
220
+
aes_decrypt_generic(inv_rndkeys, key->nrounds, out, in);
221
+
} else {
222
+
aes_decrypt_generic(key->inv_k.inv_rndkeys, key->nrounds,
223
+
out, in);
224
+
}
225
+
}
226
+
227
+
#define aes_mod_init_arch aes_mod_init_arch
228
+
static void aes_mod_init_arch(void)
229
+
{
230
+
if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
231
+
(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_VEC_CRYPTO))
232
+
static_branch_enable(&have_vec_crypto);
233
+
}
234
+
235
+
#endif /* !CONFIG_SPE */