Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: x86/aegis128 - eliminate some indirect calls
Instead of using a struct of function pointers to decide whether to call
the encryption or decryption assembly functions, use a conditional
branch on a bool. Force-inline the functions to avoid actually
generating the branch. This improves performance slightly since
indirect calls are slow. Remove the now-unnecessary CFI stubs.
Note that just force-inlining the existing functions might cause the
compiler to optimize out the indirect branches, but that would not be a
reliable way to do it and the CFI stubs would still be required.
Reviewed-by: Ondrej Mosnacek <omosnace@redhat.com>
Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
authored by
Eric Biggers
and committed by
Herbert Xu
b8d2e7ba
ebb445f5
+40
-43
+4
-5
arch/x86/crypto/aegis128-aesni-asm.S
+4
-5
arch/x86/crypto/aegis128-aesni-asm.S
···
7
7
*/
8
8
9
9
#include <linux/linkage.h>
10
-
#include <linux/cfi_types.h>
11
10
#include <asm/frame.h>
12
11
13
12
#define STATE0 %xmm0
···
402
403
* void crypto_aegis128_aesni_enc(void *state, unsigned int length,
403
404
* const void *src, void *dst);
404
405
*/
405
-
SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc)
406
+
SYM_FUNC_START(crypto_aegis128_aesni_enc)
406
407
FRAME_BEGIN
407
408
408
409
cmp $0x10, LEN
···
499
500
* void crypto_aegis128_aesni_enc_tail(void *state, unsigned int length,
500
501
* const void *src, void *dst);
501
502
*/
502
-
SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc_tail)
503
+
SYM_FUNC_START(crypto_aegis128_aesni_enc_tail)
503
504
FRAME_BEGIN
504
505
505
506
/* load the state: */
···
556
557
* void crypto_aegis128_aesni_dec(void *state, unsigned int length,
557
558
* const void *src, void *dst);
558
559
*/
559
-
SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec)
560
+
SYM_FUNC_START(crypto_aegis128_aesni_dec)
560
561
FRAME_BEGIN
561
562
562
563
cmp $0x10, LEN
···
653
654
* void crypto_aegis128_aesni_dec_tail(void *state, unsigned int length,
654
655
* const void *src, void *dst);
655
656
*/
656
-
SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec_tail)
657
+
SYM_FUNC_START(crypto_aegis128_aesni_dec_tail)
657
658
FRAME_BEGIN
658
659
659
660
/* load the state: */
+36
-38
arch/x86/crypto/aegis128-aesni-glue.c
+36
-38
arch/x86/crypto/aegis128-aesni-glue.c
···
56
56
struct aegis_block key;
57
57
};
58
58
59
-
struct aegis_crypt_ops {
60
-
int (*skcipher_walk_init)(struct skcipher_walk *walk,
61
-
struct aead_request *req, bool atomic);
62
-
63
-
void (*crypt_blocks)(void *state, unsigned int length, const void *src,
64
-
void *dst);
65
-
void (*crypt_tail)(void *state, unsigned int length, const void *src,
66
-
void *dst);
67
-
};
68
-
69
59
static void crypto_aegis128_aesni_process_ad(
70
60
struct aegis_state *state, struct scatterlist *sg_src,
71
61
unsigned int assoclen)
···
104
114
}
105
115
}
106
116
107
-
static void crypto_aegis128_aesni_process_crypt(
108
-
struct aegis_state *state, struct skcipher_walk *walk,
109
-
const struct aegis_crypt_ops *ops)
117
+
static __always_inline void
118
+
crypto_aegis128_aesni_process_crypt(struct aegis_state *state,
119
+
struct skcipher_walk *walk, bool enc)
110
120
{
111
121
while (walk->nbytes >= AEGIS128_BLOCK_SIZE) {
112
-
ops->crypt_blocks(state,
113
-
round_down(walk->nbytes, AEGIS128_BLOCK_SIZE),
114
-
walk->src.virt.addr, walk->dst.virt.addr);
122
+
if (enc)
123
+
crypto_aegis128_aesni_enc(
124
+
state,
125
+
round_down(walk->nbytes,
126
+
AEGIS128_BLOCK_SIZE),
127
+
walk->src.virt.addr,
128
+
walk->dst.virt.addr);
129
+
else
130
+
crypto_aegis128_aesni_dec(
131
+
state,
132
+
round_down(walk->nbytes,
133
+
AEGIS128_BLOCK_SIZE),
134
+
walk->src.virt.addr,
135
+
walk->dst.virt.addr);
115
136
skcipher_walk_done(walk, walk->nbytes % AEGIS128_BLOCK_SIZE);
116
137
}
117
138
118
139
if (walk->nbytes) {
119
-
ops->crypt_tail(state, walk->nbytes, walk->src.virt.addr,
120
-
walk->dst.virt.addr);
140
+
if (enc)
141
+
crypto_aegis128_aesni_enc_tail(state, walk->nbytes,
142
+
walk->src.virt.addr,
143
+
walk->dst.virt.addr);
144
+
else
145
+
crypto_aegis128_aesni_dec_tail(state, walk->nbytes,
146
+
walk->src.virt.addr,
147
+
walk->dst.virt.addr);
121
148
skcipher_walk_done(walk, 0);
122
149
}
123
150
}
···
169
162
return 0;
170
163
}
171
164
172
-
static void crypto_aegis128_aesni_crypt(struct aead_request *req,
173
-
struct aegis_block *tag_xor,
174
-
unsigned int cryptlen,
175
-
const struct aegis_crypt_ops *ops)
165
+
static __always_inline void
166
+
crypto_aegis128_aesni_crypt(struct aead_request *req,
167
+
struct aegis_block *tag_xor,
168
+
unsigned int cryptlen, bool enc)
176
169
{
177
170
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
178
171
struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(tfm);
179
172
struct skcipher_walk walk;
180
173
struct aegis_state state;
181
174
182
-
ops->skcipher_walk_init(&walk, req, true);
175
+
if (enc)
176
+
skcipher_walk_aead_encrypt(&walk, req, true);
177
+
else
178
+
skcipher_walk_aead_decrypt(&walk, req, true);
183
179
184
180
kernel_fpu_begin();
185
181
186
182
crypto_aegis128_aesni_init(&state, ctx->key.bytes, req->iv);
187
183
crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen);
188
-
crypto_aegis128_aesni_process_crypt(&state, &walk, ops);
184
+
crypto_aegis128_aesni_process_crypt(&state, &walk, enc);
189
185
crypto_aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen);
190
186
191
187
kernel_fpu_end();
···
196
186
197
187
static int crypto_aegis128_aesni_encrypt(struct aead_request *req)
198
188
{
199
-
static const struct aegis_crypt_ops OPS = {
200
-
.skcipher_walk_init = skcipher_walk_aead_encrypt,
201
-
.crypt_blocks = crypto_aegis128_aesni_enc,
202
-
.crypt_tail = crypto_aegis128_aesni_enc_tail,
203
-
};
204
-
205
189
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
206
190
struct aegis_block tag = {};
207
191
unsigned int authsize = crypto_aead_authsize(tfm);
208
192
unsigned int cryptlen = req->cryptlen;
209
193
210
-
crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);
194
+
crypto_aegis128_aesni_crypt(req, &tag, cryptlen, true);
211
195
212
196
scatterwalk_map_and_copy(tag.bytes, req->dst,
213
197
req->assoclen + cryptlen, authsize, 1);
···
212
208
{
213
209
static const struct aegis_block zeros = {};
214
210
215
-
static const struct aegis_crypt_ops OPS = {
216
-
.skcipher_walk_init = skcipher_walk_aead_decrypt,
217
-
.crypt_blocks = crypto_aegis128_aesni_dec,
218
-
.crypt_tail = crypto_aegis128_aesni_dec_tail,
219
-
};
220
-
221
211
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
222
212
struct aegis_block tag;
223
213
unsigned int authsize = crypto_aead_authsize(tfm);
···
220
222
scatterwalk_map_and_copy(tag.bytes, req->src,
221
223
req->assoclen + cryptlen, authsize, 0);
222
224
223
-
crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS);
225
+
crypto_aegis128_aesni_crypt(req, &tag, cryptlen, false);
224
226
225
227
return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0;
226
228
}