Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: skcipher - Use restrict rather than hand-rolling accesses
Rather than accessing 'alg' directly to avoid the aliasing issue
which leads to unnecessary reloads, use the __restrict keyword
to explicitly tell the compiler that there is no aliasing.
This generates equivalent if not superior code on x86 with gcc 12.
Note that in skcipher_walk_virt the alg assignment is moved after
might_sleep_if because that function is a compiler barrier and
forces a reload.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
+31
-34
+23
-28
crypto/skcipher.c
+23
-28
crypto/skcipher.c
···
293
293
return skcipher_walk_next(walk);
294
294
}
295
295
296
-
int skcipher_walk_virt(struct skcipher_walk *walk,
297
-
struct skcipher_request *req, bool atomic)
296
+
int skcipher_walk_virt(struct skcipher_walk *__restrict walk,
297
+
struct skcipher_request *__restrict req, bool atomic)
298
298
{
299
-
const struct skcipher_alg *alg =
300
-
crypto_skcipher_alg(crypto_skcipher_reqtfm(req));
299
+
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
300
+
struct skcipher_alg *alg;
301
301
302
302
might_sleep_if(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP);
303
+
304
+
alg = crypto_skcipher_alg(tfm);
303
305
304
306
walk->total = req->cryptlen;
305
307
walk->nbytes = 0;
···
318
316
scatterwalk_start(&walk->in, req->src);
319
317
scatterwalk_start(&walk->out, req->dst);
320
318
321
-
/*
322
-
* Accessing 'alg' directly generates better code than using the
323
-
* crypto_skcipher_blocksize() and similar helper functions here, as it
324
-
* prevents the algorithm pointer from being repeatedly reloaded.
325
-
*/
326
-
walk->blocksize = alg->base.cra_blocksize;
327
-
walk->ivsize = alg->co.ivsize;
328
-
walk->alignmask = alg->base.cra_alignmask;
319
+
walk->blocksize = crypto_skcipher_blocksize(tfm);
320
+
walk->ivsize = crypto_skcipher_ivsize(tfm);
321
+
walk->alignmask = crypto_skcipher_alignmask(tfm);
329
322
330
323
if (alg->co.base.cra_type != &crypto_skcipher_type)
331
324
walk->stride = alg->co.chunksize;
···
331
334
}
332
335
EXPORT_SYMBOL_GPL(skcipher_walk_virt);
333
336
334
-
static int skcipher_walk_aead_common(struct skcipher_walk *walk,
335
-
struct aead_request *req, bool atomic)
337
+
static int skcipher_walk_aead_common(struct skcipher_walk *__restrict walk,
338
+
struct aead_request *__restrict req,
339
+
bool atomic)
336
340
{
337
-
const struct aead_alg *alg = crypto_aead_alg(crypto_aead_reqtfm(req));
341
+
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
338
342
339
343
walk->nbytes = 0;
340
344
walk->iv = req->iv;
···
351
353
scatterwalk_start_at_pos(&walk->in, req->src, req->assoclen);
352
354
scatterwalk_start_at_pos(&walk->out, req->dst, req->assoclen);
353
355
354
-
/*
355
-
* Accessing 'alg' directly generates better code than using the
356
-
* crypto_aead_blocksize() and similar helper functions here, as it
357
-
* prevents the algorithm pointer from being repeatedly reloaded.
358
-
*/
359
-
walk->blocksize = alg->base.cra_blocksize;
360
-
walk->stride = alg->chunksize;
361
-
walk->ivsize = alg->ivsize;
362
-
walk->alignmask = alg->base.cra_alignmask;
356
+
walk->blocksize = crypto_aead_blocksize(tfm);
357
+
walk->stride = crypto_aead_chunksize(tfm);
358
+
walk->ivsize = crypto_aead_ivsize(tfm);
359
+
walk->alignmask = crypto_aead_alignmask(tfm);
363
360
364
361
return skcipher_walk_first(walk);
365
362
}
366
363
367
-
int skcipher_walk_aead_encrypt(struct skcipher_walk *walk,
368
-
struct aead_request *req, bool atomic)
364
+
int skcipher_walk_aead_encrypt(struct skcipher_walk *__restrict walk,
365
+
struct aead_request *__restrict req,
366
+
bool atomic)
369
367
{
370
368
walk->total = req->cryptlen;
371
369
···
369
375
}
370
376
EXPORT_SYMBOL_GPL(skcipher_walk_aead_encrypt);
371
377
372
-
int skcipher_walk_aead_decrypt(struct skcipher_walk *walk,
373
-
struct aead_request *req, bool atomic)
378
+
int skcipher_walk_aead_decrypt(struct skcipher_walk *__restrict walk,
379
+
struct aead_request *__restrict req,
380
+
bool atomic)
374
381
{
375
382
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
376
383
+8
-6
include/crypto/internal/skcipher.h
+8
-6
include/crypto/internal/skcipher.h
···
197
197
struct lskcipher_instance *inst);
198
198
199
199
int skcipher_walk_done(struct skcipher_walk *walk, int res);
200
-
int skcipher_walk_virt(struct skcipher_walk *walk,
201
-
struct skcipher_request *req,
200
+
int skcipher_walk_virt(struct skcipher_walk *__restrict walk,
201
+
struct skcipher_request *__restrict req,
202
202
bool atomic);
203
-
int skcipher_walk_aead_encrypt(struct skcipher_walk *walk,
204
-
struct aead_request *req, bool atomic);
205
-
int skcipher_walk_aead_decrypt(struct skcipher_walk *walk,
206
-
struct aead_request *req, bool atomic);
203
+
int skcipher_walk_aead_encrypt(struct skcipher_walk *__restrict walk,
204
+
struct aead_request *__restrict req,
205
+
bool atomic);
206
+
int skcipher_walk_aead_decrypt(struct skcipher_walk *__restrict walk,
207
+
struct aead_request *__restrict req,
208
+
bool atomic);
207
209
208
210
static inline void skcipher_walk_abort(struct skcipher_walk *walk)
209
211
{