Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: ahash - Add virtual address support
This patch adds virtual address support to ahash. Virtual addresses
were previously only supported through shash. The user may choose
to use virtual addresses with ahash by calling ahash_request_set_virt
instead of ahash_request_set_crypt.
The API will take care of translating this to an SG list if necessary,
unless the algorithm declares that it supports chaining. Therefore
in order for an ahash algorithm to support chaining, it must also
support virtual addresses directly.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
+293
-34
+251
-31
crypto/ahash.c
+251
-31
crypto/ahash.c
···
29
29
#define CRYPTO_ALG_TYPE_AHASH_MASK 0x0000000e
30
30
31
31
struct crypto_hash_walk {
32
-
char *data;
32
+
const char *data;
33
33
34
34
unsigned int offset;
35
35
unsigned int flags;
···
48
48
int (*op)(struct ahash_request *req);
49
49
crypto_completion_t compl;
50
50
void *data;
51
+
struct scatterlist sg;
52
+
const u8 *src;
53
+
u8 *page;
54
+
unsigned int offset;
55
+
unsigned int nbytes;
51
56
};
52
57
53
58
static void ahash_reqchain_done(void *data, int err);
54
59
static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt);
55
-
static void ahash_restore_req(struct ahash_request *req);
60
+
static void ahash_restore_req(struct ahash_save_req_state *state);
61
+
static void ahash_def_finup_done1(void *data, int err);
56
62
static int ahash_def_finup(struct ahash_request *req);
57
63
58
64
static int hash_walk_next(struct crypto_hash_walk *walk)
···
94
88
struct crypto_hash_walk *walk)
95
89
{
96
90
walk->total = req->nbytes;
91
+
walk->entrylen = 0;
97
92
98
-
if (!walk->total) {
99
-
walk->entrylen = 0;
93
+
if (!walk->total)
100
94
return 0;
95
+
96
+
walk->flags = req->base.flags;
97
+
98
+
if (ahash_request_isvirt(req)) {
99
+
walk->data = req->svirt;
100
+
walk->total = 0;
101
+
return req->nbytes;
101
102
}
102
103
103
104
walk->sg = req->src;
104
-
walk->flags = req->base.flags;
105
105
106
106
return hash_walk_new_entry(walk);
107
107
}
108
108
109
109
static int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err)
110
110
{
111
+
if ((walk->flags & CRYPTO_AHASH_REQ_VIRT))
112
+
return err;
113
+
111
114
walk->data -= walk->offset;
112
115
113
116
kunmap_local(walk->data);
···
202
187
struct scatterlist *sg;
203
188
unsigned int offset;
204
189
int err;
190
+
191
+
if (ahash_request_isvirt(req))
192
+
return crypto_shash_digest(desc, req->svirt, nbytes,
193
+
req->result);
205
194
206
195
if (nbytes &&
207
196
(sg = req->src, offset = sg->offset,
···
300
281
}
301
282
EXPORT_SYMBOL_GPL(crypto_ahash_setkey);
302
283
284
+
static bool ahash_request_hasvirt(struct ahash_request *req)
285
+
{
286
+
struct ahash_request *r2;
287
+
288
+
if (ahash_request_isvirt(req))
289
+
return true;
290
+
291
+
list_for_each_entry(r2, &req->base.list, base.list)
292
+
if (ahash_request_isvirt(r2))
293
+
return true;
294
+
295
+
return false;
296
+
}
297
+
298
+
static int ahash_reqchain_virt(struct ahash_save_req_state *state,
299
+
int err, u32 mask)
300
+
{
301
+
struct ahash_request *req = state->cur;
302
+
303
+
for (;;) {
304
+
unsigned len = state->nbytes;
305
+
306
+
req->base.err = err;
307
+
308
+
if (!state->offset)
309
+
break;
310
+
311
+
if (state->offset == len || err) {
312
+
u8 *result = req->result;
313
+
314
+
ahash_request_set_virt(req, state->src, result, len);
315
+
state->offset = 0;
316
+
break;
317
+
}
318
+
319
+
len -= state->offset;
320
+
321
+
len = min(PAGE_SIZE, len);
322
+
memcpy(state->page, state->src + state->offset, len);
323
+
state->offset += len;
324
+
req->nbytes = len;
325
+
326
+
err = state->op(req);
327
+
if (err == -EINPROGRESS) {
328
+
if (!list_empty(&state->head) ||
329
+
state->offset < state->nbytes)
330
+
err = -EBUSY;
331
+
break;
332
+
}
333
+
334
+
if (err == -EBUSY)
335
+
break;
336
+
}
337
+
338
+
return err;
339
+
}
340
+
303
341
static int ahash_reqchain_finish(struct ahash_save_req_state *state,
304
342
int err, u32 mask)
305
343
{
306
344
struct ahash_request *req0 = state->req0;
307
345
struct ahash_request *req = state->cur;
346
+
struct crypto_ahash *tfm;
308
347
struct ahash_request *n;
348
+
bool update;
309
349
310
-
req->base.err = err;
350
+
err = ahash_reqchain_virt(state, err, mask);
351
+
if (err == -EINPROGRESS || err == -EBUSY)
352
+
goto out;
311
353
312
354
if (req != req0)
313
355
list_add_tail(&req->base.list, &req0->base.list);
356
+
357
+
tfm = crypto_ahash_reqtfm(req);
358
+
update = state->op == crypto_ahash_alg(tfm)->update;
314
359
315
360
list_for_each_entry_safe(req, n, &state->head, base.list) {
316
361
list_del_init(&req->base.list);
···
383
300
req->base.complete = ahash_reqchain_done;
384
301
req->base.data = state;
385
302
state->cur = req;
303
+
304
+
if (update && ahash_request_isvirt(req) && req->nbytes) {
305
+
unsigned len = req->nbytes;
306
+
u8 *result = req->result;
307
+
308
+
state->src = req->svirt;
309
+
state->nbytes = len;
310
+
311
+
len = min(PAGE_SIZE, len);
312
+
313
+
memcpy(state->page, req->svirt, len);
314
+
state->offset = len;
315
+
316
+
ahash_request_set_crypt(req, &state->sg, result, len);
317
+
}
318
+
386
319
err = state->op(req);
387
320
388
321
if (err == -EINPROGRESS) {
389
-
if (!list_empty(&state->head))
322
+
if (!list_empty(&state->head) ||
323
+
state->offset < state->nbytes)
390
324
err = -EBUSY;
391
325
goto out;
392
326
}
···
411
311
if (err == -EBUSY)
412
312
goto out;
413
313
414
-
req->base.err = err;
314
+
err = ahash_reqchain_virt(state, err, mask);
315
+
if (err == -EINPROGRESS || err == -EBUSY)
316
+
goto out;
317
+
415
318
list_add_tail(&req->base.list, &req0->base.list);
416
319
}
417
320
418
-
ahash_restore_req(req0);
321
+
ahash_restore_req(state);
419
322
420
323
out:
421
324
return err;
···
432
329
data = state->data;
433
330
434
331
if (err == -EINPROGRESS) {
435
-
if (!list_empty(&state->head))
332
+
if (!list_empty(&state->head) || state->offset < state->nbytes)
436
333
return;
437
334
goto notify;
438
335
}
···
449
346
int (*op)(struct ahash_request *req))
450
347
{
451
348
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
349
+
bool update = op == crypto_ahash_alg(tfm)->update;
452
350
struct ahash_save_req_state *state;
453
351
struct ahash_save_req_state state0;
352
+
struct ahash_request *r2;
353
+
u8 *page = NULL;
454
354
int err;
455
355
456
-
if (!ahash_request_chained(req) || crypto_ahash_req_chain(tfm))
356
+
if (crypto_ahash_req_chain(tfm) ||
357
+
(!ahash_request_chained(req) &&
358
+
(!update || !ahash_request_isvirt(req))))
457
359
return op(req);
458
360
459
-
state = &state0;
361
+
if (update && ahash_request_hasvirt(req)) {
362
+
gfp_t gfp;
363
+
u32 flags;
460
364
365
+
flags = ahash_request_flags(req);
366
+
gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
367
+
GFP_KERNEL : GFP_ATOMIC;
368
+
page = (void *)__get_free_page(gfp);
369
+
err = -ENOMEM;
370
+
if (!page)
371
+
goto out_set_chain;
372
+
}
373
+
374
+
state = &state0;
461
375
if (ahash_is_async(tfm)) {
462
376
err = ahash_save_req(req, ahash_reqchain_done);
463
-
if (err) {
464
-
struct ahash_request *r2;
465
-
466
-
req->base.err = err;
467
-
list_for_each_entry(r2, &req->base.list, base.list)
468
-
r2->base.err = err;
469
-
470
-
return err;
471
-
}
377
+
if (err)
378
+
goto out_free_page;
472
379
473
380
state = req->base.data;
474
381
}
475
382
476
383
state->op = op;
477
384
state->cur = req;
385
+
state->page = page;
386
+
state->offset = 0;
387
+
state->nbytes = 0;
478
388
INIT_LIST_HEAD(&state->head);
479
389
list_splice_init(&req->base.list, &state->head);
390
+
391
+
if (page)
392
+
sg_init_one(&state->sg, page, PAGE_SIZE);
393
+
394
+
if (update && ahash_request_isvirt(req) && req->nbytes) {
395
+
unsigned len = req->nbytes;
396
+
u8 *result = req->result;
397
+
398
+
state->src = req->svirt;
399
+
state->nbytes = len;
400
+
401
+
len = min(PAGE_SIZE, len);
402
+
403
+
memcpy(page, req->svirt, len);
404
+
state->offset = len;
405
+
406
+
ahash_request_set_crypt(req, &state->sg, result, len);
407
+
}
480
408
481
409
err = op(req);
482
410
if (err == -EBUSY || err == -EINPROGRESS)
483
411
return -EBUSY;
484
412
485
413
return ahash_reqchain_finish(state, err, ~0);
414
+
415
+
out_free_page:
416
+
if (page) {
417
+
memset(page, 0, PAGE_SIZE);
418
+
free_page((unsigned long)page);
419
+
}
420
+
421
+
out_set_chain:
422
+
req->base.err = err;
423
+
list_for_each_entry(r2, &req->base.list, base.list)
424
+
r2->base.err = err;
425
+
426
+
return err;
486
427
}
487
428
488
429
int crypto_ahash_init(struct ahash_request *req)
···
578
431
req->base.complete = cplt;
579
432
req->base.data = state;
580
433
state->req0 = req;
434
+
state->page = NULL;
581
435
582
436
return 0;
583
437
}
584
438
585
-
static void ahash_restore_req(struct ahash_request *req)
439
+
static void ahash_restore_req(struct ahash_save_req_state *state)
586
440
{
587
-
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
588
-
struct ahash_save_req_state *state;
441
+
struct ahash_request *req = state->req0;
442
+
struct crypto_ahash *tfm;
589
443
444
+
free_page((unsigned long)state->page);
445
+
446
+
tfm = crypto_ahash_reqtfm(req);
590
447
if (!ahash_is_async(tfm))
591
448
return;
592
449
···
672
521
return err;
673
522
}
674
523
675
-
if (!crypto_ahash_alg(tfm)->finup)
524
+
if (!crypto_ahash_alg(tfm)->finup ||
525
+
(!crypto_ahash_req_chain(tfm) && ahash_request_hasvirt(req)))
676
526
return ahash_def_finup(req);
677
527
678
528
return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->finup);
679
529
}
680
530
EXPORT_SYMBOL_GPL(crypto_ahash_finup);
531
+
532
+
static int ahash_def_digest_finish(struct ahash_save_req_state *state, int err)
533
+
{
534
+
struct ahash_request *req = state->req0;
535
+
struct crypto_ahash *tfm;
536
+
537
+
if (err)
538
+
goto out;
539
+
540
+
tfm = crypto_ahash_reqtfm(req);
541
+
if (ahash_is_async(tfm))
542
+
req->base.complete = ahash_def_finup_done1;
543
+
544
+
err = crypto_ahash_update(req);
545
+
if (err == -EINPROGRESS || err == -EBUSY)
546
+
return err;
547
+
548
+
out:
549
+
ahash_restore_req(state);
550
+
return err;
551
+
}
552
+
553
+
static void ahash_def_digest_done(void *data, int err)
554
+
{
555
+
struct ahash_save_req_state *state0 = data;
556
+
struct ahash_save_req_state state;
557
+
struct ahash_request *areq;
558
+
559
+
state = *state0;
560
+
areq = state.req0;
561
+
if (err == -EINPROGRESS)
562
+
goto out;
563
+
564
+
areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
565
+
566
+
err = ahash_def_digest_finish(state0, err);
567
+
if (err == -EINPROGRESS || err == -EBUSY)
568
+
return;
569
+
570
+
out:
571
+
state.compl(state.data, err);
572
+
}
573
+
574
+
static int ahash_def_digest(struct ahash_request *req)
575
+
{
576
+
struct ahash_save_req_state *state;
577
+
int err;
578
+
579
+
err = ahash_save_req(req, ahash_def_digest_done);
580
+
if (err)
581
+
return err;
582
+
583
+
state = req->base.data;
584
+
585
+
err = crypto_ahash_init(req);
586
+
if (err == -EINPROGRESS || err == -EBUSY)
587
+
return err;
588
+
589
+
return ahash_def_digest_finish(state, err);
590
+
}
681
591
682
592
int crypto_ahash_digest(struct ahash_request *req)
683
593
{
···
761
549
return err;
762
550
}
763
551
552
+
if (!crypto_ahash_req_chain(tfm) && ahash_request_hasvirt(req))
553
+
return ahash_def_digest(req);
554
+
764
555
if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
765
556
return -ENOKEY;
766
557
···
779
564
if (err == -EINPROGRESS)
780
565
return;
781
566
782
-
ahash_restore_req(areq);
567
+
ahash_restore_req(state);
783
568
ahash_request_complete(areq, err);
784
569
}
785
570
786
-
static int ahash_def_finup_finish1(struct ahash_request *req, int err)
571
+
static int ahash_def_finup_finish1(struct ahash_save_req_state *state, int err)
787
572
{
788
-
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
573
+
struct ahash_request *req = state->req0;
574
+
struct crypto_ahash *tfm;
789
575
790
576
if (err)
791
577
goto out;
792
578
579
+
tfm = crypto_ahash_reqtfm(req);
793
580
if (ahash_is_async(tfm))
794
581
req->base.complete = ahash_def_finup_done2;
795
582
···
800
583
return err;
801
584
802
585
out:
803
-
ahash_restore_req(req);
586
+
ahash_restore_req(state);
804
587
return err;
805
588
}
806
589
···
817
600
818
601
areq->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
819
602
820
-
err = ahash_def_finup_finish1(areq, err);
603
+
err = ahash_def_finup_finish1(state0, err);
821
604
if (err == -EINPROGRESS || err == -EBUSY)
822
605
return;
823
606
···
827
610
828
611
static int ahash_def_finup(struct ahash_request *req)
829
612
{
613
+
struct ahash_save_req_state *state;
830
614
int err;
831
615
832
616
err = ahash_save_req(req, ahash_def_finup_done1);
833
617
if (err)
834
618
return err;
835
619
620
+
state = req->base.data;
621
+
836
622
err = crypto_ahash_update(req);
837
623
if (err == -EINPROGRESS || err == -EBUSY)
838
624
return err;
839
625
840
-
return ahash_def_finup_finish1(req, err);
626
+
return ahash_def_finup_finish1(state, err);
841
627
}
842
628
843
629
int crypto_ahash_export(struct ahash_request *req, void *out)
+36
-2
include/crypto/hash.h
+36
-2
include/crypto/hash.h
···
12
12
#include <linux/crypto.h>
13
13
#include <linux/string.h>
14
14
15
+
/* Set this bit for virtual address instead of SG list. */
16
+
#define CRYPTO_AHASH_REQ_VIRT 0x00000001
17
+
15
18
struct crypto_ahash;
16
19
17
20
/**
···
55
52
struct crypto_async_request base;
56
53
57
54
unsigned int nbytes;
58
-
struct scatterlist *src;
55
+
union {
56
+
struct scatterlist *src;
57
+
const u8 *svirt;
58
+
};
59
59
u8 *result;
60
60
61
61
void *__ctx[] CRYPTO_MINALIGN_ATTR;
···
616
610
crypto_completion_t compl,
617
611
void *data)
618
612
{
613
+
u32 keep = CRYPTO_AHASH_REQ_VIRT;
614
+
619
615
req->base.complete = compl;
620
616
req->base.data = data;
621
-
req->base.flags = flags;
617
+
flags &= ~keep;
618
+
req->base.flags &= keep;
619
+
req->base.flags |= flags;
622
620
crypto_reqchain_init(&req->base);
623
621
}
624
622
···
646
636
req->src = src;
647
637
req->nbytes = nbytes;
648
638
req->result = result;
639
+
req->base.flags &= ~CRYPTO_AHASH_REQ_VIRT;
640
+
}
641
+
642
+
/**
643
+
* ahash_request_set_virt() - set virtual address data buffers
644
+
* @req: ahash_request handle to be updated
645
+
* @src: source virtual address
646
+
* @result: buffer that is filled with the message digest -- the caller must
647
+
* ensure that the buffer has sufficient space by, for example, calling
648
+
* crypto_ahash_digestsize()
649
+
* @nbytes: number of bytes to process from the source virtual address
650
+
*
651
+
* By using this call, the caller references the source virtual address.
652
+
* The source virtual address points to the data the message digest is to
653
+
* be calculated for.
654
+
*/
655
+
static inline void ahash_request_set_virt(struct ahash_request *req,
656
+
const u8 *src, u8 *result,
657
+
unsigned int nbytes)
658
+
{
659
+
req->svirt = src;
660
+
req->nbytes = nbytes;
661
+
req->result = result;
662
+
req->base.flags |= CRYPTO_AHASH_REQ_VIRT;
649
663
}
650
664
651
665
static inline void ahash_request_chain(struct ahash_request *req,
+5
include/crypto/internal/hash.h
+5
include/crypto/internal/hash.h
···
252
252
return crypto_request_chained(&req->base);
253
253
}
254
254
255
+
static inline bool ahash_request_isvirt(struct ahash_request *req)
256
+
{
257
+
return req->base.flags & CRYPTO_AHASH_REQ_VIRT;
258
+
}
259
+
255
260
static inline bool crypto_ahash_req_chain(struct crypto_ahash *tfm)
256
261
{
257
262
return crypto_tfm_req_chain(&tfm->base);
+1
-1
include/linux/crypto.h
+1
-1
include/linux/crypto.h