Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: hisilicon/sec2 - implement full backlog mode for sec
This patch introduces a hierarchical backlog mechanism to cache
user data in high-throughput encryption/decryption scenarios,
the implementation addresses packet loss issues when hardware
queues overflow during peak loads.
First, we use sec_alloc_req_id to obtain an exclusive resource
from the pre-allocated resource pool of each queue, if no resource
is allocated, perform the DMA map operation on the request memory.
When the task is ready, we will attempt to send it to the hardware,
if the hardware queue is already full, we cache the request into
the backlog list, then return an EBUSY status to the upper layer
and instruct the packet-sending thread to pause transmission.
Simultaneously, when the hardware completes a task, it triggers
the sec callback function, within this function, reattempt to send
the requests from the backlog list and wake up the sending thread
until the hardware queue becomes fully occupied again.
In addition, it handles such exceptions like the hardware is reset
when packets are sent, it will switch to the software computing
and release occupied resources.
Signed-off-by: Wenkai Lin <linwenkai6@hisilicon.com>
Signed-off-by: Chenghai Huang <huangchenghai2@huawei.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
authored by
Wenkai Lin
and committed by
Herbert Xu
f0ae287c
f9c4923c
+461
-184
+56
-7
drivers/crypto/hisilicon/sec2/sec.h
+56
-7
drivers/crypto/hisilicon/sec2/sec.h
···
7
7
#include <linux/hisi_acc_qm.h>
8
8
#include "sec_crypto.h"
9
9
10
+
#define SEC_PBUF_SZ 512
11
+
#define SEC_MAX_MAC_LEN 64
12
+
#define SEC_IV_SIZE 24
13
+
#define SEC_SGE_NR_NUM 4
14
+
#define SEC_SGL_ALIGN_SIZE 64
15
+
10
16
/* Algorithm resource per hardware SEC queue */
11
17
struct sec_alg_res {
12
18
u8 *pbuf;
···
26
20
u16 depth;
27
21
};
28
22
23
+
struct sec_hw_sge {
24
+
dma_addr_t buf;
25
+
void *page_ctrl;
26
+
__le32 len;
27
+
__le32 pad;
28
+
__le32 pad0;
29
+
__le32 pad1;
30
+
};
31
+
32
+
struct sec_hw_sgl {
33
+
dma_addr_t next_dma;
34
+
__le16 entry_sum_in_chain;
35
+
__le16 entry_sum_in_sgl;
36
+
__le16 entry_length_in_sgl;
37
+
__le16 pad0;
38
+
__le64 pad1[5];
39
+
struct sec_hw_sgl *next;
40
+
struct sec_hw_sge sge_entries[SEC_SGE_NR_NUM];
41
+
} __aligned(SEC_SGL_ALIGN_SIZE);
42
+
43
+
struct sec_src_dst_buf {
44
+
struct sec_hw_sgl in;
45
+
struct sec_hw_sgl out;
46
+
};
47
+
48
+
struct sec_request_buf {
49
+
union {
50
+
struct sec_src_dst_buf data_buf;
51
+
__u8 pbuf[SEC_PBUF_SZ];
52
+
};
53
+
dma_addr_t in_dma;
54
+
dma_addr_t out_dma;
55
+
};
56
+
29
57
/* Cipher request of SEC private */
30
58
struct sec_cipher_req {
31
59
struct hisi_acc_hw_sgl *c_out;
···
69
29
struct skcipher_request *sk_req;
70
30
u32 c_len;
71
31
bool encrypt;
32
+
__u8 c_ivin_buf[SEC_IV_SIZE];
72
33
};
73
34
74
35
struct sec_aead_req {
···
78
37
u8 *a_ivin;
79
38
dma_addr_t a_ivin_dma;
80
39
struct aead_request *aead_req;
40
+
__u8 a_ivin_buf[SEC_IV_SIZE];
41
+
__u8 out_mac_buf[SEC_MAX_MAC_LEN];
42
+
};
43
+
44
+
struct sec_instance_backlog {
45
+
struct list_head list;
46
+
spinlock_t lock;
81
47
};
82
48
83
49
/* SEC request of Crypto */
···
103
55
dma_addr_t in_dma;
104
56
struct sec_cipher_req c_req;
105
57
struct sec_aead_req aead_req;
106
-
struct list_head backlog_head;
58
+
struct crypto_async_request *base;
107
59
108
60
int err_type;
109
61
int req_id;
110
62
u32 flag;
111
63
112
-
/* Status of the SEC request */
113
-
bool fake_busy;
114
64
bool use_pbuf;
65
+
66
+
struct list_head list;
67
+
struct sec_instance_backlog *backlog;
68
+
struct sec_request_buf buf;
115
69
};
116
70
117
71
/**
···
169
119
struct sec_alg_res *res;
170
120
struct sec_ctx *ctx;
171
121
spinlock_t req_lock;
172
-
struct list_head backlog;
122
+
spinlock_t id_lock;
173
123
struct hisi_acc_sgl_pool *c_in_pool;
174
124
struct hisi_acc_sgl_pool *c_out_pool;
125
+
struct sec_instance_backlog backlog;
126
+
u16 send_head;
175
127
};
176
128
177
129
enum sec_alg_type {
···
190
138
191
139
/* Half queues for encipher, and half for decipher */
192
140
u32 hlf_q_num;
193
-
194
-
/* Threshold for fake busy, trigger to return -EBUSY to user */
195
-
u32 fake_req_limit;
196
141
197
142
/* Current cyclic index to select a queue for encipher */
198
143
atomic_t enc_qcyclic;
+405
-177
drivers/crypto/hisilicon/sec2/sec_crypto.c
+405
-177
drivers/crypto/hisilicon/sec2/sec_crypto.c
···
67
67
#define SEC_MAX_CCM_AAD_LEN 65279
68
68
#define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth))
69
69
70
-
#define SEC_PBUF_SZ 512
71
70
#define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ
72
71
#define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE)
73
72
#define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \
···
101
102
#define IV_LAST_BYTE_MASK 0xFF
102
103
#define IV_CTR_INIT 0x1
103
104
#define IV_BYTE_OFFSET 0x8
105
+
#define SEC_GCM_MIN_AUTH_SZ 0x8
106
+
#define SEC_RETRY_MAX_CNT 5U
104
107
105
108
static DEFINE_MUTEX(sec_algs_lock);
106
109
static unsigned int sec_available_devs;
···
117
116
struct aead_alg alg;
118
117
};
119
118
120
-
/* Get an en/de-cipher queue cyclically to balance load over queues of TFM */
121
-
static inline u32 sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req)
122
-
{
123
-
if (req->c_req.encrypt)
124
-
return (u32)atomic_inc_return(&ctx->enc_qcyclic) %
125
-
ctx->hlf_q_num;
126
-
127
-
return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num +
128
-
ctx->hlf_q_num;
129
-
}
130
-
131
-
static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req)
132
-
{
133
-
if (req->c_req.encrypt)
134
-
atomic_dec(&ctx->enc_qcyclic);
135
-
else
136
-
atomic_dec(&ctx->dec_qcyclic);
137
-
}
119
+
static int sec_aead_soft_crypto(struct sec_ctx *ctx,
120
+
struct aead_request *aead_req,
121
+
bool encrypt);
122
+
static int sec_skcipher_soft_crypto(struct sec_ctx *ctx,
123
+
struct skcipher_request *sreq, bool encrypt);
138
124
139
125
static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx)
140
126
{
141
127
int req_id;
142
128
143
-
spin_lock_bh(&qp_ctx->req_lock);
129
+
spin_lock_bh(&qp_ctx->id_lock);
144
130
req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC);
145
-
spin_unlock_bh(&qp_ctx->req_lock);
146
-
if (unlikely(req_id < 0)) {
147
-
dev_err(req->ctx->dev, "alloc req id fail!\n");
148
-
return req_id;
149
-
}
150
-
151
-
req->qp_ctx = qp_ctx;
152
-
qp_ctx->req_list[req_id] = req;
153
-
131
+
spin_unlock_bh(&qp_ctx->id_lock);
154
132
return req_id;
155
133
}
156
134
···
143
163
return;
144
164
}
145
165
146
-
qp_ctx->req_list[req_id] = NULL;
147
-
req->qp_ctx = NULL;
148
-
149
-
spin_lock_bh(&qp_ctx->req_lock);
166
+
spin_lock_bh(&qp_ctx->id_lock);
150
167
idr_remove(&qp_ctx->req_idr, req_id);
151
-
spin_unlock_bh(&qp_ctx->req_lock);
168
+
spin_unlock_bh(&qp_ctx->id_lock);
152
169
}
153
170
154
171
static u8 pre_parse_finished_bd(struct bd_status *status, void *resp)
···
206
229
return 0;
207
230
}
208
231
232
+
static int qp_send_message(struct sec_req *req)
233
+
{
234
+
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
235
+
int ret;
236
+
237
+
if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1)
238
+
return -EBUSY;
239
+
240
+
spin_lock_bh(&qp_ctx->req_lock);
241
+
if (atomic_read(&qp_ctx->qp->qp_status.used) == qp_ctx->qp->sq_depth - 1) {
242
+
spin_unlock_bh(&qp_ctx->req_lock);
243
+
return -EBUSY;
244
+
}
245
+
246
+
if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2) {
247
+
req->sec_sqe.type2.tag = cpu_to_le16((u16)qp_ctx->send_head);
248
+
qp_ctx->req_list[qp_ctx->send_head] = req;
249
+
}
250
+
251
+
ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe);
252
+
if (ret) {
253
+
spin_unlock_bh(&qp_ctx->req_lock);
254
+
return ret;
255
+
}
256
+
if (qp_ctx->ctx->type_supported == SEC_BD_TYPE2)
257
+
qp_ctx->send_head = (qp_ctx->send_head + 1) % qp_ctx->qp->sq_depth;
258
+
259
+
spin_unlock_bh(&qp_ctx->req_lock);
260
+
261
+
atomic64_inc(&req->ctx->sec->debug.dfx.send_cnt);
262
+
return -EINPROGRESS;
263
+
}
264
+
265
+
static void sec_alg_send_backlog_soft(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
266
+
{
267
+
struct sec_req *req, *tmp;
268
+
int ret;
269
+
270
+
list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) {
271
+
list_del(&req->list);
272
+
ctx->req_op->buf_unmap(ctx, req);
273
+
if (req->req_id >= 0)
274
+
sec_free_req_id(req);
275
+
276
+
if (ctx->alg_type == SEC_AEAD)
277
+
ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req,
278
+
req->c_req.encrypt);
279
+
else
280
+
ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req,
281
+
req->c_req.encrypt);
282
+
283
+
/* Wake up the busy thread first, then return the errno. */
284
+
crypto_request_complete(req->base, -EINPROGRESS);
285
+
crypto_request_complete(req->base, ret);
286
+
}
287
+
}
288
+
289
+
static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx)
290
+
{
291
+
struct sec_req *req, *tmp;
292
+
int ret;
293
+
294
+
spin_lock_bh(&qp_ctx->backlog.lock);
295
+
list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) {
296
+
ret = qp_send_message(req);
297
+
switch (ret) {
298
+
case -EINPROGRESS:
299
+
list_del(&req->list);
300
+
crypto_request_complete(req->base, -EINPROGRESS);
301
+
break;
302
+
case -EBUSY:
303
+
/* Device is busy and stop send any request. */
304
+
goto unlock;
305
+
default:
306
+
/* Release memory resources and send all requests through software. */
307
+
sec_alg_send_backlog_soft(ctx, qp_ctx);
308
+
goto unlock;
309
+
}
310
+
}
311
+
312
+
unlock:
313
+
spin_unlock_bh(&qp_ctx->backlog.lock);
314
+
}
315
+
209
316
static void sec_req_cb(struct hisi_qp *qp, void *resp)
210
317
{
211
318
struct sec_qp_ctx *qp_ctx = qp->qp_ctx;
···
334
273
ctx->req_op->callback(ctx, req, err);
335
274
}
336
275
337
-
static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req)
276
+
static int sec_alg_send_message_retry(struct sec_req *req)
338
277
{
339
-
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
278
+
int ctr = 0;
340
279
int ret;
341
280
342
-
if (ctx->fake_req_limit <=
343
-
atomic_read(&qp_ctx->qp->qp_status.used) &&
344
-
!(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG))
345
-
return -EBUSY;
346
-
347
-
spin_lock_bh(&qp_ctx->req_lock);
348
-
ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe);
349
-
if (ctx->fake_req_limit <=
350
-
atomic_read(&qp_ctx->qp->qp_status.used) && !ret) {
351
-
list_add_tail(&req->backlog_head, &qp_ctx->backlog);
352
-
atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
353
-
atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt);
354
-
spin_unlock_bh(&qp_ctx->req_lock);
355
-
return -EBUSY;
356
-
}
357
-
spin_unlock_bh(&qp_ctx->req_lock);
358
-
359
-
if (unlikely(ret == -EBUSY))
360
-
return -ENOBUFS;
361
-
362
-
if (likely(!ret)) {
363
-
ret = -EINPROGRESS;
364
-
atomic64_inc(&ctx->sec->debug.dfx.send_cnt);
365
-
}
281
+
do {
282
+
ret = qp_send_message(req);
283
+
} while (ret == -EBUSY && ctr++ < SEC_RETRY_MAX_CNT);
366
284
367
285
return ret;
368
286
}
369
287
370
-
/* Get DMA memory resources */
288
+
static int sec_alg_try_enqueue(struct sec_req *req)
289
+
{
290
+
/* Check if any request is already backlogged */
291
+
if (!list_empty(&req->backlog->list))
292
+
return -EBUSY;
293
+
294
+
/* Try to enqueue to HW ring */
295
+
return qp_send_message(req);
296
+
}
297
+
298
+
299
+
static int sec_alg_send_message_maybacklog(struct sec_req *req)
300
+
{
301
+
int ret;
302
+
303
+
ret = sec_alg_try_enqueue(req);
304
+
if (ret != -EBUSY)
305
+
return ret;
306
+
307
+
spin_lock_bh(&req->backlog->lock);
308
+
ret = sec_alg_try_enqueue(req);
309
+
if (ret == -EBUSY)
310
+
list_add_tail(&req->list, &req->backlog->list);
311
+
spin_unlock_bh(&req->backlog->lock);
312
+
313
+
return ret;
314
+
}
315
+
316
+
static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req)
317
+
{
318
+
if (req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)
319
+
return sec_alg_send_message_maybacklog(req);
320
+
321
+
return sec_alg_send_message_retry(req);
322
+
}
323
+
371
324
static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res)
372
325
{
373
326
u16 q_depth = res->depth;
···
633
558
634
559
spin_lock_init(&qp_ctx->req_lock);
635
560
idr_init(&qp_ctx->req_idr);
636
-
INIT_LIST_HEAD(&qp_ctx->backlog);
561
+
spin_lock_init(&qp_ctx->backlog.lock);
562
+
spin_lock_init(&qp_ctx->id_lock);
563
+
INIT_LIST_HEAD(&qp_ctx->backlog.list);
564
+
qp_ctx->send_head = 0;
637
565
638
566
ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx);
639
567
if (ret)
···
680
602
ctx->hlf_q_num = sec->ctx_q_num >> 1;
681
603
682
604
ctx->pbuf_supported = ctx->sec->iommu_used;
683
-
684
-
/* Half of queue depth is taken as fake requests limit in the queue. */
685
-
ctx->fake_req_limit = ctx->qps[0]->sq_depth >> 1;
686
605
ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx),
687
606
GFP_KERNEL);
688
607
if (!ctx->qp_ctx) {
···
781
706
int ret;
782
707
783
708
ctx->alg_type = SEC_SKCIPHER;
784
-
crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req));
709
+
crypto_skcipher_set_reqsize_dma(tfm, sizeof(struct sec_req));
785
710
ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm);
786
711
if (ctx->c_ctx.ivsize > SEC_IV_SIZE) {
787
712
pr_err("get error skcipher iv size!\n");
···
958
883
static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req,
959
884
struct scatterlist *src)
960
885
{
961
-
struct sec_aead_req *a_req = &req->aead_req;
962
-
struct aead_request *aead_req = a_req->aead_req;
886
+
struct aead_request *aead_req = req->aead_req.aead_req;
963
887
struct sec_cipher_req *c_req = &req->c_req;
964
888
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
889
+
struct sec_request_buf *buf = &req->buf;
965
890
struct device *dev = ctx->dev;
966
891
int copy_size, pbuf_length;
967
892
int req_id = req->req_id;
968
893
struct crypto_aead *tfm;
894
+
u8 *mac_offset, *pbuf;
969
895
size_t authsize;
970
-
u8 *mac_offset;
971
896
972
897
if (ctx->alg_type == SEC_AEAD)
973
898
copy_size = aead_req->cryptlen + aead_req->assoclen;
974
899
else
975
900
copy_size = c_req->c_len;
976
901
977
-
pbuf_length = sg_copy_to_buffer(src, sg_nents(src),
978
-
qp_ctx->res[req_id].pbuf, copy_size);
902
+
903
+
pbuf = req->req_id < 0 ? buf->pbuf : qp_ctx->res[req_id].pbuf;
904
+
pbuf_length = sg_copy_to_buffer(src, sg_nents(src), pbuf, copy_size);
979
905
if (unlikely(pbuf_length != copy_size)) {
980
906
dev_err(dev, "copy src data to pbuf error!\n");
981
907
return -EINVAL;
···
984
908
if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) {
985
909
tfm = crypto_aead_reqtfm(aead_req);
986
910
authsize = crypto_aead_authsize(tfm);
987
-
mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize;
988
-
memcpy(a_req->out_mac, mac_offset, authsize);
911
+
mac_offset = pbuf + copy_size - authsize;
912
+
memcpy(req->aead_req.out_mac, mac_offset, authsize);
913
+
}
914
+
915
+
if (req->req_id < 0) {
916
+
buf->in_dma = dma_map_single(dev, buf->pbuf, SEC_PBUF_SZ, DMA_BIDIRECTIONAL);
917
+
if (unlikely(dma_mapping_error(dev, buf->in_dma)))
918
+
return -ENOMEM;
919
+
920
+
buf->out_dma = buf->in_dma;
921
+
return 0;
989
922
}
990
923
991
924
req->in_dma = qp_ctx->res[req_id].pbuf_dma;
···
1009
924
struct aead_request *aead_req = req->aead_req.aead_req;
1010
925
struct sec_cipher_req *c_req = &req->c_req;
1011
926
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
927
+
struct sec_request_buf *buf = &req->buf;
1012
928
int copy_size, pbuf_length;
1013
929
int req_id = req->req_id;
1014
930
···
1018
932
else
1019
933
copy_size = c_req->c_len;
1020
934
1021
-
pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst),
1022
-
qp_ctx->res[req_id].pbuf, copy_size);
935
+
if (req->req_id < 0)
936
+
pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), buf->pbuf, copy_size);
937
+
else
938
+
pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), qp_ctx->res[req_id].pbuf,
939
+
copy_size);
1023
940
if (unlikely(pbuf_length != copy_size))
1024
941
dev_err(ctx->dev, "copy pbuf data to dst error!\n");
942
+
943
+
if (req->req_id < 0)
944
+
dma_unmap_single(ctx->dev, buf->in_dma, SEC_PBUF_SZ, DMA_BIDIRECTIONAL);
1025
945
}
1026
946
1027
947
static int sec_aead_mac_init(struct sec_aead_req *req)
···
1049
957
return 0;
1050
958
}
1051
959
1052
-
static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req,
1053
-
struct scatterlist *src, struct scatterlist *dst)
960
+
static void fill_sg_to_hw_sge(struct scatterlist *sgl, struct sec_hw_sge *hw_sge)
961
+
{
962
+
hw_sge->buf = sg_dma_address(sgl);
963
+
hw_sge->len = cpu_to_le32(sg_dma_len(sgl));
964
+
hw_sge->page_ctrl = sg_virt(sgl);
965
+
}
966
+
967
+
static int sec_cipher_to_hw_sgl(struct device *dev, struct scatterlist *src,
968
+
struct sec_hw_sgl *src_in, dma_addr_t *hw_sgl_dma,
969
+
int dma_dir)
970
+
{
971
+
struct sec_hw_sge *curr_hw_sge = src_in->sge_entries;
972
+
u32 i, sg_n, sg_n_mapped;
973
+
struct scatterlist *sg;
974
+
u32 sge_var = 0;
975
+
976
+
sg_n = sg_nents(src);
977
+
sg_n_mapped = dma_map_sg(dev, src, sg_n, dma_dir);
978
+
if (unlikely(!sg_n_mapped)) {
979
+
dev_err(dev, "dma mapping for SG error!\n");
980
+
return -EINVAL;
981
+
} else if (unlikely(sg_n_mapped > SEC_SGE_NR_NUM)) {
982
+
dev_err(dev, "the number of entries in input scatterlist error!\n");
983
+
dma_unmap_sg(dev, src, sg_n, dma_dir);
984
+
return -EINVAL;
985
+
}
986
+
987
+
for_each_sg(src, sg, sg_n_mapped, i) {
988
+
fill_sg_to_hw_sge(sg, curr_hw_sge);
989
+
curr_hw_sge++;
990
+
sge_var++;
991
+
}
992
+
993
+
src_in->entry_sum_in_sgl = cpu_to_le16(sge_var);
994
+
src_in->entry_sum_in_chain = cpu_to_le16(SEC_SGE_NR_NUM);
995
+
src_in->entry_length_in_sgl = cpu_to_le16(SEC_SGE_NR_NUM);
996
+
*hw_sgl_dma = dma_map_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir);
997
+
if (unlikely(dma_mapping_error(dev, *hw_sgl_dma))) {
998
+
dma_unmap_sg(dev, src, sg_n, dma_dir);
999
+
return -ENOMEM;
1000
+
}
1001
+
1002
+
return 0;
1003
+
}
1004
+
1005
+
static void sec_cipher_put_hw_sgl(struct device *dev, struct scatterlist *src,
1006
+
dma_addr_t src_in, int dma_dir)
1007
+
{
1008
+
dma_unmap_single(dev, src_in, sizeof(struct sec_hw_sgl), dma_dir);
1009
+
dma_unmap_sg(dev, src, sg_nents(src), dma_dir);
1010
+
}
1011
+
1012
+
static int sec_cipher_map_sgl(struct device *dev, struct sec_req *req,
1013
+
struct scatterlist *src, struct scatterlist *dst)
1014
+
{
1015
+
struct sec_hw_sgl *src_in = &req->buf.data_buf.in;
1016
+
struct sec_hw_sgl *dst_out = &req->buf.data_buf.out;
1017
+
int ret;
1018
+
1019
+
if (dst == src) {
1020
+
ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma,
1021
+
DMA_BIDIRECTIONAL);
1022
+
req->buf.out_dma = req->buf.in_dma;
1023
+
return ret;
1024
+
}
1025
+
1026
+
ret = sec_cipher_to_hw_sgl(dev, src, src_in, &req->buf.in_dma, DMA_TO_DEVICE);
1027
+
if (unlikely(ret))
1028
+
return ret;
1029
+
1030
+
ret = sec_cipher_to_hw_sgl(dev, dst, dst_out, &req->buf.out_dma,
1031
+
DMA_FROM_DEVICE);
1032
+
if (unlikely(ret)) {
1033
+
sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE);
1034
+
return ret;
1035
+
}
1036
+
1037
+
return 0;
1038
+
}
1039
+
1040
+
static int sec_cipher_map_inner(struct sec_ctx *ctx, struct sec_req *req,
1041
+
struct scatterlist *src, struct scatterlist *dst)
1054
1042
{
1055
1043
struct sec_cipher_req *c_req = &req->c_req;
1056
1044
struct sec_aead_req *a_req = &req->aead_req;
···
1150
978
a_req->out_mac_dma = res->pbuf_dma +
1151
979
SEC_PBUF_MAC_OFFSET;
1152
980
}
1153
-
ret = sec_cipher_pbuf_map(ctx, req, src);
1154
-
1155
-
return ret;
981
+
return sec_cipher_pbuf_map(ctx, req, src);
1156
982
}
983
+
1157
984
c_req->c_ivin = res->c_ivin;
1158
985
c_req->c_ivin_dma = res->c_ivin_dma;
1159
986
if (ctx->alg_type == SEC_AEAD) {
···
1201
1030
return 0;
1202
1031
}
1203
1032
1033
+
static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req,
1034
+
struct scatterlist *src, struct scatterlist *dst)
1035
+
{
1036
+
struct sec_aead_req *a_req = &req->aead_req;
1037
+
struct sec_cipher_req *c_req = &req->c_req;
1038
+
bool is_aead = (ctx->alg_type == SEC_AEAD);
1039
+
struct device *dev = ctx->dev;
1040
+
int ret = -ENOMEM;
1041
+
1042
+
if (req->req_id >= 0)
1043
+
return sec_cipher_map_inner(ctx, req, src, dst);
1044
+
1045
+
c_req->c_ivin = c_req->c_ivin_buf;
1046
+
c_req->c_ivin_dma = dma_map_single(dev, c_req->c_ivin,
1047
+
SEC_IV_SIZE, DMA_TO_DEVICE);
1048
+
if (unlikely(dma_mapping_error(dev, c_req->c_ivin_dma)))
1049
+
return -ENOMEM;
1050
+
1051
+
if (is_aead) {
1052
+
a_req->a_ivin = a_req->a_ivin_buf;
1053
+
a_req->out_mac = a_req->out_mac_buf;
1054
+
a_req->a_ivin_dma = dma_map_single(dev, a_req->a_ivin,
1055
+
SEC_IV_SIZE, DMA_TO_DEVICE);
1056
+
if (unlikely(dma_mapping_error(dev, a_req->a_ivin_dma)))
1057
+
goto free_c_ivin_dma;
1058
+
1059
+
a_req->out_mac_dma = dma_map_single(dev, a_req->out_mac,
1060
+
SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL);
1061
+
if (unlikely(dma_mapping_error(dev, a_req->out_mac_dma)))
1062
+
goto free_a_ivin_dma;
1063
+
}
1064
+
if (req->use_pbuf) {
1065
+
ret = sec_cipher_pbuf_map(ctx, req, src);
1066
+
if (unlikely(ret))
1067
+
goto free_out_mac_dma;
1068
+
1069
+
return 0;
1070
+
}
1071
+
1072
+
if (!c_req->encrypt && is_aead) {
1073
+
ret = sec_aead_mac_init(a_req);
1074
+
if (unlikely(ret)) {
1075
+
dev_err(dev, "fail to init mac data for ICV!\n");
1076
+
goto free_out_mac_dma;
1077
+
}
1078
+
}
1079
+
1080
+
ret = sec_cipher_map_sgl(dev, req, src, dst);
1081
+
if (unlikely(ret)) {
1082
+
dev_err(dev, "fail to dma map input sgl buffers!\n");
1083
+
goto free_out_mac_dma;
1084
+
}
1085
+
1086
+
return 0;
1087
+
1088
+
free_out_mac_dma:
1089
+
if (is_aead)
1090
+
dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL);
1091
+
free_a_ivin_dma:
1092
+
if (is_aead)
1093
+
dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE);
1094
+
free_c_ivin_dma:
1095
+
dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE);
1096
+
return ret;
1097
+
}
1098
+
1204
1099
static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req,
1205
1100
struct scatterlist *src, struct scatterlist *dst)
1206
1101
{
1102
+
struct sec_aead_req *a_req = &req->aead_req;
1207
1103
struct sec_cipher_req *c_req = &req->c_req;
1208
1104
struct device *dev = ctx->dev;
1105
+
1106
+
if (req->req_id >= 0) {
1107
+
if (req->use_pbuf) {
1108
+
sec_cipher_pbuf_unmap(ctx, req, dst);
1109
+
} else {
1110
+
if (dst != src) {
1111
+
hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out, DMA_FROM_DEVICE);
1112
+
hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_TO_DEVICE);
1113
+
} else {
1114
+
hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_BIDIRECTIONAL);
1115
+
}
1116
+
}
1117
+
return;
1118
+
}
1209
1119
1210
1120
if (req->use_pbuf) {
1211
1121
sec_cipher_pbuf_unmap(ctx, req, dst);
1212
1122
} else {
1213
1123
if (dst != src) {
1214
-
hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out, DMA_FROM_DEVICE);
1215
-
hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_TO_DEVICE);
1124
+
sec_cipher_put_hw_sgl(dev, dst, req->buf.out_dma, DMA_FROM_DEVICE);
1125
+
sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_TO_DEVICE);
1216
1126
} else {
1217
-
hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_BIDIRECTIONAL);
1127
+
sec_cipher_put_hw_sgl(dev, src, req->buf.in_dma, DMA_BIDIRECTIONAL);
1218
1128
}
1129
+
}
1130
+
1131
+
dma_unmap_single(dev, c_req->c_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE);
1132
+
if (ctx->alg_type == SEC_AEAD) {
1133
+
dma_unmap_single(dev, a_req->a_ivin_dma, SEC_IV_SIZE, DMA_TO_DEVICE);
1134
+
dma_unmap_single(dev, a_req->out_mac_dma, SEC_MAX_MAC_LEN, DMA_BIDIRECTIONAL);
1219
1135
}
1220
1136
}
1221
1137
···
1520
1262
1521
1263
sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
1522
1264
sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
1523
-
sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma);
1524
-
sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1265
+
if (req->req_id < 0) {
1266
+
sec_sqe->type2.data_src_addr = cpu_to_le64(req->buf.in_dma);
1267
+
sec_sqe->type2.data_dst_addr = cpu_to_le64(req->buf.out_dma);
1268
+
} else {
1269
+
sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma);
1270
+
sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1271
+
}
1272
+
if (sec_sqe->type2.data_src_addr != sec_sqe->type2.data_dst_addr)
1273
+
de = 0x1 << SEC_DE_OFFSET;
1525
1274
1526
1275
sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) <<
1527
1276
SEC_CMODE_OFFSET);
···
1554
1289
1555
1290
sec_sqe->sdm_addr_type |= da_type;
1556
1291
scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET;
1557
-
if (req->in_dma != c_req->c_out_dma)
1558
-
de = 0x1 << SEC_DE_OFFSET;
1559
1292
1560
1293
sec_sqe->sds_sa_type = (de | scene | sa_type);
1561
1294
1562
1295
sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len);
1563
-
sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id);
1564
1296
1565
1297
return 0;
1566
1298
}
···
1574
1312
1575
1313
sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma);
1576
1314
sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma);
1577
-
sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma);
1578
-
sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1315
+
if (req->req_id < 0) {
1316
+
sec_sqe3->data_src_addr = cpu_to_le64(req->buf.in_dma);
1317
+
sec_sqe3->data_dst_addr = cpu_to_le64(req->buf.out_dma);
1318
+
} else {
1319
+
sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma);
1320
+
sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma);
1321
+
}
1322
+
if (sec_sqe3->data_src_addr != sec_sqe3->data_dst_addr)
1323
+
bd_param |= 0x1 << SEC_DE_OFFSET_V3;
1579
1324
1580
1325
sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) |
1581
1326
c_ctx->c_mode;
···
1608
1339
}
1609
1340
1610
1341
bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3;
1611
-
if (req->in_dma != c_req->c_out_dma)
1612
-
bd_param |= 0x1 << SEC_DE_OFFSET_V3;
1613
1342
1614
1343
bd_param |= SEC_BD_TYPE3;
1615
1344
sec_sqe3->bd_param = cpu_to_le32(bd_param);
···
1639
1372
size_t sz;
1640
1373
u8 *iv;
1641
1374
1642
-
if (req->c_req.encrypt)
1643
-
sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst;
1644
-
else
1645
-
sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src;
1646
-
1647
1375
if (alg_type == SEC_SKCIPHER) {
1376
+
sgl = req->c_req.encrypt ? sk_req->dst : sk_req->src;
1648
1377
iv = sk_req->iv;
1649
1378
cryptlen = sk_req->cryptlen;
1650
1379
} else {
1380
+
sgl = req->c_req.encrypt ? aead_req->dst : aead_req->src;
1651
1381
iv = aead_req->iv;
1652
1382
cryptlen = aead_req->cryptlen;
1653
1383
}
···
1655
1391
if (unlikely(sz != iv_size))
1656
1392
dev_err(req->ctx->dev, "copy output iv error!\n");
1657
1393
} else {
1658
-
sz = cryptlen / iv_size;
1659
-
if (cryptlen % iv_size)
1660
-
sz += 1;
1394
+
sz = (cryptlen + iv_size - 1) / iv_size;
1661
1395
ctr_iv_inc(iv, iv_size, sz);
1662
1396
}
1663
-
}
1664
-
1665
-
static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx,
1666
-
struct sec_qp_ctx *qp_ctx)
1667
-
{
1668
-
struct sec_req *backlog_req = NULL;
1669
-
1670
-
spin_lock_bh(&qp_ctx->req_lock);
1671
-
if (ctx->fake_req_limit >=
1672
-
atomic_read(&qp_ctx->qp->qp_status.used) &&
1673
-
!list_empty(&qp_ctx->backlog)) {
1674
-
backlog_req = list_first_entry(&qp_ctx->backlog,
1675
-
typeof(*backlog_req), backlog_head);
1676
-
list_del(&backlog_req->backlog_head);
1677
-
}
1678
-
spin_unlock_bh(&qp_ctx->req_lock);
1679
-
1680
-
return backlog_req;
1681
1397
}
1682
1398
1683
1399
static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req,
1684
1400
int err)
1685
1401
{
1686
-
struct skcipher_request *sk_req = req->c_req.sk_req;
1687
1402
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1688
-
struct skcipher_request *backlog_sk_req;
1689
-
struct sec_req *backlog_req;
1690
1403
1691
-
sec_free_req_id(req);
1404
+
if (req->req_id >= 0)
1405
+
sec_free_req_id(req);
1692
1406
1693
1407
/* IV output at encrypto of CBC/CTR mode */
1694
1408
if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC ||
1695
1409
ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt)
1696
1410
sec_update_iv(req, SEC_SKCIPHER);
1697
1411
1698
-
while (1) {
1699
-
backlog_req = sec_back_req_clear(ctx, qp_ctx);
1700
-
if (!backlog_req)
1701
-
break;
1702
-
1703
-
backlog_sk_req = backlog_req->c_req.sk_req;
1704
-
skcipher_request_complete(backlog_sk_req, -EINPROGRESS);
1705
-
atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt);
1706
-
}
1707
-
1708
-
skcipher_request_complete(sk_req, err);
1412
+
crypto_request_complete(req->base, err);
1413
+
sec_alg_send_backlog(ctx, qp_ctx);
1709
1414
}
1710
1415
1711
1416
static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req)
···
1913
1680
struct aead_request *a_req = req->aead_req.aead_req;
1914
1681
struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
1915
1682
size_t authsize = crypto_aead_authsize(tfm);
1916
-
struct sec_aead_req *aead_req = &req->aead_req;
1917
-
struct sec_cipher_req *c_req = &req->c_req;
1918
1683
struct sec_qp_ctx *qp_ctx = req->qp_ctx;
1919
-
struct aead_request *backlog_aead_req;
1920
-
struct sec_req *backlog_req;
1921
1684
size_t sz;
1922
1685
1923
-
if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt)
1924
-
sec_update_iv(req, SEC_AEAD);
1686
+
if (!err && req->c_req.encrypt) {
1687
+
if (c->c_ctx.c_mode == SEC_CMODE_CBC)
1688
+
sec_update_iv(req, SEC_AEAD);
1925
1689
1926
-
/* Copy output mac */
1927
-
if (!err && c_req->encrypt) {
1928
-
struct scatterlist *sgl = a_req->dst;
1929
-
1930
-
sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), aead_req->out_mac,
1690
+
sz = sg_pcopy_from_buffer(a_req->dst, sg_nents(a_req->dst), req->aead_req.out_mac,
1931
1691
authsize, a_req->cryptlen + a_req->assoclen);
1932
1692
if (unlikely(sz != authsize)) {
1933
1693
dev_err(c->dev, "copy out mac err!\n");
···
1928
1702
}
1929
1703
}
1930
1704
1931
-
sec_free_req_id(req);
1705
+
if (req->req_id >= 0)
1706
+
sec_free_req_id(req);
1932
1707
1933
-
while (1) {
1934
-
backlog_req = sec_back_req_clear(c, qp_ctx);
1935
-
if (!backlog_req)
1936
-
break;
1937
-
1938
-
backlog_aead_req = backlog_req->aead_req.aead_req;
1939
-
aead_request_complete(backlog_aead_req, -EINPROGRESS);
1940
-
atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt);
1941
-
}
1942
-
1943
-
aead_request_complete(a_req, err);
1708
+
crypto_request_complete(req->base, err);
1709
+
sec_alg_send_backlog(c, qp_ctx);
1944
1710
}
1945
1711
1946
-
static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req)
1712
+
static void sec_request_uninit(struct sec_req *req)
1947
1713
{
1948
-
sec_free_req_id(req);
1949
-
sec_free_queue_id(ctx, req);
1714
+
if (req->req_id >= 0)
1715
+
sec_free_req_id(req);
1950
1716
}
1951
1717
1952
1718
static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req)
1953
1719
{
1954
1720
struct sec_qp_ctx *qp_ctx;
1955
-
int queue_id;
1721
+
int i;
1956
1722
1957
-
/* To load balance */
1958
-
queue_id = sec_alloc_queue_id(ctx, req);
1959
-
qp_ctx = &ctx->qp_ctx[queue_id];
1960
-
1961
-
req->req_id = sec_alloc_req_id(req, qp_ctx);
1962
-
if (unlikely(req->req_id < 0)) {
1963
-
sec_free_queue_id(ctx, req);
1964
-
return req->req_id;
1723
+
for (i = 0; i < ctx->sec->ctx_q_num; i++) {
1724
+
qp_ctx = &ctx->qp_ctx[i];
1725
+
req->req_id = sec_alloc_req_id(req, qp_ctx);
1726
+
if (req->req_id >= 0)
1727
+
break;
1965
1728
}
1729
+
1730
+
req->qp_ctx = qp_ctx;
1731
+
req->backlog = &qp_ctx->backlog;
1966
1732
1967
1733
return 0;
1968
1734
}
1969
1735
1970
1736
static int sec_process(struct sec_ctx *ctx, struct sec_req *req)
1971
1737
{
1972
-
struct sec_cipher_req *c_req = &req->c_req;
1973
1738
int ret;
1974
1739
1975
1740
ret = sec_request_init(ctx, req);
···
1977
1760
sec_update_iv(req, ctx->alg_type);
1978
1761
1979
1762
ret = ctx->req_op->bd_send(ctx, req);
1980
-
if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) ||
1981
-
(ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) {
1763
+
if (unlikely((ret != -EBUSY && ret != -EINPROGRESS))) {
1982
1764
dev_err_ratelimited(ctx->dev, "send sec request failed!\n");
1983
1765
goto err_send_req;
1984
1766
}
···
1988
1772
/* As failing, restore the IV from user */
1989
1773
if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) {
1990
1774
if (ctx->alg_type == SEC_SKCIPHER)
1991
-
memcpy(req->c_req.sk_req->iv, c_req->c_ivin,
1775
+
memcpy(req->c_req.sk_req->iv, req->c_req.c_ivin,
1992
1776
ctx->c_ctx.ivsize);
1993
1777
else
1994
-
memcpy(req->aead_req.aead_req->iv, c_req->c_ivin,
1778
+
memcpy(req->aead_req.aead_req->iv, req->c_req.c_ivin,
1995
1779
ctx->c_ctx.ivsize);
1996
1780
}
1997
1781
1998
1782
sec_request_untransfer(ctx, req);
1783
+
1999
1784
err_uninit_req:
2000
-
sec_request_uninit(ctx, req);
1785
+
sec_request_uninit(req);
1786
+
if (ctx->alg_type == SEC_AEAD)
1787
+
ret = sec_aead_soft_crypto(ctx, req->aead_req.aead_req,
1788
+
req->c_req.encrypt);
1789
+
else
1790
+
ret = sec_skcipher_soft_crypto(ctx, req->c_req.sk_req,
1791
+
req->c_req.encrypt);
2001
1792
return ret;
2002
1793
}
2003
1794
···
2078
1855
struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2079
1856
int ret;
2080
1857
2081
-
crypto_aead_set_reqsize(tfm, sizeof(struct sec_req));
1858
+
crypto_aead_set_reqsize_dma(tfm, sizeof(struct sec_req));
2082
1859
ctx->alg_type = SEC_AEAD;
2083
1860
ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm);
2084
1861
if (ctx->c_ctx.ivsize < SEC_AIV_SIZE ||
···
2315
2092
static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt)
2316
2093
{
2317
2094
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req);
2318
-
struct sec_req *req = skcipher_request_ctx(sk_req);
2095
+
struct sec_req *req = skcipher_request_ctx_dma(sk_req);
2319
2096
struct sec_ctx *ctx = crypto_skcipher_ctx(tfm);
2320
2097
bool need_fallback = false;
2321
2098
int ret;
···
2330
2107
req->c_req.sk_req = sk_req;
2331
2108
req->c_req.encrypt = encrypt;
2332
2109
req->ctx = ctx;
2110
+
req->base = &sk_req->base;
2333
2111
2334
2112
ret = sec_skcipher_param_check(ctx, req, &need_fallback);
2335
2113
if (unlikely(ret))
···
2465
2241
return -EINVAL;
2466
2242
if (unlikely(ctx->a_ctx.a_key_len & WORD_MASK))
2467
2243
return -EINVAL;
2244
+
} else if (c_mode == SEC_CMODE_GCM) {
2245
+
if (unlikely(sz < SEC_GCM_MIN_AUTH_SZ))
2246
+
return -EINVAL;
2468
2247
}
2469
2248
2470
2249
return 0;
···
2541
2314
static int sec_aead_crypto(struct aead_request *a_req, bool encrypt)
2542
2315
{
2543
2316
struct crypto_aead *tfm = crypto_aead_reqtfm(a_req);
2544
-
struct sec_req *req = aead_request_ctx(a_req);
2317
+
struct sec_req *req = aead_request_ctx_dma(a_req);
2545
2318
struct sec_ctx *ctx = crypto_aead_ctx(tfm);
2546
2319
size_t sz = crypto_aead_authsize(tfm);
2547
2320
bool need_fallback = false;
···
2551
2324
req->aead_req.aead_req = a_req;
2552
2325
req->c_req.encrypt = encrypt;
2553
2326
req->ctx = ctx;
2327
+
req->base = &a_req->base;
2554
2328
req->c_req.c_len = a_req->cryptlen - (req->c_req.encrypt ? 0 : sz);
2555
2329
2556
2330
ret = sec_aead_param_check(ctx, req, &need_fallback);