Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: starfive - Update hash dma usage
Current hash uses sw fallback for non-word aligned input scatterlists.
Add support for unaligned cases utilizing the data valid mask for dma.
Signed-off-by: Jia Jie Ho <jiajie.ho@starfivetech.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
authored by
Jia Jie Ho
and committed by
Herbert Xu
b6e9eb69
2ccf7a5d
+117
-181
-9
drivers/crypto/starfive/jh7110-cryp.c
-9
drivers/crypto/starfive/jh7110-cryp.c
···
103
103
tasklet_schedule(&cryp->aes_done);
104
104
}
105
105
106
-
if (status & STARFIVE_IE_FLAG_HASH_DONE) {
107
-
mask |= STARFIVE_IE_MASK_HASH_DONE;
108
-
writel(mask, cryp->base + STARFIVE_IE_MASK_OFFSET);
109
-
tasklet_schedule(&cryp->hash_done);
110
-
}
111
-
112
106
return IRQ_HANDLED;
113
107
}
114
108
···
126
132
"Error remapping memory for platform device\n");
127
133
128
134
tasklet_init(&cryp->aes_done, starfive_aes_done_task, (unsigned long)cryp);
129
-
tasklet_init(&cryp->hash_done, starfive_hash_done_task, (unsigned long)cryp);
130
135
131
136
cryp->phys_base = res->start;
132
137
cryp->dma_maxburst = 32;
···
213
220
reset_control_assert(cryp->rst);
214
221
215
222
tasklet_kill(&cryp->aes_done);
216
-
tasklet_kill(&cryp->hash_done);
217
223
218
224
return ret;
219
225
}
···
226
234
starfive_rsa_unregister_algs();
227
235
228
236
tasklet_kill(&cryp->aes_done);
229
-
tasklet_kill(&cryp->hash_done);
230
237
231
238
crypto_engine_stop(cryp->engine);
232
239
crypto_engine_exit(cryp->engine);
+2
-2
drivers/crypto/starfive/jh7110-cryp.h
+2
-2
drivers/crypto/starfive/jh7110-cryp.h
···
91
91
#define STARFIVE_HASH_KEY_DONE BIT(13)
92
92
u32 key_done :1;
93
93
u32 key_flag :1;
94
+
#define STARFIVE_HASH_HMAC_DONE BIT(15)
94
95
u32 hmac_done :1;
95
96
#define STARFIVE_HASH_BUSY BIT(16)
96
97
u32 busy :1;
···
190
189
struct scatter_walk out_walk;
191
190
struct crypto_engine *engine;
192
191
struct tasklet_struct aes_done;
193
-
struct tasklet_struct hash_done;
192
+
struct completion dma_done;
194
193
size_t assoclen;
195
194
size_t total_in;
196
195
size_t total_out;
···
238
237
int starfive_aes_register_algs(void);
239
238
void starfive_aes_unregister_algs(void);
240
239
241
-
void starfive_hash_done_task(unsigned long param);
242
240
void starfive_aes_done_task(unsigned long param);
243
241
#endif
+115
-170
drivers/crypto/starfive/jh7110-hash.c
+115
-170
drivers/crypto/starfive/jh7110-hash.c
···
36
36
#define STARFIVE_HASH_BUFLEN SHA512_BLOCK_SIZE
37
37
#define STARFIVE_HASH_RESET 0x2
38
38
39
-
static inline int starfive_hash_wait_busy(struct starfive_cryp_ctx *ctx)
39
+
static inline int starfive_hash_wait_busy(struct starfive_cryp_dev *cryp)
40
40
{
41
-
struct starfive_cryp_dev *cryp = ctx->cryp;
42
41
u32 status;
43
42
44
43
return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status,
45
44
!(status & STARFIVE_HASH_BUSY), 10, 100000);
45
+
}
46
+
47
+
static inline int starfive_hash_wait_hmac_done(struct starfive_cryp_dev *cryp)
48
+
{
49
+
u32 status;
50
+
51
+
return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status,
52
+
(status & STARFIVE_HASH_HMAC_DONE), 10, 100000);
46
53
}
47
54
48
55
static inline int starfive_hash_wait_key_done(struct starfive_cryp_ctx *ctx)
···
91
84
return 0;
92
85
}
93
86
94
-
static void starfive_hash_start(void *param)
87
+
static void starfive_hash_start(struct starfive_cryp_dev *cryp)
95
88
{
96
-
struct starfive_cryp_ctx *ctx = param;
97
-
struct starfive_cryp_request_ctx *rctx = ctx->rctx;
98
-
struct starfive_cryp_dev *cryp = ctx->cryp;
99
-
union starfive_alg_cr alg_cr;
100
89
union starfive_hash_csr csr;
101
-
u32 stat;
102
-
103
-
dma_unmap_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE);
104
-
105
-
alg_cr.v = 0;
106
-
alg_cr.clear = 1;
107
-
108
-
writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
109
90
110
91
csr.v = readl(cryp->base + STARFIVE_HASH_SHACSR);
111
92
csr.firstb = 0;
112
93
csr.final = 1;
113
-
114
-
stat = readl(cryp->base + STARFIVE_IE_MASK_OFFSET);
115
-
stat &= ~STARFIVE_IE_MASK_HASH_DONE;
116
-
writel(stat, cryp->base + STARFIVE_IE_MASK_OFFSET);
117
94
writel(csr.v, cryp->base + STARFIVE_HASH_SHACSR);
118
95
}
119
96
120
-
static int starfive_hash_xmit_dma(struct starfive_cryp_ctx *ctx)
97
+
static void starfive_hash_dma_callback(void *param)
121
98
{
122
-
struct starfive_cryp_request_ctx *rctx = ctx->rctx;
123
-
struct starfive_cryp_dev *cryp = ctx->cryp;
124
-
struct dma_async_tx_descriptor *in_desc;
125
-
union starfive_alg_cr alg_cr;
126
-
int total_len;
127
-
int ret;
99
+
struct starfive_cryp_dev *cryp = param;
128
100
129
-
if (!rctx->total) {
130
-
starfive_hash_start(ctx);
131
-
return 0;
132
-
}
101
+
complete(&cryp->dma_done);
102
+
}
133
103
134
-
writel(rctx->total, cryp->base + STARFIVE_DMA_IN_LEN_OFFSET);
135
-
136
-
total_len = rctx->total;
137
-
total_len = (total_len & 0x3) ? (((total_len >> 2) + 1) << 2) : total_len;
138
-
sg_dma_len(rctx->in_sg) = total_len;
139
-
140
-
alg_cr.v = 0;
141
-
alg_cr.start = 1;
142
-
alg_cr.hash_dma_en = 1;
143
-
144
-
writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
145
-
146
-
ret = dma_map_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE);
147
-
if (!ret)
148
-
return dev_err_probe(cryp->dev, -EINVAL, "dma_map_sg() error\n");
149
-
150
-
cryp->cfg_in.direction = DMA_MEM_TO_DEV;
151
-
cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
104
+
static void starfive_hash_dma_init(struct starfive_cryp_dev *cryp)
105
+
{
106
+
cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_16_BYTES;
152
107
cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
153
108
cryp->cfg_in.src_maxburst = cryp->dma_maxburst;
154
109
cryp->cfg_in.dst_maxburst = cryp->dma_maxburst;
···
118
149
119
150
dmaengine_slave_config(cryp->tx, &cryp->cfg_in);
120
151
121
-
in_desc = dmaengine_prep_slave_sg(cryp->tx, rctx->in_sg,
122
-
ret, DMA_MEM_TO_DEV,
123
-
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
152
+
init_completion(&cryp->dma_done);
153
+
}
124
154
125
-
if (!in_desc)
126
-
return -EINVAL;
155
+
static int starfive_hash_dma_xfer(struct starfive_cryp_dev *cryp,
156
+
struct scatterlist *sg)
157
+
{
158
+
struct dma_async_tx_descriptor *in_desc;
159
+
union starfive_alg_cr alg_cr;
160
+
int ret = 0;
127
161
128
-
in_desc->callback = starfive_hash_start;
129
-
in_desc->callback_param = ctx;
162
+
alg_cr.v = 0;
163
+
alg_cr.start = 1;
164
+
alg_cr.hash_dma_en = 1;
165
+
writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
166
+
167
+
writel(sg_dma_len(sg), cryp->base + STARFIVE_DMA_IN_LEN_OFFSET);
168
+
sg_dma_len(sg) = ALIGN(sg_dma_len(sg), sizeof(u32));
169
+
170
+
in_desc = dmaengine_prep_slave_sg(cryp->tx, sg, 1, DMA_MEM_TO_DEV,
171
+
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
172
+
if (!in_desc) {
173
+
ret = -EINVAL;
174
+
goto end;
175
+
}
176
+
177
+
reinit_completion(&cryp->dma_done);
178
+
in_desc->callback = starfive_hash_dma_callback;
179
+
in_desc->callback_param = cryp;
130
180
131
181
dmaengine_submit(in_desc);
132
182
dma_async_issue_pending(cryp->tx);
133
183
134
-
return 0;
135
-
}
184
+
if (!wait_for_completion_timeout(&cryp->dma_done,
185
+
msecs_to_jiffies(1000)))
186
+
ret = -ETIMEDOUT;
136
187
137
-
static int starfive_hash_xmit(struct starfive_cryp_ctx *ctx)
138
-
{
139
-
struct starfive_cryp_request_ctx *rctx = ctx->rctx;
140
-
struct starfive_cryp_dev *cryp = ctx->cryp;
141
-
int ret = 0;
188
+
end:
189
+
alg_cr.v = 0;
190
+
alg_cr.clear = 1;
191
+
writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET);
142
192
143
-
rctx->csr.hash.v = 0;
144
-
rctx->csr.hash.reset = 1;
145
-
writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR);
146
-
147
-
if (starfive_hash_wait_busy(ctx))
148
-
return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error resetting engine.\n");
149
-
150
-
rctx->csr.hash.v = 0;
151
-
rctx->csr.hash.mode = ctx->hash_mode;
152
-
rctx->csr.hash.ie = 1;
153
-
154
-
if (ctx->is_hmac) {
155
-
ret = starfive_hash_hmac_key(ctx);
156
-
if (ret)
157
-
return ret;
158
-
} else {
159
-
rctx->csr.hash.start = 1;
160
-
rctx->csr.hash.firstb = 1;
161
-
writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR);
162
-
}
163
-
164
-
return starfive_hash_xmit_dma(ctx);
193
+
return ret;
165
194
}
166
195
167
196
static int starfive_hash_copy_hash(struct ahash_request *req)
···
182
215
return 0;
183
216
}
184
217
185
-
void starfive_hash_done_task(unsigned long param)
218
+
static void starfive_hash_done_task(struct starfive_cryp_dev *cryp)
186
219
{
187
-
struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param;
188
220
int err = cryp->err;
189
221
190
222
if (!err)
191
223
err = starfive_hash_copy_hash(cryp->req.hreq);
192
224
193
-
/* Reset to clear hash_done in irq register*/
194
-
writel(STARFIVE_HASH_RESET, cryp->base + STARFIVE_HASH_SHACSR);
195
-
196
225
crypto_finalize_hash_request(cryp->engine, cryp->req.hreq, err);
197
-
}
198
-
199
-
static int starfive_hash_check_aligned(struct scatterlist *sg, size_t total, size_t align)
200
-
{
201
-
int len = 0;
202
-
203
-
if (!total)
204
-
return 0;
205
-
206
-
if (!IS_ALIGNED(total, align))
207
-
return -EINVAL;
208
-
209
-
while (sg) {
210
-
if (!IS_ALIGNED(sg->offset, sizeof(u32)))
211
-
return -EINVAL;
212
-
213
-
if (!IS_ALIGNED(sg->length, align))
214
-
return -EINVAL;
215
-
216
-
len += sg->length;
217
-
sg = sg_next(sg);
218
-
}
219
-
220
-
if (len != total)
221
-
return -EINVAL;
222
-
223
-
return 0;
224
226
}
225
227
226
228
static int starfive_hash_one_request(struct crypto_engine *engine, void *areq)
···
197
261
struct ahash_request *req = container_of(areq, struct ahash_request,
198
262
base);
199
263
struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
264
+
struct starfive_cryp_request_ctx *rctx = ctx->rctx;
200
265
struct starfive_cryp_dev *cryp = ctx->cryp;
266
+
struct scatterlist *tsg;
267
+
int ret, src_nents, i;
201
268
202
-
if (!cryp)
203
-
return -ENODEV;
269
+
writel(STARFIVE_HASH_RESET, cryp->base + STARFIVE_HASH_SHACSR);
204
270
205
-
return starfive_hash_xmit(ctx);
271
+
if (starfive_hash_wait_busy(cryp))
272
+
return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error resetting hardware\n");
273
+
274
+
rctx->csr.hash.v = 0;
275
+
rctx->csr.hash.mode = ctx->hash_mode;
276
+
277
+
if (ctx->is_hmac) {
278
+
ret = starfive_hash_hmac_key(ctx);
279
+
if (ret)
280
+
return ret;
281
+
} else {
282
+
rctx->csr.hash.start = 1;
283
+
rctx->csr.hash.firstb = 1;
284
+
writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR);
285
+
}
286
+
287
+
/* No input message, get digest and end. */
288
+
if (!rctx->total)
289
+
goto hash_start;
290
+
291
+
starfive_hash_dma_init(cryp);
292
+
293
+
for_each_sg(rctx->in_sg, tsg, rctx->in_sg_len, i) {
294
+
src_nents = dma_map_sg(cryp->dev, tsg, 1, DMA_TO_DEVICE);
295
+
if (src_nents == 0)
296
+
return dev_err_probe(cryp->dev, -ENOMEM,
297
+
"dma_map_sg error\n");
298
+
299
+
ret = starfive_hash_dma_xfer(cryp, tsg);
300
+
dma_unmap_sg(cryp->dev, tsg, 1, DMA_TO_DEVICE);
301
+
if (ret)
302
+
return ret;
303
+
}
304
+
305
+
hash_start:
306
+
starfive_hash_start(cryp);
307
+
308
+
if (starfive_hash_wait_busy(cryp))
309
+
return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error generating digest\n");
310
+
311
+
if (ctx->is_hmac)
312
+
cryp->err = starfive_hash_wait_hmac_done(cryp);
313
+
314
+
starfive_hash_done_task(cryp);
315
+
316
+
return 0;
206
317
}
207
318
208
319
static int starfive_hash_init(struct ahash_request *req)
···
320
337
return crypto_ahash_finup(&rctx->ahash_fbk_req);
321
338
}
322
339
323
-
static int starfive_hash_digest_fb(struct ahash_request *req)
324
-
{
325
-
struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req);
326
-
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
327
-
struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm);
328
-
329
-
ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk);
330
-
ahash_request_set_callback(&rctx->ahash_fbk_req, req->base.flags,
331
-
req->base.complete, req->base.data);
332
-
333
-
ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src,
334
-
req->result, req->nbytes);
335
-
336
-
return crypto_ahash_digest(&rctx->ahash_fbk_req);
337
-
}
338
-
339
340
static int starfive_hash_digest(struct ahash_request *req)
340
341
{
341
342
struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
···
336
369
rctx->digsize = crypto_ahash_digestsize(tfm);
337
370
rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total);
338
371
ctx->rctx = rctx;
339
-
340
-
if (starfive_hash_check_aligned(rctx->in_sg, rctx->total, rctx->blksize))
341
-
return starfive_hash_digest_fb(req);
342
372
343
373
return crypto_transfer_hash_request_to_engine(cryp->engine, req);
344
374
}
···
370
406
371
407
static int starfive_hash_init_tfm(struct crypto_ahash *hash,
372
408
const char *alg_name,
373
-
unsigned int mode)
409
+
unsigned int mode,
410
+
bool is_hmac)
374
411
{
375
412
struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
376
413
···
391
426
crypto_ahash_set_reqsize(hash, sizeof(struct starfive_cryp_request_ctx) +
392
427
crypto_ahash_reqsize(ctx->ahash_fbk));
393
428
394
-
ctx->keylen = 0;
429
+
ctx->is_hmac = is_hmac;
395
430
ctx->hash_mode = mode;
396
431
397
432
return 0;
···
494
529
static int starfive_sha224_init_tfm(struct crypto_ahash *hash)
495
530
{
496
531
return starfive_hash_init_tfm(hash, "sha224-generic",
497
-
STARFIVE_HASH_SHA224);
532
+
STARFIVE_HASH_SHA224, 0);
498
533
}
499
534
500
535
static int starfive_sha256_init_tfm(struct crypto_ahash *hash)
501
536
{
502
537
return starfive_hash_init_tfm(hash, "sha256-generic",
503
-
STARFIVE_HASH_SHA256);
538
+
STARFIVE_HASH_SHA256, 0);
504
539
}
505
540
506
541
static int starfive_sha384_init_tfm(struct crypto_ahash *hash)
507
542
{
508
543
return starfive_hash_init_tfm(hash, "sha384-generic",
509
-
STARFIVE_HASH_SHA384);
544
+
STARFIVE_HASH_SHA384, 0);
510
545
}
511
546
512
547
static int starfive_sha512_init_tfm(struct crypto_ahash *hash)
513
548
{
514
549
return starfive_hash_init_tfm(hash, "sha512-generic",
515
-
STARFIVE_HASH_SHA512);
550
+
STARFIVE_HASH_SHA512, 0);
516
551
}
517
552
518
553
static int starfive_sm3_init_tfm(struct crypto_ahash *hash)
519
554
{
520
555
return starfive_hash_init_tfm(hash, "sm3-generic",
521
-
STARFIVE_HASH_SM3);
556
+
STARFIVE_HASH_SM3, 0);
522
557
}
523
558
524
559
static int starfive_hmac_sha224_init_tfm(struct crypto_ahash *hash)
525
560
{
526
-
struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
527
-
528
-
ctx->is_hmac = true;
529
-
530
561
return starfive_hash_init_tfm(hash, "hmac(sha224-generic)",
531
-
STARFIVE_HASH_SHA224);
562
+
STARFIVE_HASH_SHA224, 1);
532
563
}
533
564
534
565
static int starfive_hmac_sha256_init_tfm(struct crypto_ahash *hash)
535
566
{
536
-
struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
537
-
538
-
ctx->is_hmac = true;
539
-
540
567
return starfive_hash_init_tfm(hash, "hmac(sha256-generic)",
541
-
STARFIVE_HASH_SHA256);
568
+
STARFIVE_HASH_SHA256, 1);
542
569
}
543
570
544
571
static int starfive_hmac_sha384_init_tfm(struct crypto_ahash *hash)
545
572
{
546
-
struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
547
-
548
-
ctx->is_hmac = true;
549
-
550
573
return starfive_hash_init_tfm(hash, "hmac(sha384-generic)",
551
-
STARFIVE_HASH_SHA384);
574
+
STARFIVE_HASH_SHA384, 1);
552
575
}
553
576
554
577
static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash)
555
578
{
556
-
struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
557
-
558
-
ctx->is_hmac = true;
559
-
560
579
return starfive_hash_init_tfm(hash, "hmac(sha512-generic)",
561
-
STARFIVE_HASH_SHA512);
580
+
STARFIVE_HASH_SHA512, 1);
562
581
}
563
582
564
583
static int starfive_hmac_sm3_init_tfm(struct crypto_ahash *hash)
565
584
{
566
-
struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash);
567
-
568
-
ctx->is_hmac = true;
569
-
570
585
return starfive_hash_init_tfm(hash, "hmac(sm3-generic)",
571
-
STARFIVE_HASH_SM3);
586
+
STARFIVE_HASH_SM3, 1);
572
587
}
573
588
574
589
static struct ahash_engine_alg algs_sha2_sm3[] = {