crypto: hisilicon/hpre: extend tag field to 64 bits for better performance

+26 -123

2 changed files

expand all

drivers

crypto

hisilicon

hpre

hpre.h

hpre_crypto.c

+2 -3

drivers/crypto/hisilicon/hpre/hpre.h

··· 94 94 __le64 key; 95 95 __le64 in; 96 96 __le64 out; 97 - __le16 tag; 98 - __le16 resv2; 99 - #define _HPRE_SQE_ALIGN_EXT 7 97 + __le64 tag; 98 + #define _HPRE_SQE_ALIGN_EXT 6 100 99 __le32 rsvd1[_HPRE_SQE_ALIGN_EXT]; 101 100 }; 102 101

+24 -120

drivers/crypto/hisilicon/hpre/hpre_crypto.c

··· 108 108 struct hpre_ctx { 109 109 struct hisi_qp *qp; 110 110 struct device *dev; 111 - struct hpre_asym_request **req_list; 112 111 struct hpre *hpre; 113 112 spinlock_t req_lock; 114 113 unsigned int key_sz; 115 114 bool crt_g2_mode; 116 - struct idr req_idr; 117 115 union { 118 116 struct hpre_rsa_ctx rsa; 119 117 struct hpre_dh_ctx dh; ··· 134 136 struct kpp_request *ecdh; 135 137 } areq; 136 138 int err; 137 - int req_id; 138 139 hpre_cb cb; 139 140 struct timespec64 req_time; 140 141 }; ··· 148 151 return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1); 149 152 } 150 153 151 - static int hpre_alloc_req_id(struct hpre_ctx *ctx) 152 - { 153 - unsigned long flags; 154 - int id; 155 - 156 - spin_lock_irqsave(&ctx->req_lock, flags); 157 - id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC); 158 - spin_unlock_irqrestore(&ctx->req_lock, flags); 159 - 160 - return id; 161 - } 162 - 163 - static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id) 164 - { 165 - unsigned long flags; 166 - 167 - spin_lock_irqsave(&ctx->req_lock, flags); 168 - idr_remove(&ctx->req_idr, req_id); 169 - spin_unlock_irqrestore(&ctx->req_lock, flags); 170 - } 171 - 172 - static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req) 173 - { 174 - struct hpre_ctx *ctx; 175 - struct hpre_dfx *dfx; 176 - int id; 177 - 178 - ctx = hpre_req->ctx; 179 - id = hpre_alloc_req_id(ctx); 180 - if (unlikely(id < 0)) 181 - return -EINVAL; 182 - 183 - ctx->req_list[id] = hpre_req; 184 - hpre_req->req_id = id; 185 - 186 - dfx = ctx->hpre->debug.dfx; 187 - if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) 188 - ktime_get_ts64(&hpre_req->req_time); 189 - 190 - return id; 191 - } 192 - 193 - static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) 154 + static void hpre_dfx_add_req_time(struct hpre_asym_request *hpre_req) 194 155 { 195 156 struct hpre_ctx *ctx = hpre_req->ctx; 196 - int id = hpre_req->req_id; 157 + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 197 158 198 - if (hpre_req->req_id >= 0) { 199 - hpre_req->req_id = HPRE_INVLD_REQ_ID; 200 - ctx->req_list[id] = NULL; 201 - hpre_free_req_id(ctx, id); 202 - } 159 + if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) 160 + ktime_get_ts64(&hpre_req->req_time); 203 161 } 204 162 205 163 static struct hisi_qp *hpre_get_qp_and_start(u8 type) ··· 292 340 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, 293 341 void **kreq) 294 342 { 295 - struct hpre_asym_request *req; 296 343 unsigned int err, done, alg; 297 - int id; 298 344 299 345 #define HPRE_NO_HW_ERR 0 300 346 #define HPRE_HW_TASK_DONE 3 301 347 #define HREE_HW_ERR_MASK GENMASK(10, 0) 302 348 #define HREE_SQE_DONE_MASK GENMASK(1, 0) 303 349 #define HREE_ALG_TYPE_MASK GENMASK(4, 0) 304 - id = (int)le16_to_cpu(sqe->tag); 305 - req = ctx->req_list[id]; 306 - hpre_rm_req_from_ctx(req); 307 - *kreq = req; 350 + *kreq = (void *)le64_to_cpu(sqe->tag); 308 351 309 352 err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) & 310 353 HREE_HW_ERR_MASK; 311 - 312 354 done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) & 313 355 HREE_SQE_DONE_MASK; 314 - 315 356 if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) 316 357 return 0; 317 358 ··· 315 370 return -EINVAL; 316 371 } 317 372 318 - static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) 319 - { 320 - struct hpre *hpre; 321 - 322 - if (!ctx || !qp || qlen < 0) 323 - return -EINVAL; 324 - 325 - spin_lock_init(&ctx->req_lock); 326 - ctx->qp = qp; 327 - ctx->dev = &qp->qm->pdev->dev; 328 - 329 - hpre = container_of(ctx->qp->qm, struct hpre, qm); 330 - ctx->hpre = hpre; 331 - ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL); 332 - if (!ctx->req_list) 333 - return -ENOMEM; 334 - ctx->key_sz = 0; 335 - ctx->crt_g2_mode = false; 336 - idr_init(&ctx->req_idr); 337 - 338 - return 0; 339 - } 340 - 341 373 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all) 342 374 { 343 375 if (is_clear_all) { 344 - idr_destroy(&ctx->req_idr); 345 - kfree(ctx->req_list); 346 376 hisi_qm_free_qps(&ctx->qp, 1); 347 377 } 348 378 ··· 387 467 388 468 static void hpre_alg_cb(struct hisi_qp *qp, void *resp) 389 469 { 390 - struct hpre_ctx *ctx = qp->qp_ctx; 391 - struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 470 + struct hpre_asym_request *h_req; 392 471 struct hpre_sqe *sqe = resp; 393 - struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)]; 394 472 395 - if (unlikely(!req)) { 396 - atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value); 473 + h_req = (struct hpre_asym_request *)le64_to_cpu(sqe->tag); 474 + if (unlikely(!h_req)) { 475 + pr_err("Failed to get request, and qp_id is %u\n", qp->qp_id); 397 476 return; 398 477 } 399 478 400 - req->cb(ctx, resp); 401 - } 402 - 403 - static void hpre_stop_qp_and_put(struct hisi_qp *qp) 404 - { 405 - hisi_qm_stop_qp(qp); 406 - hisi_qm_free_qps(&qp, 1); 479 + h_req->cb(h_req->ctx, resp); 407 480 } 408 481 409 482 static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) 410 483 { 411 484 struct hisi_qp *qp; 412 - int ret; 485 + struct hpre *hpre; 413 486 414 487 qp = hpre_get_qp_and_start(type); 415 488 if (IS_ERR(qp)) ··· 410 497 411 498 qp->qp_ctx = ctx; 412 499 qp->req_cb = hpre_alg_cb; 500 + spin_lock_init(&ctx->req_lock); 501 + ctx->qp = qp; 502 + ctx->dev = &qp->qm->pdev->dev; 503 + hpre = container_of(ctx->qp->qm, struct hpre, qm); 504 + ctx->hpre = hpre; 505 + ctx->key_sz = 0; 506 + ctx->crt_g2_mode = false; 413 507 414 - ret = hpre_ctx_set(ctx, qp, qp->sq_depth); 415 - if (ret) 416 - hpre_stop_qp_and_put(qp); 417 - 418 - return ret; 508 + return 0; 419 509 } 420 510 421 511 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) 422 512 { 423 513 struct hpre_asym_request *h_req; 424 514 struct hpre_sqe *msg; 425 - int req_id; 426 515 void *tmp; 427 516 428 517 if (is_rsa) { ··· 464 549 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 465 550 h_req->ctx = ctx; 466 551 467 - req_id = hpre_add_req_to_ctx(h_req); 468 - if (req_id < 0) 469 - return -EBUSY; 470 - 471 - msg->tag = cpu_to_le16((u16)req_id); 552 + hpre_dfx_add_req_time(h_req); 553 + msg->tag = cpu_to_le64((uintptr_t)h_req); 472 554 473 555 return 0; 474 556 } ··· 531 619 return -EINPROGRESS; 532 620 533 621 clear_all: 534 - hpre_rm_req_from_ctx(hpre_req); 535 622 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 536 623 537 624 return ret; ··· 739 828 return -EINPROGRESS; 740 829 741 830 clear_all: 742 - hpre_rm_req_from_ctx(hpre_req); 743 831 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 744 832 745 833 return ret; ··· 793 883 return -EINPROGRESS; 794 884 795 885 clear_all: 796 - hpre_rm_req_from_ctx(hpre_req); 797 886 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 798 887 799 888 return ret; ··· 1255 1346 return 0; 1256 1347 } 1257 1348 1258 - static bool hpre_key_is_zero(char *key, unsigned short key_sz) 1349 + static bool hpre_key_is_zero(const char *key, unsigned short key_sz) 1259 1350 { 1260 1351 int i; 1261 1352 ··· 1397 1488 { 1398 1489 struct hpre_asym_request *h_req; 1399 1490 struct hpre_sqe *msg; 1400 - int req_id; 1401 1491 void *tmp; 1402 1492 1403 1493 if (req->dst_len < ctx->key_sz << 1) { ··· 1418 1510 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 1419 1511 h_req->ctx = ctx; 1420 1512 1421 - req_id = hpre_add_req_to_ctx(h_req); 1422 - if (req_id < 0) 1423 - return -EBUSY; 1424 - 1425 - msg->tag = cpu_to_le16((u16)req_id); 1513 + hpre_dfx_add_req_time(h_req); 1514 + msg->tag = cpu_to_le64((uintptr_t)h_req); 1426 1515 return 0; 1427 1516 } 1428 1517 ··· 1517 1612 return -EINPROGRESS; 1518 1613 1519 1614 clear_all: 1520 - hpre_rm_req_from_ctx(hpre_req); 1521 1615 hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 1522 1616 return ret; 1523 1617 }

Configure Feed

Configure Feed