+15

Documentation/ABI/testing/sysfs-driver-ccp

reviewed

··· 8 8 0: Not fused 9 9 1: Fused 10 10 11 11 + What: /sys/bus/pci/devices/<BDF>/boot_integrity 12 12 + Date: April 2026 13 13 + KernelVersion: 6.20 14 14 + Contact: mario.limonciello@amd.com 15 15 + Description: 16 16 + The /sys/bus/pci/devices/<BDF>/boot_integrity reports 17 17 + whether the AMD CPU or APU is used for a hardware root of trust 18 18 + during the boot process. 19 19 + Possible values: 20 20 + 0: Not hardware root of trust. 21 21 + 1: Hardware root of trust 22 22 + 23 23 + NOTE: Vendors may provide design specific alternative hardware 24 24 + root of trust implementations. 25 25 + 11 26 What: /sys/bus/pci/devices/<BDF>/debug_lock_on 12 27 Date: June 2022 13 28 KernelVersion: 5.19

+1

Documentation/devicetree/bindings/crypto/atmel,at91sam9g46-aes.yaml

reviewed

··· 16 16 - const: atmel,at91sam9g46-aes 17 17 - items: 18 18 - enum: 19 19 + - microchip,lan9691-aes 19 20 - microchip,sam9x7-aes 20 21 - microchip,sama7d65-aes 21 22 - const: atmel,at91sam9g46-aes

+1

Documentation/devicetree/bindings/crypto/atmel,at91sam9g46-sha.yaml

reviewed

··· 16 16 - const: atmel,at91sam9g46-sha 17 17 - items: 18 18 - enum: 19 19 + - microchip,lan9691-sha 19 20 - microchip,sam9x7-sha 20 21 - microchip,sama7d65-sha 21 22 - const: atmel,at91sam9g46-sha

+1

Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml

reviewed

··· 14 14 items: 15 15 - enum: 16 16 - qcom,kaanapali-inline-crypto-engine 17 17 + - qcom,milos-inline-crypto-engine 17 18 - qcom,qcs8300-inline-crypto-engine 18 19 - qcom,sa8775p-inline-crypto-engine 19 20 - qcom,sc7180-inline-crypto-engine

+1

Documentation/devicetree/bindings/crypto/qcom,prng.yaml

reviewed

··· 30 30 - qcom,sm8550-trng 31 31 - qcom,sm8650-trng 32 32 - qcom,sm8750-trng 33 33 + - qcom,x1e80100-trng 33 34 - const: qcom,trng 34 35 35 36 reg:

+2

Documentation/devicetree/bindings/crypto/xlnx,zynqmp-aes.yaml

reviewed

··· 14 14 The ZynqMP AES-GCM hardened cryptographic accelerator is used to 15 15 encrypt or decrypt the data with provided key and initialization vector. 16 16 17 17 + deprecated: true 18 18 + 17 19 properties: 18 20 compatible: 19 21 const: xlnx,zynqmp-aes

+1

Documentation/devicetree/bindings/firmware/xilinx/xlnx,zynqmp-firmware.yaml

reviewed

··· 104 104 used to encrypt or decrypt the data with provided key and initialization 105 105 vector. 106 106 type: object 107 107 + deprecated: true 107 108 108 109 required: 109 110 - compatible

+57 -36

arch/s390/crypto/paes_s390.c

reviewed

··· 40 40 #define PAES_256_PROTKEY_SIZE (32 + 32) /* key + verification pattern */ 41 41 #define PXTS_256_PROTKEY_SIZE (32 + 32 + 32) /* k1 + k2 + verification pattern */ 42 42 43 43 + static bool pkey_clrkey_allowed; 44 44 + module_param_named(clrkey, pkey_clrkey_allowed, bool, 0444); 45 45 + MODULE_PARM_DESC(clrkey, "Allow clear key material (default N)"); 46 46 + 43 47 static u8 *ctrblk; 44 48 static DEFINE_MUTEX(ctrblk_lock); 45 49 ··· 196 192 * This function may sleep - don't call in non-sleeping context. 197 193 */ 198 194 static inline int convert_key(const u8 *key, unsigned int keylen, 199 199 - struct paes_protkey *pk) 195 195 + struct paes_protkey *pk, bool tested) 200 196 { 197 197 + u32 xflags = PKEY_XFLAG_NOMEMALLOC; 201 198 int rc, i; 199 199 + 200 200 + if (tested && !pkey_clrkey_allowed) 201 201 + xflags |= PKEY_XFLAG_NOCLEARKEY; 202 202 203 203 pk->len = sizeof(pk->protkey); 204 204 ··· 217 209 } 218 210 rc = pkey_key2protkey(key, keylen, 219 211 pk->protkey, &pk->len, &pk->type, 220 220 - PKEY_XFLAG_NOMEMALLOC); 212 212 + xflags); 221 213 } 222 214 223 215 out: ··· 239 231 * unnecessary additional conversion but never to invalid data on en- 240 232 * or decrypt operations. 241 233 */ 242 242 - static int paes_convert_key(struct s390_paes_ctx *ctx) 234 234 + static int paes_convert_key(struct s390_paes_ctx *ctx, bool tested) 243 235 { 244 236 struct paes_protkey pk; 245 237 int rc; ··· 248 240 ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS; 249 241 spin_unlock_bh(&ctx->pk_lock); 250 242 251 251 - rc = convert_key(ctx->keybuf, ctx->keylen, &pk); 243 243 + rc = convert_key(ctx->keybuf, ctx->keylen, &pk, tested); 252 244 253 245 /* update context */ 254 246 spin_lock_bh(&ctx->pk_lock); ··· 271 263 * pk_type, pk_len and the protected key in the tfm context. 272 264 * See also comments on function paes_convert_key. 273 265 */ 274 274 - static int pxts_convert_key(struct s390_pxts_ctx *ctx) 266 266 + static int pxts_convert_key(struct s390_pxts_ctx *ctx, bool tested) 275 267 { 276 268 struct paes_protkey pk0, pk1; 277 269 size_t split_keylen; ··· 281 273 ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS; 282 274 spin_unlock_bh(&ctx->pk_lock); 283 275 284 284 - rc = convert_key(ctx->keybuf, ctx->keylen, &pk0); 276 276 + rc = convert_key(ctx->keybuf, ctx->keylen, &pk0, tested); 285 277 if (rc) 286 278 goto out; 287 279 ··· 295 287 } 296 288 split_keylen = ctx->keylen / 2; 297 289 rc = convert_key(ctx->keybuf + split_keylen, 298 298 - split_keylen, &pk1); 290 290 + split_keylen, &pk1, tested); 299 291 if (rc) 300 292 goto out; 301 293 if (pk0.type != pk1.type) { ··· 351 343 unsigned int key_len) 352 344 { 353 345 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 346 346 + bool tested = crypto_skcipher_tested(tfm); 354 347 long fc; 355 348 int rc; 356 349 ··· 361 352 goto out; 362 353 363 354 /* convert key into protected key */ 364 364 - rc = paes_convert_key(ctx); 355 355 + rc = paes_convert_key(ctx, tested); 365 356 if (rc) 366 357 goto out; 367 358 ··· 391 382 392 383 static int ecb_paes_do_crypt(struct s390_paes_ctx *ctx, 393 384 struct s390_pecb_req_ctx *req_ctx, 394 394 - bool maysleep) 385 385 + bool tested, bool maysleep) 395 386 { 396 387 struct ecb_param *param = &req_ctx->param; 397 388 struct skcipher_walk *walk = &req_ctx->walk; ··· 439 430 rc = -EKEYEXPIRED; 440 431 goto out; 441 432 } 442 442 - rc = paes_convert_key(ctx); 433 433 + rc = paes_convert_key(ctx, tested); 443 434 if (rc) 444 435 goto out; 445 436 spin_lock_bh(&ctx->pk_lock); ··· 459 450 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 460 451 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 461 452 struct skcipher_walk *walk = &req_ctx->walk; 453 453 + bool tested = crypto_skcipher_tested(tfm); 462 454 int rc; 463 455 464 456 /* ··· 478 468 479 469 /* Try synchronous operation if no active engine usage */ 480 470 if (!atomic_read(&ctx->via_engine_ctr)) { 481 481 - rc = ecb_paes_do_crypt(ctx, req_ctx, false); 471 471 + rc = ecb_paes_do_crypt(ctx, req_ctx, tested, false); 482 472 if (rc == 0) 483 473 goto out; 484 474 } ··· 541 531 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 542 532 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 543 533 struct skcipher_walk *walk = &req_ctx->walk; 534 534 + bool tested = crypto_skcipher_tested(tfm); 544 535 int rc; 545 536 546 537 /* walk has already been prepared */ 547 538 548 548 - rc = ecb_paes_do_crypt(ctx, req_ctx, true); 539 539 + rc = ecb_paes_do_crypt(ctx, req_ctx, tested, true); 549 540 if (rc == -EKEYEXPIRED) { 550 541 /* 551 542 * Protected key expired, conversion is in process. ··· 613 602 unsigned int key_len) 614 603 { 615 604 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 605 605 + bool tested = crypto_skcipher_tested(tfm); 616 606 long fc; 617 607 int rc; 618 608 ··· 623 611 goto out; 624 612 625 613 /* convert raw key into protected key */ 626 626 - rc = paes_convert_key(ctx); 614 614 + rc = paes_convert_key(ctx, tested); 627 615 if (rc) 628 616 goto out; 629 617 ··· 653 641 654 642 static int cbc_paes_do_crypt(struct s390_paes_ctx *ctx, 655 643 struct s390_pcbc_req_ctx *req_ctx, 656 656 - bool maysleep) 644 644 + bool tested, bool maysleep) 657 645 { 658 646 struct cbc_param *param = &req_ctx->param; 659 647 struct skcipher_walk *walk = &req_ctx->walk; ··· 705 693 rc = -EKEYEXPIRED; 706 694 goto out; 707 695 } 708 708 - rc = paes_convert_key(ctx); 696 696 + rc = paes_convert_key(ctx, tested); 709 697 if (rc) 710 698 goto out; 711 699 spin_lock_bh(&ctx->pk_lock); ··· 725 713 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 726 714 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 727 715 struct skcipher_walk *walk = &req_ctx->walk; 716 716 + bool tested = crypto_skcipher_tested(tfm); 728 717 int rc; 729 718 730 719 /* ··· 744 731 745 732 /* Try synchronous operation if no active engine usage */ 746 733 if (!atomic_read(&ctx->via_engine_ctr)) { 747 747 - rc = cbc_paes_do_crypt(ctx, req_ctx, false); 734 734 + rc = cbc_paes_do_crypt(ctx, req_ctx, tested, false); 748 735 if (rc == 0) 749 736 goto out; 750 737 } ··· 807 794 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 808 795 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 809 796 struct skcipher_walk *walk = &req_ctx->walk; 797 797 + bool tested = crypto_skcipher_tested(tfm); 810 798 int rc; 811 799 812 800 /* walk has already been prepared */ 813 801 814 814 - rc = cbc_paes_do_crypt(ctx, req_ctx, true); 802 802 + rc = cbc_paes_do_crypt(ctx, req_ctx, tested, true); 815 803 if (rc == -EKEYEXPIRED) { 816 804 /* 817 805 * Protected key expired, conversion is in process. ··· 879 865 unsigned int key_len) 880 866 { 881 867 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 868 868 + bool tested = crypto_skcipher_tested(tfm); 882 869 long fc; 883 870 int rc; 884 871 ··· 889 874 goto out; 890 875 891 876 /* convert raw key into protected key */ 892 892 - rc = paes_convert_key(ctx); 877 877 + rc = paes_convert_key(ctx, tested); 893 878 if (rc) 894 879 goto out; 895 880 ··· 934 919 935 920 static int ctr_paes_do_crypt(struct s390_paes_ctx *ctx, 936 921 struct s390_pctr_req_ctx *req_ctx, 937 937 - bool maysleep) 922 922 + bool tested, bool maysleep) 938 923 { 939 924 struct ctr_param *param = &req_ctx->param; 940 925 struct skcipher_walk *walk = &req_ctx->walk; ··· 994 979 rc = -EKEYEXPIRED; 995 980 goto out; 996 981 } 997 997 - rc = paes_convert_key(ctx); 982 982 + rc = paes_convert_key(ctx, tested); 998 983 if (rc) { 999 984 if (locked) 1000 985 mutex_unlock(&ctrblk_lock); ··· 1021 1006 rc = -EKEYEXPIRED; 1022 1007 goto out; 1023 1008 } 1024 1024 - rc = paes_convert_key(ctx); 1009 1009 + rc = paes_convert_key(ctx, tested); 1025 1010 if (rc) 1026 1011 goto out; 1027 1012 spin_lock_bh(&ctx->pk_lock); ··· 1044 1029 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1045 1030 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 1046 1031 struct skcipher_walk *walk = &req_ctx->walk; 1032 1032 + bool tested = crypto_skcipher_tested(tfm); 1047 1033 int rc; 1048 1034 1049 1035 /* ··· 1062 1046 1063 1047 /* Try synchronous operation if no active engine usage */ 1064 1048 if (!atomic_read(&ctx->via_engine_ctr)) { 1065 1065 - rc = ctr_paes_do_crypt(ctx, req_ctx, false); 1049 1049 + rc = ctr_paes_do_crypt(ctx, req_ctx, tested, false); 1066 1050 if (rc == 0) 1067 1051 goto out; 1068 1052 } ··· 1115 1099 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1116 1100 struct s390_paes_ctx *ctx = crypto_skcipher_ctx(tfm); 1117 1101 struct skcipher_walk *walk = &req_ctx->walk; 1102 1102 + bool tested = crypto_skcipher_tested(tfm); 1118 1103 int rc; 1119 1104 1120 1105 /* walk has already been prepared */ 1121 1106 1122 1122 - rc = ctr_paes_do_crypt(ctx, req_ctx, true); 1107 1107 + rc = ctr_paes_do_crypt(ctx, req_ctx, tested, true); 1123 1108 if (rc == -EKEYEXPIRED) { 1124 1109 /* 1125 1110 * Protected key expired, conversion is in process. ··· 1207 1190 unsigned int in_keylen) 1208 1191 { 1209 1192 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); 1193 1193 + bool tested = crypto_skcipher_tested(tfm); 1210 1194 u8 ckey[2 * AES_MAX_KEY_SIZE]; 1211 1195 unsigned int ckey_len; 1212 1196 long fc; ··· 1223 1205 goto out; 1224 1206 1225 1207 /* convert raw key(s) into protected key(s) */ 1226 1226 - rc = pxts_convert_key(ctx); 1208 1208 + rc = pxts_convert_key(ctx, tested); 1227 1209 if (rc) 1228 1210 goto out; 1229 1211 ··· 1273 1255 1274 1256 static int xts_paes_do_crypt_fullkey(struct s390_pxts_ctx *ctx, 1275 1257 struct s390_pxts_req_ctx *req_ctx, 1276 1276 - bool maysleep) 1258 1258 + bool tested, bool maysleep) 1277 1259 { 1278 1260 struct xts_full_km_param *param = &req_ctx->param.full_km_param; 1279 1261 struct skcipher_walk *walk = &req_ctx->walk; ··· 1317 1299 rc = -EKEYEXPIRED; 1318 1300 goto out; 1319 1301 } 1320 1320 - rc = pxts_convert_key(ctx); 1302 1302 + rc = pxts_convert_key(ctx, tested); 1321 1303 if (rc) 1322 1304 goto out; 1323 1305 spin_lock_bh(&ctx->pk_lock); ··· 1336 1318 struct xts_km_param *param, 1337 1319 struct skcipher_walk *walk, 1338 1320 unsigned int keylen, 1339 1339 - unsigned int offset, bool maysleep) 1321 1321 + unsigned int offset, 1322 1322 + bool tested, bool maysleep) 1340 1323 { 1341 1324 struct xts_pcc_param pcc_param; 1342 1325 unsigned long cc = 1; ··· 1356 1337 rc = -EKEYEXPIRED; 1357 1338 break; 1358 1339 } 1359 1359 - rc = pxts_convert_key(ctx); 1340 1340 + rc = pxts_convert_key(ctx, tested); 1360 1341 if (rc) 1361 1342 break; 1362 1343 continue; ··· 1370 1351 1371 1352 static int xts_paes_do_crypt_2keys(struct s390_pxts_ctx *ctx, 1372 1353 struct s390_pxts_req_ctx *req_ctx, 1373 1373 - bool maysleep) 1354 1354 + bool tested, bool maysleep) 1374 1355 { 1375 1356 struct xts_km_param *param = &req_ctx->param.km_param; 1376 1357 struct skcipher_walk *walk = &req_ctx->walk; ··· 1388 1369 1389 1370 if (!req_ctx->param_init_done) { 1390 1371 rc = __xts_2keys_prep_param(ctx, param, walk, 1391 1391 - keylen, offset, maysleep); 1372 1372 + keylen, offset, tested, maysleep); 1392 1373 if (rc) 1393 1374 goto out; 1394 1375 req_ctx->param_init_done = true; ··· 1411 1392 rc = -EKEYEXPIRED; 1412 1393 goto out; 1413 1394 } 1414 1414 - rc = pxts_convert_key(ctx); 1395 1395 + rc = pxts_convert_key(ctx, tested); 1415 1396 if (rc) 1416 1397 goto out; 1417 1398 spin_lock_bh(&ctx->pk_lock); ··· 1427 1408 1428 1409 static int xts_paes_do_crypt(struct s390_pxts_ctx *ctx, 1429 1410 struct s390_pxts_req_ctx *req_ctx, 1430 1430 - bool maysleep) 1411 1411 + bool tested, bool maysleep) 1431 1412 { 1432 1413 int pk_state, rc = 0; 1433 1414 ··· 1455 1436 switch (ctx->fc) { 1456 1437 case CPACF_KM_PXTS_128: 1457 1438 case CPACF_KM_PXTS_256: 1458 1458 - rc = xts_paes_do_crypt_2keys(ctx, req_ctx, maysleep); 1439 1439 + rc = xts_paes_do_crypt_2keys(ctx, req_ctx, tested, maysleep); 1459 1440 break; 1460 1441 case CPACF_KM_PXTS_128_FULL: 1461 1442 case CPACF_KM_PXTS_256_FULL: 1462 1462 - rc = xts_paes_do_crypt_fullkey(ctx, req_ctx, maysleep); 1443 1443 + rc = xts_paes_do_crypt_fullkey(ctx, req_ctx, tested, maysleep); 1463 1444 break; 1464 1445 default: 1465 1446 rc = -EINVAL; ··· 1476 1457 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1477 1458 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); 1478 1459 struct skcipher_walk *walk = &req_ctx->walk; 1460 1460 + bool tested = crypto_skcipher_tested(tfm); 1479 1461 int rc; 1480 1462 1481 1463 /* ··· 1495 1475 1496 1476 /* Try synchronous operation if no active engine usage */ 1497 1477 if (!atomic_read(&ctx->via_engine_ctr)) { 1498 1498 - rc = xts_paes_do_crypt(ctx, req_ctx, false); 1478 1478 + rc = xts_paes_do_crypt(ctx, req_ctx, tested, false); 1499 1479 if (rc == 0) 1500 1480 goto out; 1501 1481 } ··· 1558 1538 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 1559 1539 struct s390_pxts_ctx *ctx = crypto_skcipher_ctx(tfm); 1560 1540 struct skcipher_walk *walk = &req_ctx->walk; 1541 1541 + bool tested = crypto_skcipher_tested(tfm); 1561 1542 int rc; 1562 1543 1563 1544 /* walk has already been prepared */ 1564 1545 1565 1565 - rc = xts_paes_do_crypt(ctx, req_ctx, true); 1546 1546 + rc = xts_paes_do_crypt(ctx, req_ctx, tested, true); 1566 1547 if (rc == -EKEYEXPIRED) { 1567 1548 /* 1568 1549 * Protected key expired, conversion is in process.

+20 -9

arch/s390/crypto/phmac_s390.c

reviewed

··· 23 23 static struct crypto_engine *phmac_crypto_engine; 24 24 #define MAX_QLEN 10 25 25 26 26 + static bool pkey_clrkey_allowed; 27 27 + module_param_named(clrkey, pkey_clrkey_allowed, bool, 0444); 28 28 + MODULE_PARM_DESC(clrkey, "Allow clear key material (default N)"); 29 29 + 26 30 /* 27 31 * A simple hash walk helper 28 32 */ ··· 315 311 * This function may sleep - don't call in non-sleeping context. 316 312 */ 317 313 static inline int convert_key(const u8 *key, unsigned int keylen, 318 318 - struct phmac_protkey *pk) 314 314 + struct phmac_protkey *pk, bool tested) 319 315 { 316 316 + u32 xflags = PKEY_XFLAG_NOMEMALLOC; 320 317 int rc, i; 318 318 + 319 319 + if (tested && !pkey_clrkey_allowed) 320 320 + xflags |= PKEY_XFLAG_NOCLEARKEY; 321 321 322 322 pk->len = sizeof(pk->protkey); 323 323 ··· 336 328 } 337 329 rc = pkey_key2protkey(key, keylen, 338 330 pk->protkey, &pk->len, &pk->type, 339 339 - PKEY_XFLAG_NOMEMALLOC); 331 331 + xflags); 340 332 } 341 333 342 334 out: ··· 358 350 * unnecessary additional conversion but never to invalid data on the 359 351 * hash operation. 360 352 */ 361 361 - static int phmac_convert_key(struct phmac_tfm_ctx *tfm_ctx) 353 353 + static int phmac_convert_key(struct phmac_tfm_ctx *tfm_ctx, bool tested) 362 354 { 363 355 struct phmac_protkey pk; 364 356 int rc; ··· 367 359 tfm_ctx->pk_state = PK_STATE_CONVERT_IN_PROGRESS; 368 360 spin_unlock_bh(&tfm_ctx->pk_lock); 369 361 370 370 - rc = convert_key(tfm_ctx->keybuf, tfm_ctx->keylen, &pk); 362 362 + rc = convert_key(tfm_ctx->keybuf, tfm_ctx->keylen, &pk, tested); 371 363 372 364 /* update context */ 373 365 spin_lock_bh(&tfm_ctx->pk_lock); ··· 412 404 struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx; 413 405 struct hash_walk_helper *hwh = &req_ctx->hwh; 414 406 unsigned int bs = crypto_ahash_blocksize(tfm); 407 407 + bool tested = crypto_ahash_tested(tfm); 415 408 unsigned int offset, k, n; 416 409 int rc = 0; 417 410 ··· 453 444 rc = -EKEYEXPIRED; 454 445 goto out; 455 446 } 456 456 - rc = phmac_convert_key(tfm_ctx); 447 447 + rc = phmac_convert_key(tfm_ctx, tested); 457 448 if (rc) 458 449 goto out; 459 450 spin_lock_bh(&tfm_ctx->pk_lock); ··· 489 480 rc = -EKEYEXPIRED; 490 481 goto out; 491 482 } 492 492 - rc = phmac_convert_key(tfm_ctx); 483 483 + rc = phmac_convert_key(tfm_ctx, tested); 493 484 if (rc) 494 485 goto out; 495 486 spin_lock_bh(&tfm_ctx->pk_lock); ··· 526 517 struct kmac_sha2_ctx *ctx = &req_ctx->kmac_ctx; 527 518 unsigned int ds = crypto_ahash_digestsize(tfm); 528 519 unsigned int bs = crypto_ahash_blocksize(tfm); 520 520 + bool tested = crypto_ahash_tested(tfm); 529 521 unsigned int k, n; 530 522 int rc = 0; 531 523 ··· 547 537 rc = -EKEYEXPIRED; 548 538 goto out; 549 539 } 550 550 - rc = phmac_convert_key(tfm_ctx); 540 540 + rc = phmac_convert_key(tfm_ctx, tested); 551 541 if (rc) 552 542 goto out; 553 543 spin_lock_bh(&tfm_ctx->pk_lock); ··· 751 741 struct phmac_tfm_ctx *tfm_ctx = crypto_ahash_ctx(tfm); 752 742 unsigned int ds = crypto_ahash_digestsize(tfm); 753 743 unsigned int bs = crypto_ahash_blocksize(tfm); 744 744 + bool tested = crypto_ahash_tested(tfm); 754 745 unsigned int tmpkeylen; 755 746 u8 *tmpkey = NULL; 756 747 int rc = 0; 757 748 758 758 - if (!crypto_ahash_tested(tfm)) { 749 749 + if (!tested) { 759 750 /* 760 751 * selftest running: key is a raw hmac clear key and needs 761 752 * to get embedded into a 'clear key token' in order to have ··· 781 770 goto out; 782 771 783 772 /* convert raw key into protected key */ 784 784 - rc = phmac_convert_key(tfm_ctx); 773 773 + rc = phmac_convert_key(tfm_ctx, tested); 785 774 if (rc) 786 775 goto out; 787 776

+7 -1

arch/s390/include/asm/pkey.h

reviewed

··· 21 21 * @param keylen size of the key blob in bytes 22 22 * @param protkey pointer to buffer receiving the protected key 23 23 * @param xflags additional execution flags (see PKEY_XFLAG_* definitions below) 24 24 - * As of now the only supported flag is PKEY_XFLAG_NOMEMALLOC. 24 24 + * As of now the only supported flags are PKEY_XFLAG_NOMEMALLOC 25 25 + * and PKEY_XFLAG_NOCLEARKEY. 25 26 * @return 0 on success, negative errno value on failure 26 27 */ 27 28 int pkey_key2protkey(const u8 *key, u32 keylen, ··· 38 37 * also the CRYPTO_ALG_ALLOCATES_MEMORY flag in crypto.h. 39 38 */ 40 39 #define PKEY_XFLAG_NOMEMALLOC 0x0001 40 40 + 41 41 + /* 42 42 + * Do not accept a clear key token as source for a protected key. 43 43 + */ 44 44 + #define PKEY_XFLAG_NOCLEARKEY 0x0002 41 45 42 46 #endif /* _KAPI_PKEY_H */

+6 -12

crypto/acompress.c

reviewed

··· 60 60 return nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, sizeof(racomp), &racomp); 61 61 } 62 62 63 63 - static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg) 64 64 - __maybe_unused; 65 65 - 66 66 - static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg) 63 63 + static void __maybe_unused crypto_acomp_show(struct seq_file *m, 64 64 + struct crypto_alg *alg) 67 65 { 68 66 seq_puts(m, "type : acomp\n"); 69 67 } ··· 335 337 336 338 for (i = 0; i < count; i++) { 337 339 ret = crypto_register_acomp(&algs[i]); 338 338 - if (ret) 339 339 - goto err; 340 340 + if (ret) { 341 341 + crypto_unregister_acomps(algs, i); 342 342 + return ret; 343 343 + } 340 344 } 341 345 342 346 return 0; 343 343 - 344 344 - err: 345 345 - for (--i; i >= 0; --i) 346 346 - crypto_unregister_acomp(&algs[i]); 347 347 - 348 348 - return ret; 349 347 } 350 348 EXPORT_SYMBOL_GPL(crypto_register_acomps); 351 349

+2 -3

crypto/aead.c

reviewed

··· 151 151 return nla_put(skb, CRYPTOCFGA_REPORT_AEAD, sizeof(raead), &raead); 152 152 } 153 153 154 154 - static void crypto_aead_show(struct seq_file *m, struct crypto_alg *alg) 155 155 - __maybe_unused; 156 156 - static void crypto_aead_show(struct seq_file *m, struct crypto_alg *alg) 154 154 + static void __maybe_unused crypto_aead_show(struct seq_file *m, 155 155 + struct crypto_alg *alg) 157 156 { 158 157 struct aead_alg *aead = container_of(alg, struct aead_alg, base); 159 158

+6 -11

crypto/ahash.c

reviewed

··· 801 801 return nla_put(skb, CRYPTOCFGA_REPORT_HASH, sizeof(rhash), &rhash); 802 802 } 803 803 804 804 - static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) 805 805 - __maybe_unused; 806 806 - static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) 804 804 + static void __maybe_unused crypto_ahash_show(struct seq_file *m, 805 805 + struct crypto_alg *alg) 807 806 { 808 807 seq_printf(m, "type : ahash\n"); 809 808 seq_printf(m, "async : %s\n", ··· 1019 1020 1020 1021 for (i = 0; i < count; i++) { 1021 1022 ret = crypto_register_ahash(&algs[i]); 1022 1022 - if (ret) 1023 1023 - goto err; 1023 1023 + if (ret) { 1024 1024 + crypto_unregister_ahashes(algs, i); 1025 1025 + return ret; 1026 1026 + } 1024 1027 } 1025 1028 1026 1029 return 0; 1027 1027 - 1028 1028 - err: 1029 1029 - for (--i; i >= 0; --i) 1030 1030 - crypto_unregister_ahash(&algs[i]); 1031 1031 - 1032 1032 - return ret; 1033 1030 } 1034 1031 EXPORT_SYMBOL_GPL(crypto_register_ahashes); 1035 1032

+2 -4

crypto/akcipher.c

reviewed

··· 46 46 sizeof(rakcipher), &rakcipher); 47 47 } 48 48 49 49 - static void crypto_akcipher_show(struct seq_file *m, struct crypto_alg *alg) 50 50 - __maybe_unused; 51 51 - 52 52 - static void crypto_akcipher_show(struct seq_file *m, struct crypto_alg *alg) 49 49 + static void __maybe_unused crypto_akcipher_show(struct seq_file *m, 50 50 + struct crypto_alg *alg) 53 51 { 54 52 seq_puts(m, "type : akcipher\n"); 55 53 }

+5 -9

crypto/algapi.c

reviewed

··· 511 511 512 512 for (i = 0; i < count; i++) { 513 513 ret = crypto_register_alg(&algs[i]); 514 514 - if (ret) 515 515 - goto err; 514 514 + if (ret) { 515 515 + crypto_unregister_algs(algs, i); 516 516 + return ret; 517 517 + } 516 518 } 517 519 518 520 return 0; 519 519 - 520 520 - err: 521 521 - for (--i; i >= 0; --i) 522 522 - crypto_unregister_alg(&algs[i]); 523 523 - 524 524 - return ret; 525 521 } 526 522 EXPORT_SYMBOL_GPL(crypto_register_algs); 527 523 ··· 525 529 { 526 530 int i; 527 531 528 528 - for (i = 0; i < count; i++) 532 532 + for (i = count - 1; i >= 0; --i) 529 533 crypto_unregister_alg(&algs[i]); 530 534 } 531 535 EXPORT_SYMBOL_GPL(crypto_unregister_algs);

+1 -1

crypto/blowfish_common.c

reviewed

··· 306 306 307 307 /* 308 308 * The blowfish encipher, processes 64-bit blocks. 309 309 - * NOTE: This function MUSTN'T respect endianess 309 309 + * NOTE: This function MUSTN'T respect endianness 310 310 */ 311 311 static void encrypt_block(struct bf_ctx *bctx, u32 *dst, u32 *src) 312 312 {

+12 -21

crypto/crypto_engine.c

reviewed

··· 524 524 525 525 for (i = 0; i < count; i++) { 526 526 ret = crypto_engine_register_aead(&algs[i]); 527 527 - if (ret) 528 528 - goto err; 527 527 + if (ret) { 528 528 + crypto_engine_unregister_aeads(algs, i); 529 529 + return ret; 530 530 + } 529 531 } 530 532 531 533 return 0; 532 532 - 533 533 - err: 534 534 - crypto_engine_unregister_aeads(algs, i); 535 535 - 536 536 - return ret; 537 534 } 538 535 EXPORT_SYMBOL_GPL(crypto_engine_register_aeads); 539 536 ··· 563 566 564 567 for (i = 0; i < count; i++) { 565 568 ret = crypto_engine_register_ahash(&algs[i]); 566 566 - if (ret) 567 567 - goto err; 569 569 + if (ret) { 570 570 + crypto_engine_unregister_ahashes(algs, i); 571 571 + return ret; 572 572 + } 568 573 } 569 574 570 575 return 0; 571 571 - 572 572 - err: 573 573 - crypto_engine_unregister_ahashes(algs, i); 574 574 - 575 575 - return ret; 576 576 } 577 577 EXPORT_SYMBOL_GPL(crypto_engine_register_ahashes); 578 578 ··· 632 638 633 639 for (i = 0; i < count; i++) { 634 640 ret = crypto_engine_register_skcipher(&algs[i]); 635 635 - if (ret) 636 636 - goto err; 641 641 + if (ret) { 642 642 + crypto_engine_unregister_skciphers(algs, i); 643 643 + return ret; 644 644 + } 637 645 } 638 646 639 647 return 0; 640 640 - 641 641 - err: 642 642 - crypto_engine_unregister_skciphers(algs, i); 643 643 - 644 644 - return ret; 645 648 } 646 649 EXPORT_SYMBOL_GPL(crypto_engine_register_skciphers); 647 650

+17 -32

crypto/drbg.c

reviewed

··· 226 226 * @entropy buffer of seed data to be checked 227 227 * 228 228 * return: 229 229 - * 0 on success 230 230 - * -EAGAIN on when the CTRNG is not yet primed 231 231 - * < 0 on error 229 229 + * %true on success 230 230 + * %false when the CTRNG is not yet primed 232 231 */ 233 233 - static int drbg_fips_continuous_test(struct drbg_state *drbg, 234 234 - const unsigned char *entropy) 232 232 + static bool drbg_fips_continuous_test(struct drbg_state *drbg, 233 233 + const unsigned char *entropy) 235 234 { 236 235 unsigned short entropylen = drbg_sec_strength(drbg->core->flags); 237 237 - int ret = 0; 238 236 239 237 if (!IS_ENABLED(CONFIG_CRYPTO_FIPS)) 240 240 - return 0; 238 238 + return true; 241 239 242 240 /* skip test if we test the overall system */ 243 241 if (list_empty(&drbg->test_data.list)) 244 244 - return 0; 242 242 + return true; 245 243 /* only perform test in FIPS mode */ 246 244 if (!fips_enabled) 247 247 - return 0; 245 245 + return true; 248 246 249 247 if (!drbg->fips_primed) { 250 248 /* Priming of FIPS test */ 251 249 memcpy(drbg->prev, entropy, entropylen); 252 250 drbg->fips_primed = true; 253 251 /* priming: another round is needed */ 254 254 - return -EAGAIN; 252 252 + return false; 255 253 } 256 256 - ret = memcmp(drbg->prev, entropy, entropylen); 257 257 - if (!ret) 254 254 + if (!memcmp(drbg->prev, entropy, entropylen)) 258 255 panic("DRBG continuous self test failed\n"); 259 256 memcpy(drbg->prev, entropy, entropylen); 260 257 261 258 /* the test shall pass when the two values are not equal */ 262 262 - return 0; 259 259 + return true; 263 260 } 264 261 265 262 /****************************************************************** ··· 842 845 return ret; 843 846 } 844 847 845 845 - static inline int drbg_get_random_bytes(struct drbg_state *drbg, 846 846 - unsigned char *entropy, 847 847 - unsigned int entropylen) 848 848 + static inline void drbg_get_random_bytes(struct drbg_state *drbg, 849 849 + unsigned char *entropy, 850 850 + unsigned int entropylen) 848 851 { 849 849 - int ret; 850 850 - 851 851 - do { 852 852 + do 852 853 get_random_bytes(entropy, entropylen); 853 853 - ret = drbg_fips_continuous_test(drbg, entropy); 854 854 - if (ret && ret != -EAGAIN) 855 855 - return ret; 856 856 - } while (ret); 857 857 - 858 858 - return 0; 854 854 + while (!drbg_fips_continuous_test(drbg, entropy)); 859 855 } 860 856 861 857 static int drbg_seed_from_random(struct drbg_state *drbg) ··· 865 875 drbg_string_fill(&data, entropy, entropylen); 866 876 list_add_tail(&data.list, &seedlist); 867 877 868 868 - ret = drbg_get_random_bytes(drbg, entropy, entropylen); 869 869 - if (ret) 870 870 - goto out; 878 878 + drbg_get_random_bytes(drbg, entropy, entropylen); 871 879 872 880 ret = __drbg_seed(drbg, &seedlist, true, DRBG_SEED_STATE_FULL); 873 881 874 874 - out: 875 882 memzero_explicit(entropy, entropylen); 876 883 return ret; 877 884 } ··· 943 956 if (!rng_is_initialized()) 944 957 new_seed_state = DRBG_SEED_STATE_PARTIAL; 945 958 946 946 - ret = drbg_get_random_bytes(drbg, entropy, entropylen); 947 947 - if (ret) 948 948 - goto out; 959 959 + drbg_get_random_bytes(drbg, entropy, entropylen); 949 960 950 961 if (!drbg->jent) { 951 962 drbg_string_fill(&data1, entropy, entropylen);

+10 -19

crypto/ecc.c

reviewed

··· 90 90 } 91 91 EXPORT_SYMBOL(ecc_digits_from_bytes); 92 92 93 93 - static u64 *ecc_alloc_digits_space(unsigned int ndigits) 94 94 - { 95 95 - size_t len = ndigits * sizeof(u64); 96 96 - 97 97 - if (!len) 98 98 - return NULL; 99 99 - 100 100 - return kmalloc(len, GFP_KERNEL); 101 101 - } 102 102 - 103 103 - static void ecc_free_digits_space(u64 *space) 104 104 - { 105 105 - kfree_sensitive(space); 106 106 - } 107 107 - 108 93 struct ecc_point *ecc_alloc_point(unsigned int ndigits) 109 94 { 110 110 - struct ecc_point *p = kmalloc(sizeof(*p), GFP_KERNEL); 95 95 + struct ecc_point *p; 96 96 + size_t ndigits_sz; 111 97 98 98 + if (!ndigits) 99 99 + return NULL; 100 100 + 101 101 + p = kmalloc(sizeof(*p), GFP_KERNEL); 112 102 if (!p) 113 103 return NULL; 114 104 115 115 - p->x = ecc_alloc_digits_space(ndigits); 105 105 + ndigits_sz = ndigits * sizeof(u64); 106 106 + p->x = kmalloc(ndigits_sz, GFP_KERNEL); 116 107 if (!p->x) 117 108 goto err_alloc_x; 118 109 119 119 - p->y = ecc_alloc_digits_space(ndigits); 110 110 + p->y = kmalloc(ndigits_sz, GFP_KERNEL); 120 111 if (!p->y) 121 112 goto err_alloc_y; 122 113 ··· 116 125 return p; 117 126 118 127 err_alloc_y: 119 119 - ecc_free_digits_space(p->x); 128 128 + kfree(p->x); 120 129 err_alloc_x: 121 130 kfree(p); 122 131 return NULL;

+1 -1

crypto/fips.c

reviewed

··· 22 22 EXPORT_SYMBOL_GPL(fips_fail_notif_chain); 23 23 24 24 /* Process kernel command-line parameter at boot time. fips=0 or fips=1 */ 25 25 - static int fips_enable(char *str) 25 25 + static int __init fips_enable(char *str) 26 26 { 27 27 if (kstrtoint(str, 0, &fips_enabled)) 28 28 return 0;

+1 -4

crypto/khazad.c

reviewed

··· 859 859 860 860 static int __init khazad_mod_init(void) 861 861 { 862 862 - int ret = 0; 863 863 - 864 864 - ret = crypto_register_alg(&khazad_alg); 865 865 - return ret; 862 862 + return crypto_register_alg(&khazad_alg); 866 863 } 867 864 868 865 static void __exit khazad_mod_fini(void)

+2 -4

crypto/kpp.c

reviewed

··· 29 29 return nla_put(skb, CRYPTOCFGA_REPORT_KPP, sizeof(rkpp), &rkpp); 30 30 } 31 31 32 32 - static void crypto_kpp_show(struct seq_file *m, struct crypto_alg *alg) 33 33 - __maybe_unused; 34 34 - 35 35 - static void crypto_kpp_show(struct seq_file *m, struct crypto_alg *alg) 32 32 + static void __maybe_unused crypto_kpp_show(struct seq_file *m, 33 33 + struct crypto_alg *alg) 36 34 { 37 35 seq_puts(m, "type : kpp\n"); 38 36 }

+4 -8

crypto/lskcipher.c

reviewed

··· 384 384 385 385 for (i = 0; i < count; i++) { 386 386 ret = crypto_register_lskcipher(&algs[i]); 387 387 - if (ret) 388 388 - goto err; 387 387 + if (ret) { 388 388 + crypto_unregister_lskciphers(algs, i); 389 389 + return ret; 390 390 + } 389 391 } 390 392 391 393 return 0; 392 392 - 393 393 - err: 394 394 - for (--i; i >= 0; --i) 395 395 - crypto_unregister_lskcipher(&algs[i]); 396 396 - 397 397 - return ret; 398 394 } 399 395 EXPORT_SYMBOL_GPL(crypto_register_lskciphers); 400 396

+6 -11

crypto/rng.c

reviewed

··· 77 77 return nla_put(skb, CRYPTOCFGA_REPORT_RNG, sizeof(rrng), &rrng); 78 78 } 79 79 80 80 - static void crypto_rng_show(struct seq_file *m, struct crypto_alg *alg) 81 81 - __maybe_unused; 82 82 - static void crypto_rng_show(struct seq_file *m, struct crypto_alg *alg) 80 80 + static void __maybe_unused crypto_rng_show(struct seq_file *m, 81 81 + struct crypto_alg *alg) 83 82 { 84 83 seq_printf(m, "type : rng\n"); 85 84 seq_printf(m, "seedsize : %u\n", seedsize(alg)); ··· 202 203 203 204 for (i = 0; i < count; i++) { 204 205 ret = crypto_register_rng(algs + i); 205 205 - if (ret) 206 206 - goto err; 206 206 + if (ret) { 207 207 + crypto_unregister_rngs(algs, i); 208 208 + return ret; 209 209 + } 207 210 } 208 211 209 212 return 0; 210 210 - 211 211 - err: 212 212 - for (--i; i >= 0; --i) 213 213 - crypto_unregister_rng(algs + i); 214 214 - 215 215 - return ret; 216 213 } 217 214 EXPORT_SYMBOL_GPL(crypto_register_rngs); 218 215

+6 -12

crypto/scompress.c

reviewed

··· 58 58 sizeof(rscomp), &rscomp); 59 59 } 60 60 61 61 - static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) 62 62 - __maybe_unused; 63 63 - 64 64 - static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) 61 61 + static void __maybe_unused crypto_scomp_show(struct seq_file *m, 62 62 + struct crypto_alg *alg) 65 63 { 66 64 seq_puts(m, "type : scomp\n"); 67 65 } ··· 381 383 382 384 for (i = 0; i < count; i++) { 383 385 ret = crypto_register_scomp(&algs[i]); 384 384 - if (ret) 385 385 - goto err; 386 386 + if (ret) { 387 387 + crypto_unregister_scomps(algs, i); 388 388 + return ret; 389 389 + } 386 390 } 387 391 388 392 return 0; 389 389 - 390 390 - err: 391 391 - for (--i; i >= 0; --i) 392 392 - crypto_unregister_scomp(&algs[i]); 393 393 - 394 394 - return ret; 395 393 } 396 394 EXPORT_SYMBOL_GPL(crypto_register_scomps); 397 395

+6 -11

crypto/shash.c

reviewed

··· 346 346 return nla_put(skb, CRYPTOCFGA_REPORT_HASH, sizeof(rhash), &rhash); 347 347 } 348 348 349 349 - static void crypto_shash_show(struct seq_file *m, struct crypto_alg *alg) 350 350 - __maybe_unused; 351 351 - static void crypto_shash_show(struct seq_file *m, struct crypto_alg *alg) 349 349 + static void __maybe_unused crypto_shash_show(struct seq_file *m, 350 350 + struct crypto_alg *alg) 352 351 { 353 352 struct shash_alg *salg = __crypto_shash_alg(alg); 354 353 ··· 541 542 542 543 for (i = 0; i < count; i++) { 543 544 ret = crypto_register_shash(&algs[i]); 544 544 - if (ret) 545 545 - goto err; 545 545 + if (ret) { 546 546 + crypto_unregister_shashes(algs, i); 547 547 + return ret; 548 548 + } 546 549 } 547 550 548 551 return 0; 549 549 - 550 550 - err: 551 551 - for (--i; i >= 0; --i) 552 552 - crypto_unregister_shash(&algs[i]); 553 553 - 554 554 - return ret; 555 552 } 556 553 EXPORT_SYMBOL_GPL(crypto_register_shashes); 557 554

+2 -2

crypto/simd.c

reviewed

··· 352 352 353 353 ctx->cryptd_tfm = cryptd_tfm; 354 354 355 355 - reqsize = crypto_aead_reqsize(cryptd_aead_child(cryptd_tfm)); 356 356 - reqsize = max(reqsize, crypto_aead_reqsize(&cryptd_tfm->base)); 355 355 + reqsize = max(crypto_aead_reqsize(cryptd_aead_child(cryptd_tfm)), 356 356 + crypto_aead_reqsize(&cryptd_tfm->base)); 357 357 reqsize += sizeof(struct aead_request); 358 358 359 359 crypto_aead_set_reqsize(tfm, reqsize);

+6 -11

crypto/skcipher.c

reviewed

··· 570 570 skcipher->free(skcipher); 571 571 } 572 572 573 573 - static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg) 574 574 - __maybe_unused; 575 575 - static void crypto_skcipher_show(struct seq_file *m, struct crypto_alg *alg) 573 573 + static void __maybe_unused crypto_skcipher_show(struct seq_file *m, 574 574 + struct crypto_alg *alg) 576 575 { 577 576 struct skcipher_alg *skcipher = __crypto_skcipher_alg(alg); 578 577 ··· 740 741 741 742 for (i = 0; i < count; i++) { 742 743 ret = crypto_register_skcipher(&algs[i]); 743 743 - if (ret) 744 744 - goto err; 744 744 + if (ret) { 745 745 + crypto_unregister_skciphers(algs, i); 746 746 + return ret; 747 747 + } 745 748 } 746 749 747 750 return 0; 748 748 - 749 749 - err: 750 750 - for (--i; i >= 0; --i) 751 751 - crypto_unregister_skcipher(&algs[i]); 752 752 - 753 753 - return ret; 754 751 } 755 752 EXPORT_SYMBOL_GPL(crypto_register_skciphers); 756 753

+25

crypto/testmgr.c

reviewed

··· 4080 4080 .aead = __VECS(aegis128_tv_template) 4081 4081 } 4082 4082 }, { 4083 4083 + .alg = "authenc(hmac(md5),cbc(des3_ede))", 4084 4084 + .generic_driver = "authenc(hmac-md5-lib,cbc(des3_ede-generic))", 4085 4085 + .test = alg_test_aead, 4086 4086 + .suite = { 4087 4087 + .aead = __VECS(hmac_md5_des3_ede_cbc_tv_temp) 4088 4088 + } 4089 4089 + }, { 4083 4090 .alg = "authenc(hmac(md5),ecb(cipher_null))", 4084 4091 .generic_driver = "authenc(hmac-md5-lib,ecb-cipher_null)", 4085 4092 .test = alg_test_aead, ··· 4131 4124 .test = alg_test_null, 4132 4125 .fips_allowed = 1, 4133 4126 }, { 4127 4127 + .alg = "authenc(hmac(sha224),cbc(aes))", 4128 4128 + .generic_driver = "authenc(hmac-sha224-lib,cbc(aes-generic))", 4129 4129 + .test = alg_test_aead, 4130 4130 + .suite = { 4131 4131 + .aead = __VECS(hmac_sha224_aes_cbc_tv_temp) 4132 4132 + } 4133 4133 + }, { 4134 4134 .alg = "authenc(hmac(sha224),cbc(des))", 4135 4135 .generic_driver = "authenc(hmac-sha224-lib,cbc(des-generic))", 4136 4136 .test = alg_test_aead, ··· 4151 4137 .suite = { 4152 4138 .aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp) 4153 4139 } 4140 4140 + }, { 4141 4141 + .alg = "authenc(hmac(sha224),rfc3686(ctr(aes)))", 4142 4142 + .test = alg_test_null, 4143 4143 + .fips_allowed = 1, 4154 4144 }, { 4155 4145 .alg = "authenc(hmac(sha256),cbc(aes))", 4156 4146 .generic_driver = "authenc(hmac-sha256-lib,cbc(aes-lib))", ··· 4192 4174 .alg = "authenc(hmac(sha256),rfc3686(ctr(aes)))", 4193 4175 .test = alg_test_null, 4194 4176 .fips_allowed = 1, 4177 4177 + }, { 4178 4178 + .alg = "authenc(hmac(sha384),cbc(aes))", 4179 4179 + .generic_driver = "authenc(hmac-sha384-lib,cbc(aes-generic))", 4180 4180 + .test = alg_test_aead, 4181 4181 + .suite = { 4182 4182 + .aead = __VECS(hmac_sha384_aes_cbc_tv_temp) 4183 4183 + } 4195 4184 }, { 4196 4185 .alg = "authenc(hmac(sha384),cbc(des))", 4197 4186 .generic_driver = "authenc(hmac-sha384-lib,cbc(des-generic))",

+655

crypto/testmgr.h

reviewed

··· 14919 14919 }, 14920 14920 }; 14921 14921 14922 14922 + static const struct aead_testvec hmac_sha224_aes_cbc_tv_temp[] = { 14923 14923 + { /* RFC 3602 Case 1 */ 14924 14924 + #ifdef __LITTLE_ENDIAN 14925 14925 + .key = "\x08\x00" /* rta length */ 14926 14926 + "\x01\x00" /* rta type */ 14927 14927 + #else 14928 14928 + .key = "\x00\x08" /* rta length */ 14929 14929 + "\x00\x01" /* rta type */ 14930 14930 + #endif 14931 14931 + "\x00\x00\x00\x10" /* enc key length */ 14932 14932 + "\x00\x00\x00\x00\x00\x00\x00\x00" 14933 14933 + "\x00\x00\x00\x00\x00\x00\x00\x00" 14934 14934 + "\x00\x00\x00\x00\x00\x00\x00\x00" 14935 14935 + "\x00\x00\x00\x00" 14936 14936 + "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" 14937 14937 + "\x51\x2e\x03\xd5\x34\x12\x00\x06", 14938 14938 + .klen = 8 + 28 + 16, 14939 14939 + .iv = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30" 14940 14940 + "\xb4\x22\xda\x80\x2c\x9f\xac\x41", 14941 14941 + .assoc = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30" 14942 14942 + "\xb4\x22\xda\x80\x2c\x9f\xac\x41", 14943 14943 + .alen = 16, 14944 14944 + .ptext = "Single block msg", 14945 14945 + .plen = 16, 14946 14946 + .ctext = "\xe3\x53\x77\x9c\x10\x79\xae\xb8" 14947 14947 + "\x27\x08\x94\x2d\xbe\x77\x18\x1a" 14948 14948 + "\x17\xe8\x00\x76\x70\x71\xd1\x72" 14949 14949 + "\xf8\xd0\x91\x51\x67\xf9\xdf\xd6" 14950 14950 + "\x0d\x56\x1a\xb3\x52\x19\x85\xae" 14951 14951 + "\x46\x74\xb6\x98", 14952 14952 + .clen = 16 + 28, 14953 14953 + }, { /* RFC 3602 Case 2 */ 14954 14954 + #ifdef __LITTLE_ENDIAN 14955 14955 + .key = "\x08\x00" /* rta length */ 14956 14956 + "\x01\x00" /* rta type */ 14957 14957 + #else 14958 14958 + .key = "\x00\x08" /* rta length */ 14959 14959 + "\x00\x01" /* rta type */ 14960 14960 + #endif 14961 14961 + "\x00\x00\x00\x10" /* enc key length */ 14962 14962 + "\x20\x21\x22\x23\x24\x25\x26\x27" 14963 14963 + "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" 14964 14964 + "\x30\x31\x32\x33\x34\x35\x36\x37" 14965 14965 + "\x38\x39\x3a\x3b" 14966 14966 + "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0" 14967 14967 + "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a", 14968 14968 + .klen = 8 + 28 + 16, 14969 14969 + .iv = "\x56\x2e\x17\x99\x6d\x09\x3d\x28" 14970 14970 + "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58", 14971 14971 + .assoc = "\x56\x2e\x17\x99\x6d\x09\x3d\x28" 14972 14972 + "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58", 14973 14973 + .alen = 16, 14974 14974 + .ptext = "\x00\x01\x02\x03\x04\x05\x06\x07" 14975 14975 + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" 14976 14976 + "\x10\x11\x12\x13\x14\x15\x16\x17" 14977 14977 + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", 14978 14978 + .plen = 32, 14979 14979 + .ctext = "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a" 14980 14980 + "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a" 14981 14981 + "\x75\x86\x60\x2d\x25\x3c\xff\xf9" 14982 14982 + "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1" 14983 14983 + "\xa1\x11\xfa\xbb\x1e\x04\x7e\xe7" 14984 14984 + "\x4c\x5f\x65\xbf\x68\x8d\x33\x9d" 14985 14985 + "\xbc\x74\x9b\xf3\x15\xf3\x8f\x8d" 14986 14986 + "\xe8\xaf\x33\xe0", 14987 14987 + 14988 14988 + .clen = 32 + 28, 14989 14989 + }, { /* RFC 3602 Case 3 */ 14990 14990 + #ifdef __LITTLE_ENDIAN 14991 14991 + .key = "\x08\x00" /* rta length */ 14992 14992 + "\x01\x00" /* rta type */ 14993 14993 + #else 14994 14994 + .key = "\x00\x08" /* rta length */ 14995 14995 + "\x00\x01" /* rta type */ 14996 14996 + #endif 14997 14997 + "\x00\x00\x00\x10" /* enc key length */ 14998 14998 + "\x11\x22\x33\x44\x55\x66\x77\x88" 14999 14999 + "\x99\xaa\xbb\xcc\xdd\xee\xff\x11" 15000 15000 + "\x22\x33\x44\x55\x66\x77\x88\x99" 15001 15001 + "\xaa\xbb\xcc\xdd" 15002 15002 + "\x6c\x3e\xa0\x47\x76\x30\xce\x21" 15003 15003 + "\xa2\xce\x33\x4a\xa7\x46\xc2\xcd", 15004 15004 + .klen = 8 + 28 + 16, 15005 15005 + .iv = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb" 15006 15006 + "\xd9\xcd\x27\xd8\x25\x68\x2c\x81", 15007 15007 + .assoc = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb" 15008 15008 + "\xd9\xcd\x27\xd8\x25\x68\x2c\x81", 15009 15009 + .alen = 16, 15010 15010 + .ptext = "This is a 48-byte message (exactly 3 AES blocks)", 15011 15011 + .plen = 48, 15012 15012 + .ctext = "\xd0\xa0\x2b\x38\x36\x45\x17\x53" 15013 15013 + "\xd4\x93\x66\x5d\x33\xf0\xe8\x86" 15014 15014 + "\x2d\xea\x54\xcd\xb2\x93\xab\xc7" 15015 15015 + "\x50\x69\x39\x27\x67\x72\xf8\xd5" 15016 15016 + "\x02\x1c\x19\x21\x6b\xad\x52\x5c" 15017 15017 + "\x85\x79\x69\x5d\x83\xba\x26\x84" 15018 15018 + "\x60\xb3\xca\x0e\xc1\xfe\xf2\x27" 15019 15019 + "\x5a\x41\xe4\x99\xa8\x19\x56\xf1" 15020 15020 + "\x44\x98\x27\x9f\x99\xb0\x4a\xad" 15021 15021 + "\x4d\xc1\x1e\x88", 15022 15022 + .clen = 48 + 28, 15023 15023 + }, { /* RFC 3602 Case 4 */ 15024 15024 + #ifdef __LITTLE_ENDIAN 15025 15025 + .key = "\x08\x00" /* rta length */ 15026 15026 + "\x01\x00" /* rta type */ 15027 15027 + #else 15028 15028 + .key = "\x00\x08" /* rta length */ 15029 15029 + "\x00\x01" /* rta type */ 15030 15030 + #endif 15031 15031 + "\x00\x00\x00\x10" /* enc key length */ 15032 15032 + "\x11\x22\x33\x44\x55\x66\x77\x88" 15033 15033 + "\x99\xaa\xbb\xcc\xdd\xee\xff\x11" 15034 15034 + "\x22\x33\x44\x55\x66\x77\x88\x99" 15035 15035 + "\xaa\xbb\xcc\xdd" 15036 15036 + "\x56\xe4\x7a\x38\xc5\x59\x89\x74" 15037 15037 + "\xbc\x46\x90\x3d\xba\x29\x03\x49", 15038 15038 + .klen = 8 + 28 + 16, 15039 15039 + .iv = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c" 15040 15040 + "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9", 15041 15041 + .assoc = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c" 15042 15042 + "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9", 15043 15043 + .alen = 16, 15044 15044 + .ptext = "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7" 15045 15045 + "\xa8\xa9\xaa\xab\xac\xad\xae\xaf" 15046 15046 + "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" 15047 15047 + "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf" 15048 15048 + "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" 15049 15049 + "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf" 15050 15050 + "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7" 15051 15051 + "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf", 15052 15052 + .plen = 64, 15053 15053 + .ctext = "\xc3\x0e\x32\xff\xed\xc0\x77\x4e" 15054 15054 + "\x6a\xff\x6a\xf0\x86\x9f\x71\xaa" 15055 15055 + "\x0f\x3a\xf0\x7a\x9a\x31\xa9\xc6" 15056 15056 + "\x84\xdb\x20\x7e\xb0\xef\x8e\x4e" 15057 15057 + "\x35\x90\x7a\xa6\x32\xc3\xff\xdf" 15058 15058 + "\x86\x8b\xb7\xb2\x9d\x3d\x46\xad" 15059 15059 + "\x83\xce\x9f\x9a\x10\x2e\xe9\x9d" 15060 15060 + "\x49\xa5\x3e\x87\xf4\xc3\xda\x55" 15061 15061 + "\xbb\xe9\x38\xf8\xb9\xbf\xcb\x7b" 15062 15062 + "\xa8\x22\x91\xea\x1e\xaf\x13\xba" 15063 15063 + "\x24\x18\x64\x9c\xcb\xb4\xa9\x16" 15064 15064 + "\x4b\x83\x9c\xec", 15065 15065 + .clen = 64 + 28, 15066 15066 + }, { /* RFC 3602 Case 5 */ 15067 15067 + #ifdef __LITTLE_ENDIAN 15068 15068 + .key = "\x08\x00" /* rta length */ 15069 15069 + "\x01\x00" /* rta type */ 15070 15070 + #else 15071 15071 + .key = "\x00\x08" /* rta length */ 15072 15072 + "\x00\x01" /* rta type */ 15073 15073 + #endif 15074 15074 + "\x00\x00\x00\x10" /* enc key length */ 15075 15075 + "\x11\x22\x33\x44\x55\x66\x77\x88" 15076 15076 + "\x99\xaa\xbb\xcc\xdd\xee\xff\x11" 15077 15077 + "\x22\x33\x44\x55\x66\x77\x88\x99" 15078 15078 + "\xaa\xbb\xcc\xdd" 15079 15079 + "\x90\xd3\x82\xb4\x10\xee\xba\x7a" 15080 15080 + "\xd9\x38\xc4\x6c\xec\x1a\x82\xbf", 15081 15081 + .klen = 8 + 28 + 16, 15082 15082 + .iv = "\xe9\x6e\x8c\x08\xab\x46\x57\x63" 15083 15083 + "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93", 15084 15084 + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" 15085 15085 + "\xe9\x6e\x8c\x08\xab\x46\x57\x63" 15086 15086 + "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93", 15087 15087 + .alen = 24, 15088 15088 + .ptext = "\x08\x00\x0e\xbd\xa7\x0a\x00\x00" 15089 15089 + "\x8e\x9c\x08\x3d\xb9\x5b\x07\x00" 15090 15090 + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" 15091 15091 + "\x10\x11\x12\x13\x14\x15\x16\x17" 15092 15092 + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" 15093 15093 + "\x20\x21\x22\x23\x24\x25\x26\x27" 15094 15094 + "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" 15095 15095 + "\x30\x31\x32\x33\x34\x35\x36\x37" 15096 15096 + "\x01\x02\x03\x04\x05\x06\x07\x08" 15097 15097 + "\x09\x0a\x0b\x0c\x0d\x0e\x0e\x01", 15098 15098 + .plen = 80, 15099 15099 + .ctext = "\xf6\x63\xc2\x5d\x32\x5c\x18\xc6" 15100 15100 + "\xa9\x45\x3e\x19\x4e\x12\x08\x49" 15101 15101 + "\xa4\x87\x0b\x66\xcc\x6b\x99\x65" 15102 15102 + "\x33\x00\x13\xb4\x89\x8d\xc8\x56" 15103 15103 + "\xa4\x69\x9e\x52\x3a\x55\xdb\x08" 15104 15104 + "\x0b\x59\xec\x3a\x8e\x4b\x7e\x52" 15105 15105 + "\x77\x5b\x07\xd1\xdb\x34\xed\x9c" 15106 15106 + "\x53\x8a\xb5\x0c\x55\x1b\x87\x4a" 15107 15107 + "\xa2\x69\xad\xd0\x47\xad\x2d\x59" 15108 15108 + "\x13\xac\x19\xb7\xcf\xba\xd4\xa6" 15109 15109 + "\x04\x5e\x83\x45\xc5\x6a\x5b\xe2" 15110 15110 + "\x5e\xd8\x59\x06\xbd\xc7\xd2\x9b" 15111 15111 + "\x0b\x65\x1f\x31\xc7\xe6\x9c\x39" 15112 15112 + "\xa3\x66\xdb\xb8", 15113 15113 + .clen = 80 + 28, 15114 15114 + }, { /* NIST SP800-38A F.2.3 CBC-AES192.Encrypt */ 15115 15115 + #ifdef __LITTLE_ENDIAN 15116 15116 + .key = "\x08\x00" /* rta length */ 15117 15117 + "\x01\x00" /* rta type */ 15118 15118 + #else 15119 15119 + .key = "\x00\x08" /* rta length */ 15120 15120 + "\x00\x01" /* rta type */ 15121 15121 + #endif 15122 15122 + "\x00\x00\x00\x18" /* enc key length */ 15123 15123 + "\x11\x22\x33\x44\x55\x66\x77\x88" 15124 15124 + "\x99\xaa\xbb\xcc\xdd\xee\xff\x11" 15125 15125 + "\x22\x33\x44\x55\x66\x77\x88\x99" 15126 15126 + "\xaa\xbb\xcc\xdd" 15127 15127 + "\x8e\x73\xb0\xf7\xda\x0e\x64\x52" 15128 15128 + "\xc8\x10\xf3\x2b\x80\x90\x79\xe5" 15129 15129 + "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b", 15130 15130 + .klen = 8 + 28 + 24, 15131 15131 + .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" 15132 15132 + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 15133 15133 + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" 15134 15134 + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 15135 15135 + .alen = 16, 15136 15136 + .ptext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" 15137 15137 + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" 15138 15138 + "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" 15139 15139 + "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" 15140 15140 + "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" 15141 15141 + "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" 15142 15142 + "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" 15143 15143 + "\xad\x2b\x41\x7b\xe6\x6c\x37\x10", 15144 15144 + .plen = 64, 15145 15145 + .ctext = "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d" 15146 15146 + "\x71\x78\x18\x3a\x9f\xa0\x71\xe8" 15147 15147 + "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4" 15148 15148 + "\xe5\xe7\x38\x76\x3f\x69\x14\x5a" 15149 15149 + "\x57\x1b\x24\x20\x12\xfb\x7a\xe0" 15150 15150 + "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0" 15151 15151 + "\x08\xb0\xe2\x79\x88\x59\x88\x81" 15152 15152 + "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd" 15153 15153 + "\x67\x35\xcd\x86\x94\x51\x3b\x3a" 15154 15154 + "\xaa\x07\xb1\xed\x18\x55\x62\x01" 15155 15155 + "\x95\xb2\x53\xb5\x20\x78\x16\xd7" 15156 15156 + "\xb8\x49\x7f\x96", 15157 15157 + 15158 15158 + .clen = 64 + 28, 15159 15159 + }, { /* NIST SP800-38A F.2.5 CBC-AES256.Encrypt */ 15160 15160 + #ifdef __LITTLE_ENDIAN 15161 15161 + .key = "\x08\x00" /* rta length */ 15162 15162 + "\x01\x00" /* rta type */ 15163 15163 + #else 15164 15164 + .key = "\x00\x08" /* rta length */ 15165 15165 + "\x00\x01" /* rta type */ 15166 15166 + #endif 15167 15167 + "\x00\x00\x00\x20" /* enc key length */ 15168 15168 + "\x11\x22\x33\x44\x55\x66\x77\x88" 15169 15169 + "\x99\xaa\xbb\xcc\xdd\xee\xff\x11" 15170 15170 + "\x22\x33\x44\x55\x66\x77\x88\x99" 15171 15171 + "\xaa\xbb\xcc\xdd" 15172 15172 + "\x60\x3d\xeb\x10\x15\xca\x71\xbe" 15173 15173 + "\x2b\x73\xae\xf0\x85\x7d\x77\x81" 15174 15174 + "\x1f\x35\x2c\x07\x3b\x61\x08\xd7" 15175 15175 + "\x2d\x98\x10\xa3\x09\x14\xdf\xf4", 15176 15176 + .klen = 8 + 28 + 32, 15177 15177 + .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" 15178 15178 + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 15179 15179 + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" 15180 15180 + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 15181 15181 + .alen = 16, 15182 15182 + .ptext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" 15183 15183 + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" 15184 15184 + "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" 15185 15185 + "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" 15186 15186 + "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" 15187 15187 + "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" 15188 15188 + "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" 15189 15189 + "\xad\x2b\x41\x7b\xe6\x6c\x37\x10", 15190 15190 + .plen = 64, 15191 15191 + .ctext = "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba" 15192 15192 + "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6" 15193 15193 + "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d" 15194 15194 + "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d" 15195 15195 + "\x39\xf2\x33\x69\xa9\xd9\xba\xcf" 15196 15196 + "\xa5\x30\xe2\x63\x04\x23\x14\x61" 15197 15197 + "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc" 15198 15198 + "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b" 15199 15199 + "\xe0\xe2\x3d\x3f\x55\x24\x2c\x4d" 15200 15200 + "\xb9\x13\x2a\xc0\x07\xbb\x3b\xda" 15201 15201 + "\xfd\xa4\x51\x32\x3f\x44\xb1\x13" 15202 15202 + "\x98\xf9\xbc\xb9", 15203 15203 + .clen = 64 + 28, 15204 15204 + }, 15205 15205 + }; 15206 15206 + 14922 15207 static const struct aead_testvec hmac_sha256_aes_cbc_tv_temp[] = { 14923 15208 { /* RFC 3602 Case 1 */ 14924 15209 #ifdef __LITTLE_ENDIAN ··· 15484 15199 "\xf3\x61\xde\x1c\xe9\xdb\xcd\xd0" 15485 15200 "\xcc\xce\xe9\x85\x57\xcf\x6f\x5f", 15486 15201 .clen = 64 + 32, 15202 15202 + }, 15203 15203 + }; 15204 15204 + 15205 15205 + static const struct aead_testvec hmac_sha384_aes_cbc_tv_temp[] = { 15206 15206 + { /* RFC 3602 Case 1 */ 15207 15207 + #ifdef __LITTLE_ENDIAN 15208 15208 + .key = "\x08\x00" /* rta length */ 15209 15209 + "\x01\x00" /* rta type */ 15210 15210 + #else 15211 15211 + .key = "\x00\x08" /* rta length */ 15212 15212 + "\x00\x01" /* rta type */ 15213 15213 + #endif 15214 15214 + "\x00\x00\x00\x10" /* enc key length */ 15215 15215 + "\x00\x00\x00\x00\x00\x00\x00\x00" 15216 15216 + "\x00\x00\x00\x00\x00\x00\x00\x00" 15217 15217 + "\x00\x00\x00\x00\x00\x00\x00\x00" 15218 15218 + "\x00\x00\x00\x00\x00\x00\x00\x00" 15219 15219 + "\x00\x00\x00\x00\x00\x00\x00\x00" 15220 15220 + "\x00\x00\x00\x00\x00\x00\x00\x00" 15221 15221 + "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" 15222 15222 + "\x51\x2e\x03\xd5\x34\x12\x00\x06", 15223 15223 + .klen = 8 + 48 + 16, 15224 15224 + .iv = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30" 15225 15225 + "\xb4\x22\xda\x80\x2c\x9f\xac\x41", 15226 15226 + .assoc = "\x3d\xaf\xba\x42\x9d\x9e\xb4\x30" 15227 15227 + "\xb4\x22\xda\x80\x2c\x9f\xac\x41", 15228 15228 + .alen = 16, 15229 15229 + .ptext = "Single block msg", 15230 15230 + .plen = 16, 15231 15231 + .ctext = "\xe3\x53\x77\x9c\x10\x79\xae\xb8" 15232 15232 + "\x27\x08\x94\x2d\xbe\x77\x18\x1a" 15233 15233 + "\x79\x1c\xf1\x22\x95\x80\xe0\x60" 15234 15234 + "\x7f\xf9\x92\x60\x83\xbd\x60\x9c" 15235 15235 + "\xf6\x62\x8b\xa9\x7d\x56\xe2\xaf" 15236 15236 + "\x80\x43\xbc\x41\x4a\x63\x0b\xa0" 15237 15237 + "\x16\x25\xe2\xfe\x0a\x96\xf6\xa5" 15238 15238 + "\x6c\x0b\xc2\x53\xb4\x27\xd9\x42", 15239 15239 + .clen = 16 + 48, 15240 15240 + }, { /* RFC 3602 Case 2 */ 15241 15241 + #ifdef __LITTLE_ENDIAN 15242 15242 + .key = "\x08\x00" /* rta length */ 15243 15243 + "\x01\x00" /* rta type */ 15244 15244 + #else 15245 15245 + .key = "\x00\x08" /* rta length */ 15246 15246 + "\x00\x01" /* rta type */ 15247 15247 + #endif 15248 15248 + "\x00\x00\x00\x10" /* enc key length */ 15249 15249 + "\x20\x21\x22\x23\x24\x25\x26\x27" 15250 15250 + "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" 15251 15251 + "\x30\x31\x32\x33\x34\x35\x36\x37" 15252 15252 + "\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f" 15253 15253 + "\x40\x41\x42\x43\x44\x45\x46\x47" 15254 15254 + "\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f" 15255 15255 + "\xc2\x86\x69\x6d\x88\x7c\x9a\xa0" 15256 15256 + "\x61\x1b\xbb\x3e\x20\x25\xa4\x5a", 15257 15257 + .klen = 8 + 48 + 16, 15258 15258 + .iv = "\x56\x2e\x17\x99\x6d\x09\x3d\x28" 15259 15259 + "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58", 15260 15260 + .assoc = "\x56\x2e\x17\x99\x6d\x09\x3d\x28" 15261 15261 + "\xdd\xb3\xba\x69\x5a\x2e\x6f\x58", 15262 15262 + .alen = 16, 15263 15263 + .ptext = "\x00\x01\x02\x03\x04\x05\x06\x07" 15264 15264 + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" 15265 15265 + "\x10\x11\x12\x13\x14\x15\x16\x17" 15266 15266 + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", 15267 15267 + .plen = 32, 15268 15268 + .ctext = "\xd2\x96\xcd\x94\xc2\xcc\xcf\x8a" 15269 15269 + "\x3a\x86\x30\x28\xb5\xe1\xdc\x0a" 15270 15270 + "\x75\x86\x60\x2d\x25\x3c\xff\xf9" 15271 15271 + "\x1b\x82\x66\xbe\xa6\xd6\x1a\xb1" 15272 15272 + "\x4e\x5b\xa8\x65\x51\xc6\x58\xaf" 15273 15273 + "\x31\x57\x50\x3d\x01\xa1\xa4\x3f" 15274 15274 + "\x42\xd1\xd7\x31\x76\x8d\xf8\xc8" 15275 15275 + "\xe4\xd2\x7e\xc5\x23\xe7\xc6\x2e" 15276 15276 + "\x2d\xfd\x9d\xc1\xac\x50\x1e\xcf" 15277 15277 + "\xa0\x10\xeb\x1a\x9c\xb7\xe1\xca", 15278 15278 + .clen = 32 + 48, 15279 15279 + }, { /* RFC 3602 Case 3 */ 15280 15280 + #ifdef __LITTLE_ENDIAN 15281 15281 + .key = "\x08\x00" /* rta length */ 15282 15282 + "\x01\x00" /* rta type */ 15283 15283 + #else 15284 15284 + .key = "\x00\x08" /* rta length */ 15285 15285 + "\x00\x01" /* rta type */ 15286 15286 + #endif 15287 15287 + "\x00\x00\x00\x10" /* enc key length */ 15288 15288 + "\x11\x22\x33\x44\x55\x66\x77\x88" 15289 15289 + "\x99\xaa\xbb\xcc\xdd\xee\xff\x11" 15290 15290 + "\x22\x33\x44\x55\x66\x77\x88\x99" 15291 15291 + "\xaa\xbb\xcc\xdd\xee\xff\x11\x22" 15292 15292 + "\x33\x44\x55\x66\x77\x88\x99\xaa" 15293 15293 + "\xbb\xcc\xdd\xee\xff\x11\x22\x33" 15294 15294 + "\x6c\x3e\xa0\x47\x76\x30\xce\x21" 15295 15295 + "\xa2\xce\x33\x4a\xa7\x46\xc2\xcd", 15296 15296 + .klen = 8 + 48 + 16, 15297 15297 + .iv = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb" 15298 15298 + "\xd9\xcd\x27\xd8\x25\x68\x2c\x81", 15299 15299 + .assoc = "\xc7\x82\xdc\x4c\x09\x8c\x66\xcb" 15300 15300 + "\xd9\xcd\x27\xd8\x25\x68\x2c\x81", 15301 15301 + .alen = 16, 15302 15302 + .ptext = "This is a 48-byte message (exactly 3 AES blocks)", 15303 15303 + .plen = 48, 15304 15304 + .ctext = "\xd0\xa0\x2b\x38\x36\x45\x17\x53" 15305 15305 + "\xd4\x93\x66\x5d\x33\xf0\xe8\x86" 15306 15306 + "\x2d\xea\x54\xcd\xb2\x93\xab\xc7" 15307 15307 + "\x50\x69\x39\x27\x67\x72\xf8\xd5" 15308 15308 + "\x02\x1c\x19\x21\x6b\xad\x52\x5c" 15309 15309 + "\x85\x79\x69\x5d\x83\xba\x26\x84" 15310 15310 + "\xa1\x52\xe7\xda\xf7\x05\xb6\xca" 15311 15311 + "\xad\x0f\x51\xed\x5a\xd3\x0f\xdf" 15312 15312 + "\xde\xeb\x3f\x31\xed\x3a\x43\x93" 15313 15313 + "\x3b\xb7\xca\xc8\x1b\xe7\x3b\x61" 15314 15314 + "\x6a\x05\xfd\x2d\x6a\x5c\xb1\x0d" 15315 15315 + "\x6e\x7a\xeb\x1c\x84\xec\xdb\xde", 15316 15316 + .clen = 48 + 48, 15317 15317 + }, { /* RFC 3602 Case 4 */ 15318 15318 + #ifdef __LITTLE_ENDIAN 15319 15319 + .key = "\x08\x00" /* rta length */ 15320 15320 + "\x01\x00" /* rta type */ 15321 15321 + #else 15322 15322 + .key = "\x00\x08" /* rta length */ 15323 15323 + "\x00\x01" /* rta type */ 15324 15324 + #endif 15325 15325 + "\x00\x00\x00\x10" /* enc key length */ 15326 15326 + "\x11\x22\x33\x44\x55\x66\x77\x88" 15327 15327 + "\x99\xaa\xbb\xcc\xdd\xee\xff\x11" 15328 15328 + "\x22\x33\x44\x55\x66\x77\x88\x99" 15329 15329 + "\xaa\xbb\xcc\xdd\xee\xff\x11\x22" 15330 15330 + "\x33\x44\x55\x66\x77\x88\x99\xaa" 15331 15331 + "\xbb\xcc\xdd\xee\xff\x11\x22\x33" 15332 15332 + "\x56\xe4\x7a\x38\xc5\x59\x89\x74" 15333 15333 + "\xbc\x46\x90\x3d\xba\x29\x03\x49", 15334 15334 + .klen = 8 + 48 + 16, 15335 15335 + .iv = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c" 15336 15336 + "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9", 15337 15337 + .assoc = "\x8c\xe8\x2e\xef\xbe\xa0\xda\x3c" 15338 15338 + "\x44\x69\x9e\xd7\xdb\x51\xb7\xd9", 15339 15339 + .alen = 16, 15340 15340 + .ptext = "\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7" 15341 15341 + "\xa8\xa9\xaa\xab\xac\xad\xae\xaf" 15342 15342 + "\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7" 15343 15343 + "\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf" 15344 15344 + "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" 15345 15345 + "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf" 15346 15346 + "\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7" 15347 15347 + "\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf", 15348 15348 + .plen = 64, 15349 15349 + .ctext = "\xc3\x0e\x32\xff\xed\xc0\x77\x4e" 15350 15350 + "\x6a\xff\x6a\xf0\x86\x9f\x71\xaa" 15351 15351 + "\x0f\x3a\xf0\x7a\x9a\x31\xa9\xc6" 15352 15352 + "\x84\xdb\x20\x7e\xb0\xef\x8e\x4e" 15353 15353 + "\x35\x90\x7a\xa6\x32\xc3\xff\xdf" 15354 15354 + "\x86\x8b\xb7\xb2\x9d\x3d\x46\xad" 15355 15355 + "\x83\xce\x9f\x9a\x10\x2e\xe9\x9d" 15356 15356 + "\x49\xa5\x3e\x87\xf4\xc3\xda\x55" 15357 15357 + "\x85\x7b\x91\xe0\x29\xeb\xd3\x59" 15358 15358 + "\x7c\xe3\x67\x14\xbe\x71\x2a\xd2" 15359 15359 + "\x8a\x1a\xd2\x35\x78\x6b\x69\xba" 15360 15360 + "\x64\xa5\x04\x00\x19\xc3\x4c\xae" 15361 15361 + "\x71\xff\x76\x9f\xbb\xc3\x29\x22" 15362 15362 + "\xc2\xc6\x51\xf1\xe6\x29\x5e\xa5", 15363 15363 + .clen = 64 + 48, 15364 15364 + }, { /* RFC 3602 Case 5 */ 15365 15365 + #ifdef __LITTLE_ENDIAN 15366 15366 + .key = "\x08\x00" /* rta length */ 15367 15367 + "\x01\x00" /* rta type */ 15368 15368 + #else 15369 15369 + .key = "\x00\x08" /* rta length */ 15370 15370 + "\x00\x01" /* rta type */ 15371 15371 + #endif 15372 15372 + "\x00\x00\x00\x10" /* enc key length */ 15373 15373 + "\x11\x22\x33\x44\x55\x66\x77\x88" 15374 15374 + "\x99\xaa\xbb\xcc\xdd\xee\xff\x11" 15375 15375 + "\x22\x33\x44\x55\x66\x77\x88\x99" 15376 15376 + "\xaa\xbb\xcc\xdd\xee\xff\x11\x22" 15377 15377 + "\x33\x44\x55\x66\x77\x88\x99\xaa" 15378 15378 + "\xbb\xcc\xdd\xee\xff\x11\x22\x33" 15379 15379 + "\x90\xd3\x82\xb4\x10\xee\xba\x7a" 15380 15380 + "\xd9\x38\xc4\x6c\xec\x1a\x82\xbf", 15381 15381 + .klen = 8 + 48 + 16, 15382 15382 + .iv = "\xe9\x6e\x8c\x08\xab\x46\x57\x63" 15383 15383 + "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93", 15384 15384 + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" 15385 15385 + "\xe9\x6e\x8c\x08\xab\x46\x57\x63" 15386 15386 + "\xfd\x09\x8d\x45\xdd\x3f\xf8\x93", 15387 15387 + .alen = 24, 15388 15388 + .ptext = "\x08\x00\x0e\xbd\xa7\x0a\x00\x00" 15389 15389 + "\x8e\x9c\x08\x3d\xb9\x5b\x07\x00" 15390 15390 + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" 15391 15391 + "\x10\x11\x12\x13\x14\x15\x16\x17" 15392 15392 + "\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" 15393 15393 + "\x20\x21\x22\x23\x24\x25\x26\x27" 15394 15394 + "\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f" 15395 15395 + "\x30\x31\x32\x33\x34\x35\x36\x37" 15396 15396 + "\x01\x02\x03\x04\x05\x06\x07\x08" 15397 15397 + "\x09\x0a\x0b\x0c\x0d\x0e\x0e\x01", 15398 15398 + .plen = 80, 15399 15399 + .ctext = "\xf6\x63\xc2\x5d\x32\x5c\x18\xc6" 15400 15400 + "\xa9\x45\x3e\x19\x4e\x12\x08\x49" 15401 15401 + "\xa4\x87\x0b\x66\xcc\x6b\x99\x65" 15402 15402 + "\x33\x00\x13\xb4\x89\x8d\xc8\x56" 15403 15403 + "\xa4\x69\x9e\x52\x3a\x55\xdb\x08" 15404 15404 + "\x0b\x59\xec\x3a\x8e\x4b\x7e\x52" 15405 15405 + "\x77\x5b\x07\xd1\xdb\x34\xed\x9c" 15406 15406 + "\x53\x8a\xb5\x0c\x55\x1b\x87\x4a" 15407 15407 + "\xa2\x69\xad\xd0\x47\xad\x2d\x59" 15408 15408 + "\x13\xac\x19\xb7\xcf\xba\xd4\xa6" 15409 15409 + "\x57\x5f\xb4\xd7\x74\x6f\x18\x97" 15410 15410 + "\xb7\xde\xfc\xf3\x4e\x0d\x29\x4d" 15411 15411 + "\xa0\xff\x39\x9e\x2d\xbf\x27\xac" 15412 15412 + "\x54\xb9\x8a\x3e\xab\x3b\xac\xd3" 15413 15413 + "\x36\x43\x74\xfc\xc2\x64\x81\x8a" 15414 15414 + "\x2c\x15\x72\xdf\x3f\x9d\x5b\xa4", 15415 15415 + .clen = 80 + 48, 15416 15416 + }, { /* NIST SP800-38A F.2.3 CBC-AES192.Encrypt */ 15417 15417 + #ifdef __LITTLE_ENDIAN 15418 15418 + .key = "\x08\x00" /* rta length */ 15419 15419 + "\x01\x00" /* rta type */ 15420 15420 + #else 15421 15421 + .key = "\x00\x08" /* rta length */ 15422 15422 + "\x00\x01" /* rta type */ 15423 15423 + #endif 15424 15424 + "\x00\x00\x00\x18" /* enc key length */ 15425 15425 + "\x11\x22\x33\x44\x55\x66\x77\x88" 15426 15426 + "\x99\xaa\xbb\xcc\xdd\xee\xff\x11" 15427 15427 + "\x22\x33\x44\x55\x66\x77\x88\x99" 15428 15428 + "\xaa\xbb\xcc\xdd\xee\xff\x11\x22" 15429 15429 + "\x33\x44\x55\x66\x77\x88\x99\xaa" 15430 15430 + "\xbb\xcc\xdd\xee\xff\x11\x22\x33" 15431 15431 + "\x8e\x73\xb0\xf7\xda\x0e\x64\x52" 15432 15432 + "\xc8\x10\xf3\x2b\x80\x90\x79\xe5" 15433 15433 + "\x62\xf8\xea\xd2\x52\x2c\x6b\x7b", 15434 15434 + .klen = 8 + 48 + 24, 15435 15435 + .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" 15436 15436 + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 15437 15437 + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" 15438 15438 + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 15439 15439 + .alen = 16, 15440 15440 + .ptext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" 15441 15441 + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" 15442 15442 + "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" 15443 15443 + "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" 15444 15444 + "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" 15445 15445 + "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" 15446 15446 + "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" 15447 15447 + "\xad\x2b\x41\x7b\xe6\x6c\x37\x10", 15448 15448 + .plen = 64, 15449 15449 + .ctext = "\x4f\x02\x1d\xb2\x43\xbc\x63\x3d" 15450 15450 + "\x71\x78\x18\x3a\x9f\xa0\x71\xe8" 15451 15451 + "\xb4\xd9\xad\xa9\xad\x7d\xed\xf4" 15452 15452 + "\xe5\xe7\x38\x76\x3f\x69\x14\x5a" 15453 15453 + "\x57\x1b\x24\x20\x12\xfb\x7a\xe0" 15454 15454 + "\x7f\xa9\xba\xac\x3d\xf1\x02\xe0" 15455 15455 + "\x08\xb0\xe2\x79\x88\x59\x88\x81" 15456 15456 + "\xd9\x20\xa9\xe6\x4f\x56\x15\xcd" 15457 15457 + "\x29\x9b\x42\x47\x0b\xbf\xf3\x54" 15458 15458 + "\x54\x95\xb0\x89\xd5\xa0\xc3\x78" 15459 15459 + "\x60\x6c\x18\x39\x6d\xc9\xfb\x2a" 15460 15460 + "\x34\x1c\xed\x95\x10\x1e\x43\x0a" 15461 15461 + "\x72\xce\x26\xbc\x74\xd9\x6f\xa2" 15462 15462 + "\xf1\xd9\xd0\xb1\xdf\x3d\x93\x14", 15463 15463 + .clen = 64 + 48, 15464 15464 + }, { /* NIST SP800-38A F.2.5 CBC-AES256.Encrypt */ 15465 15465 + #ifdef __LITTLE_ENDIAN 15466 15466 + .key = "\x08\x00" /* rta length */ 15467 15467 + "\x01\x00" /* rta type */ 15468 15468 + #else 15469 15469 + .key = "\x00\x08" /* rta length */ 15470 15470 + "\x00\x01" /* rta type */ 15471 15471 + #endif 15472 15472 + "\x00\x00\x00\x20" /* enc key length */ 15473 15473 + "\x11\x22\x33\x44\x55\x66\x77\x88" 15474 15474 + "\x99\xaa\xbb\xcc\xdd\xee\xff\x11" 15475 15475 + "\x22\x33\x44\x55\x66\x77\x88\x99" 15476 15476 + "\xaa\xbb\xcc\xdd\xee\xff\x11\x22" 15477 15477 + "\x33\x44\x55\x66\x77\x88\x99\xaa" 15478 15478 + "\xbb\xcc\xdd\xee\xff\x11\x22\x33" 15479 15479 + "\x60\x3d\xeb\x10\x15\xca\x71\xbe" 15480 15480 + "\x2b\x73\xae\xf0\x85\x7d\x77\x81" 15481 15481 + "\x1f\x35\x2c\x07\x3b\x61\x08\xd7" 15482 15482 + "\x2d\x98\x10\xa3\x09\x14\xdf\xf4", 15483 15483 + .klen = 8 + 48 + 32, 15484 15484 + .iv = "\x00\x01\x02\x03\x04\x05\x06\x07" 15485 15485 + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 15486 15486 + .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" 15487 15487 + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 15488 15488 + .alen = 16, 15489 15489 + .ptext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" 15490 15490 + "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" 15491 15491 + "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" 15492 15492 + "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" 15493 15493 + "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" 15494 15494 + "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" 15495 15495 + "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" 15496 15496 + "\xad\x2b\x41\x7b\xe6\x6c\x37\x10", 15497 15497 + .plen = 64, 15498 15498 + .ctext = "\xf5\x8c\x4c\x04\xd6\xe5\xf1\xba" 15499 15499 + "\x77\x9e\xab\xfb\x5f\x7b\xfb\xd6" 15500 15500 + "\x9c\xfc\x4e\x96\x7e\xdb\x80\x8d" 15501 15501 + "\x67\x9f\x77\x7b\xc6\x70\x2c\x7d" 15502 15502 + "\x39\xf2\x33\x69\xa9\xd9\xba\xcf" 15503 15503 + "\xa5\x30\xe2\x63\x04\x23\x14\x61" 15504 15504 + "\xb2\xeb\x05\xe2\xc3\x9b\xe9\xfc" 15505 15505 + "\xda\x6c\x19\x07\x8c\x6a\x9d\x1b" 15506 15506 + "\x9f\x50\xce\x64\xd9\xa3\xc9\x7a" 15507 15507 + "\x15\x3a\x3d\x46\x9a\x90\xf3\x06" 15508 15508 + "\x22\xad\xc5\x24\x77\x50\xb8\xfe" 15509 15509 + "\xbe\x37\x16\x86\x34\x5f\xaf\x97" 15510 15510 + "\x00\x9d\x86\xc8\x32\x4f\x72\x2f" 15511 15511 + "\x48\x97\xad\xb6\xb9\x77\x33\xbc", 15512 15512 + .clen = 64 + 48, 15487 15513 }, 15488 15514 }; 15489 15515 ··· 16447 15851 "\x8c\xf6\x07\x95\x1f\xa6\x6c\x96" 16448 15852 "\x99\xc7\x5c\x8d\xd8\xb5\x68\x7b", 16449 15853 .clen = 128 + 64, 15854 15854 + }, 15855 15855 + }; 15856 15856 + 15857 15857 + static const struct aead_testvec hmac_md5_des3_ede_cbc_tv_temp[] = { 15858 15858 + { /*Generated with cryptopp*/ 15859 15859 + #ifdef __LITTLE_ENDIAN 15860 15860 + .key = "\x08\x00" /* rta length */ 15861 15861 + "\x01\x00" /* rta type */ 15862 15862 + #else 15863 15863 + .key = "\x00\x08" /* rta length */ 15864 15864 + "\x00\x01" /* rta type */ 15865 15865 + #endif 15866 15866 + "\x00\x00\x00\x18" /* enc key length */ 15867 15867 + "\x11\x22\x33\x44\x55\x66\x77\x88" 15868 15868 + "\x99\xaa\xbb\xcc\xdd\xee\xff\x11" 15869 15869 + "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24" 15870 15870 + "\x44\x4D\x99\x5A\x12\xD6\x40\xC0" 15871 15871 + "\xEA\xC2\x84\xE8\x14\x95\xDB\xE8", 15872 15872 + .klen = 8 + 16 + 24, 15873 15873 + .iv = "\x7D\x33\x88\x93\x0F\x93\xB2\x42", 15874 15874 + .assoc = "\x00\x00\x43\x21\x00\x00\x00\x01" 15875 15875 + "\x7D\x33\x88\x93\x0F\x93\xB2\x42", 15876 15876 + .alen = 16, 15877 15877 + .ptext = "\x6f\x54\x20\x6f\x61\x4d\x79\x6e" 15878 15878 + "\x53\x20\x63\x65\x65\x72\x73\x74" 15879 15879 + "\x54\x20\x6f\x6f\x4d\x20\x6e\x61" 15880 15880 + "\x20\x79\x65\x53\x72\x63\x74\x65" 15881 15881 + "\x20\x73\x6f\x54\x20\x6f\x61\x4d" 15882 15882 + "\x79\x6e\x53\x20\x63\x65\x65\x72" 15883 15883 + "\x73\x74\x54\x20\x6f\x6f\x4d\x20" 15884 15884 + "\x6e\x61\x20\x79\x65\x53\x72\x63" 15885 15885 + "\x74\x65\x20\x73\x6f\x54\x20\x6f" 15886 15886 + "\x61\x4d\x79\x6e\x53\x20\x63\x65" 15887 15887 + "\x65\x72\x73\x74\x54\x20\x6f\x6f" 15888 15888 + "\x4d\x20\x6e\x61\x20\x79\x65\x53" 15889 15889 + "\x72\x63\x74\x65\x20\x73\x6f\x54" 15890 15890 + "\x20\x6f\x61\x4d\x79\x6e\x53\x20" 15891 15891 + "\x63\x65\x65\x72\x73\x74\x54\x20" 15892 15892 + "\x6f\x6f\x4d\x20\x6e\x61\x0a\x79", 15893 15893 + .plen = 128, 15894 15894 + .ctext = "\x0e\x2d\xb6\x97\x3c\x56\x33\xf4" 15895 15895 + "\x67\x17\x21\xc7\x6e\x8a\xd5\x49" 15896 15896 + "\x74\xb3\x49\x05\xc5\x1c\xd0\xed" 15897 15897 + "\x12\x56\x5c\x53\x96\xb6\x00\x7d" 15898 15898 + "\x90\x48\xfc\xf5\x8d\x29\x39\xcc" 15899 15899 + "\x8a\xd5\x35\x18\x36\x23\x4e\xd7" 15900 15900 + "\x76\xd1\xda\x0c\x94\x67\xbb\x04" 15901 15901 + "\x8b\xf2\x03\x6c\xa8\xcf\xb6\xea" 15902 15902 + "\x22\x64\x47\xaa\x8f\x75\x13\xbf" 15903 15903 + "\x9f\xc2\xc3\xf0\xc9\x56\xc5\x7a" 15904 15904 + "\x71\x63\x2e\x89\x7b\x1e\x12\xca" 15905 15905 + "\xe2\x5f\xaf\xd8\xa4\xf8\xc9\x7a" 15906 15906 + "\xd6\xf9\x21\x31\x62\x44\x45\xa6" 15907 15907 + "\xd6\xbc\x5a\xd3\x2d\x54\x43\xcc" 15908 15908 + "\x9d\xde\xa5\x70\xe9\x42\x45\x8a" 15909 15909 + "\x6b\xfa\xb1\x91\x13\xb0\xd9\x19" 15910 15910 + "\x99\x09\xfb\x05\x35\xc8\xcc\x38" 15911 15911 + "\xc3\x1e\x5e\xe1\xe6\x96\x84\xc8", 15912 15912 + .clen = 128 + 16, 16450 15913 }, 16451 15914 }; 16452 15915

+1

drivers/char/hw_random/airoha-trng.c

reviewed

··· 212 212 trng->rng.init = airoha_trng_init; 213 213 trng->rng.cleanup = airoha_trng_cleanup; 214 214 trng->rng.read = airoha_trng_read; 215 215 + trng->rng.quality = 900; 215 216 216 217 ret = devm_hwrng_register(dev, &trng->rng); 217 218 if (ret) {

+105 -63

drivers/char/hw_random/core.c

reviewed

··· 20 20 #include <linux/miscdevice.h> 21 21 #include <linux/module.h> 22 22 #include <linux/random.h> 23 23 + #include <linux/rcupdate.h> 23 24 #include <linux/sched.h> 24 25 #include <linux/sched/signal.h> 25 26 #include <linux/slab.h> 26 27 #include <linux/string.h> 27 28 #include <linux/uaccess.h> 29 29 + #include <linux/workqueue.h> 28 30 29 31 #define RNG_MODULE_NAME "hw_random" 30 32 31 33 #define RNG_BUFFER_SIZE (SMP_CACHE_BYTES < 32 ? 32 : SMP_CACHE_BYTES) 32 34 33 33 - static struct hwrng *current_rng; 35 35 + static struct hwrng __rcu *current_rng; 34 36 /* the current rng has been explicitly chosen by user via sysfs */ 35 37 static int cur_rng_set_by_user; 36 38 static struct task_struct *hwrng_fill; 37 39 /* list of registered rngs */ 38 40 static LIST_HEAD(rng_list); 39 39 - /* Protects rng_list and current_rng */ 41 41 + /* Protects rng_list, hwrng_fill and updating on current_rng */ 40 42 static DEFINE_MUTEX(rng_mutex); 41 43 /* Protects rng read functions, data_avail, rng_buffer and rng_fillbuf */ 42 44 static DEFINE_MUTEX(reading_mutex); ··· 66 64 return RNG_BUFFER_SIZE; 67 65 } 68 66 69 69 - static inline void cleanup_rng(struct kref *kref) 67 67 + static void cleanup_rng_work(struct work_struct *work) 70 68 { 71 71 - struct hwrng *rng = container_of(kref, struct hwrng, ref); 69 69 + struct hwrng *rng = container_of(work, struct hwrng, cleanup_work); 70 70 + 71 71 + /* 72 72 + * Hold rng_mutex here so we serialize in case they set_current_rng 73 73 + * on rng again immediately. 74 74 + */ 75 75 + mutex_lock(&rng_mutex); 76 76 + 77 77 + /* Skip if rng has been reinitialized. */ 78 78 + if (kref_read(&rng->ref)) { 79 79 + mutex_unlock(&rng_mutex); 80 80 + return; 81 81 + } 72 82 73 83 if (rng->cleanup) 74 84 rng->cleanup(rng); 75 85 76 86 complete(&rng->cleanup_done); 87 87 + mutex_unlock(&rng_mutex); 88 88 + } 89 89 + 90 90 + static inline void cleanup_rng(struct kref *kref) 91 91 + { 92 92 + struct hwrng *rng = container_of(kref, struct hwrng, ref); 93 93 + 94 94 + schedule_work(&rng->cleanup_work); 77 95 } 78 96 79 97 static int set_current_rng(struct hwrng *rng) 80 98 { 99 99 + struct hwrng *old_rng; 81 100 int err; 82 101 83 102 BUG_ON(!mutex_is_locked(&rng_mutex)); ··· 107 84 if (err) 108 85 return err; 109 86 110 110 - drop_current_rng(); 111 111 - current_rng = rng; 87 87 + old_rng = rcu_dereference_protected(current_rng, 88 88 + lockdep_is_held(&rng_mutex)); 89 89 + rcu_assign_pointer(current_rng, rng); 90 90 + 91 91 + if (old_rng) { 92 92 + synchronize_rcu(); 93 93 + kref_put(&old_rng->ref, cleanup_rng); 94 94 + } 112 95 113 96 /* if necessary, start hwrng thread */ 114 97 if (!hwrng_fill) { ··· 130 101 131 102 static void drop_current_rng(void) 132 103 { 133 133 - BUG_ON(!mutex_is_locked(&rng_mutex)); 134 134 - if (!current_rng) 104 104 + struct hwrng *rng; 105 105 + 106 106 + rng = rcu_dereference_protected(current_rng, 107 107 + lockdep_is_held(&rng_mutex)); 108 108 + if (!rng) 135 109 return; 136 110 111 111 + RCU_INIT_POINTER(current_rng, NULL); 112 112 + synchronize_rcu(); 113 113 + 114 114 + if (hwrng_fill) { 115 115 + kthread_stop(hwrng_fill); 116 116 + hwrng_fill = NULL; 117 117 + } 118 118 + 137 119 /* decrease last reference for triggering the cleanup */ 138 138 - kref_put(&current_rng->ref, cleanup_rng); 139 139 - current_rng = NULL; 120 120 + kref_put(&rng->ref, cleanup_rng); 140 121 } 141 122 142 142 - /* Returns ERR_PTR(), NULL or refcounted hwrng */ 123 123 + /* Returns NULL or refcounted hwrng */ 143 124 static struct hwrng *get_current_rng_nolock(void) 144 125 { 145 145 - if (current_rng) 146 146 - kref_get(&current_rng->ref); 126 126 + struct hwrng *rng; 147 127 148 148 - return current_rng; 128 128 + rng = rcu_dereference_protected(current_rng, 129 129 + lockdep_is_held(&rng_mutex)); 130 130 + if (rng) 131 131 + kref_get(&rng->ref); 132 132 + 133 133 + return rng; 149 134 } 150 135 151 136 static struct hwrng *get_current_rng(void) 152 137 { 153 138 struct hwrng *rng; 154 139 155 155 - if (mutex_lock_interruptible(&rng_mutex)) 156 156 - return ERR_PTR(-ERESTARTSYS); 140 140 + rcu_read_lock(); 141 141 + rng = rcu_dereference(current_rng); 142 142 + if (rng) 143 143 + kref_get(&rng->ref); 157 144 158 158 - rng = get_current_rng_nolock(); 145 145 + rcu_read_unlock(); 159 146 160 160 - mutex_unlock(&rng_mutex); 161 147 return rng; 162 148 } 163 149 164 150 static void put_rng(struct hwrng *rng) 165 151 { 166 166 - /* 167 167 - * Hold rng_mutex here so we serialize in case they set_current_rng 168 168 - * on rng again immediately. 169 169 - */ 170 170 - mutex_lock(&rng_mutex); 171 152 if (rng) 172 153 kref_put(&rng->ref, cleanup_rng); 173 173 - mutex_unlock(&rng_mutex); 174 154 } 175 155 176 156 static int hwrng_init(struct hwrng *rng) ··· 251 213 252 214 while (size) { 253 215 rng = get_current_rng(); 254 254 - if (IS_ERR(rng)) { 255 255 - err = PTR_ERR(rng); 256 256 - goto out; 257 257 - } 258 216 if (!rng) { 259 217 err = -ENODEV; 260 218 goto out; ··· 337 303 338 304 static int enable_best_rng(void) 339 305 { 340 340 - struct hwrng *rng, *new_rng = NULL; 306 306 + struct hwrng *rng, *cur_rng, *new_rng = NULL; 341 307 int ret = -ENODEV; 342 308 343 309 BUG_ON(!mutex_is_locked(&rng_mutex)); ··· 355 321 new_rng = rng; 356 322 } 357 323 358 358 - ret = ((new_rng == current_rng) ? 0 : set_current_rng(new_rng)); 324 324 + cur_rng = rcu_dereference_protected(current_rng, 325 325 + lockdep_is_held(&rng_mutex)); 326 326 + ret = ((new_rng == cur_rng) ? 0 : set_current_rng(new_rng)); 359 327 if (!ret) 360 328 cur_rng_set_by_user = 0; 361 329 ··· 407 371 struct hwrng *rng; 408 372 409 373 rng = get_current_rng(); 410 410 - if (IS_ERR(rng)) 411 411 - return PTR_ERR(rng); 412 374 413 375 ret = sysfs_emit(buf, "%s\n", rng ? rng->name : "none"); 414 376 put_rng(rng); ··· 450 416 struct hwrng *rng; 451 417 452 418 rng = get_current_rng(); 453 453 - if (IS_ERR(rng)) 454 454 - return PTR_ERR(rng); 455 419 456 420 if (!rng) /* no need to put_rng */ 457 421 return -ENODEV; ··· 464 432 struct device_attribute *attr, 465 433 const char *buf, size_t len) 466 434 { 435 435 + struct hwrng *rng; 467 436 u16 quality; 468 437 int ret = -EINVAL; 469 438 ··· 481 448 goto out; 482 449 } 483 450 484 484 - if (!current_rng) { 451 451 + rng = rcu_dereference_protected(current_rng, lockdep_is_held(&rng_mutex)); 452 452 + if (!rng) { 485 453 ret = -ENODEV; 486 454 goto out; 487 455 } 488 456 489 489 - current_rng->quality = quality; 457 457 + rng->quality = quality; 490 458 current_quality = quality; /* obsolete */ 491 459 492 460 /* the best available RNG may have changed */ ··· 523 489 struct hwrng *rng; 524 490 525 491 rng = get_current_rng(); 526 526 - if (IS_ERR(rng) || !rng) 492 492 + if (!rng) { 493 493 + /* 494 494 + * Keep the task_struct alive until kthread_stop() 495 495 + * is called to avoid UAF in drop_current_rng(). 496 496 + */ 497 497 + while (!kthread_should_stop()) { 498 498 + set_current_state(TASK_INTERRUPTIBLE); 499 499 + if (!kthread_should_stop()) 500 500 + schedule(); 501 501 + } 502 502 + set_current_state(TASK_RUNNING); 527 503 break; 504 504 + } 505 505 + 528 506 mutex_lock(&reading_mutex); 529 507 rc = rng_get_data(rng, rng_fillbuf, 530 508 rng_buffer_size(), 1); ··· 564 518 add_hwgenerator_randomness((void *)rng_fillbuf, rc, 565 519 entropy >> 10, true); 566 520 } 567 567 - hwrng_fill = NULL; 568 521 return 0; 569 522 } 570 523 571 524 int hwrng_register(struct hwrng *rng) 572 525 { 573 526 int err = -EINVAL; 574 574 - struct hwrng *tmp; 527 527 + struct hwrng *cur_rng, *tmp; 575 528 576 529 if (!rng->name || (!rng->data_read && !rng->read)) 577 530 goto out; ··· 585 540 } 586 541 list_add_tail(&rng->list, &rng_list); 587 542 543 543 + INIT_WORK(&rng->cleanup_work, cleanup_rng_work); 588 544 init_completion(&rng->cleanup_done); 589 545 complete(&rng->cleanup_done); 590 546 init_completion(&rng->dying); ··· 593 547 /* Adjust quality field to always have a proper value */ 594 548 rng->quality = min3(default_quality, 1024, rng->quality ?: 1024); 595 549 596 596 - if (!cur_rng_set_by_user && 597 597 - (!current_rng || rng->quality > current_rng->quality)) { 598 598 - /* 599 599 - * Set new rng as current as the new rng source 600 600 - * provides better entropy quality and was not 601 601 - * chosen by userspace. 602 602 - */ 603 603 - err = set_current_rng(rng); 604 604 - if (err) 605 605 - goto out_unlock; 550 550 + if (!cur_rng_set_by_user) { 551 551 + cur_rng = rcu_dereference_protected(current_rng, 552 552 + lockdep_is_held(&rng_mutex)); 553 553 + if (!cur_rng || rng->quality > cur_rng->quality) { 554 554 + /* 555 555 + * Set new rng as current as the new rng source 556 556 + * provides better entropy quality and was not 557 557 + * chosen by userspace. 558 558 + */ 559 559 + err = set_current_rng(rng); 560 560 + if (err) 561 561 + goto out_unlock; 562 562 + } 606 563 } 607 564 mutex_unlock(&rng_mutex); 608 565 return 0; ··· 618 569 619 570 void hwrng_unregister(struct hwrng *rng) 620 571 { 621 621 - struct hwrng *new_rng; 572 572 + struct hwrng *cur_rng; 622 573 int err; 623 574 624 575 mutex_lock(&rng_mutex); 625 576 626 577 list_del(&rng->list); 627 578 complete_all(&rng->dying); 628 628 - if (current_rng == rng) { 579 579 + 580 580 + cur_rng = rcu_dereference_protected(current_rng, 581 581 + lockdep_is_held(&rng_mutex)); 582 582 + if (cur_rng == rng) { 629 583 err = enable_best_rng(); 630 584 if (err) { 631 585 drop_current_rng(); ··· 636 584 } 637 585 } 638 586 639 639 - new_rng = get_current_rng_nolock(); 640 640 - if (list_empty(&rng_list)) { 641 641 - mutex_unlock(&rng_mutex); 642 642 - if (hwrng_fill) 643 643 - kthread_stop(hwrng_fill); 644 644 - } else 645 645 - mutex_unlock(&rng_mutex); 646 646 - 647 647 - if (new_rng) 648 648 - put_rng(new_rng); 649 649 - 587 587 + mutex_unlock(&rng_mutex); 650 588 wait_for_completion(&rng->cleanup_done); 651 589 } 652 590 EXPORT_SYMBOL_GPL(hwrng_unregister); ··· 724 682 static void __exit hwrng_modexit(void) 725 683 { 726 684 mutex_lock(&rng_mutex); 727 727 - BUG_ON(current_rng); 685 685 + WARN_ON(rcu_access_pointer(current_rng)); 728 686 kfree(rng_buffer); 729 687 kfree(rng_fillbuf); 730 688 mutex_unlock(&rng_mutex);

+1 -4

drivers/char/hw_random/optee-rng.c

reviewed

··· 205 205 206 206 static int optee_ctx_match(struct tee_ioctl_version_data *ver, const void *data) 207 207 { 208 208 - if (ver->impl_id == TEE_IMPL_ID_OPTEE) 209 209 - return 1; 210 210 - else 211 211 - return 0; 208 208 + return (ver->impl_id == TEE_IMPL_ID_OPTEE); 212 209 } 213 210 214 211 static int optee_rng_probe(struct device *dev)

+3 -1

drivers/crypto/allwinner/sun8i-ss/sun8i-ss.h

reviewed

··· 248 248 struct sun8i_ss_hash_reqctx { 249 249 struct sginfo t_src[MAX_SG]; 250 250 struct sginfo t_dst[MAX_SG]; 251 251 - struct ahash_request fallback_req; 252 251 u32 method; 253 252 int flow; 253 253 + 254 254 + /* Must be last as it ends in a flexible-array member. */ 255 255 + struct ahash_request fallback_req; 254 256 }; 255 257 256 258 /*

+6 -11

drivers/crypto/atmel-aes.c

reviewed

··· 2201 2201 2202 2202 static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) 2203 2203 { 2204 2204 - int i; 2205 2205 - 2206 2204 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2207 2205 if (dd->caps.has_authenc) 2208 2208 - for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) 2209 2209 - crypto_unregister_aead(&aes_authenc_algs[i]); 2206 2206 + crypto_unregister_aeads(aes_authenc_algs, 2207 2207 + ARRAY_SIZE(aes_authenc_algs)); 2210 2208 #endif 2211 2209 2212 2210 if (dd->caps.has_xts) ··· 2213 2215 if (dd->caps.has_gcm) 2214 2216 crypto_unregister_aead(&aes_gcm_alg); 2215 2217 2216 2216 - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) 2217 2217 - crypto_unregister_skcipher(&aes_algs[i]); 2218 2218 + crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); 2218 2219 } 2219 2220 2220 2221 static void atmel_aes_crypto_alg_init(struct crypto_alg *alg) ··· 2226 2229 2227 2230 static int atmel_aes_register_algs(struct atmel_aes_dev *dd) 2228 2231 { 2229 2229 - int err, i, j; 2232 2232 + int err, i; 2230 2233 2231 2234 for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { 2232 2235 atmel_aes_crypto_alg_init(&aes_algs[i].base); ··· 2269 2272 #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC) 2270 2273 /* i = ARRAY_SIZE(aes_authenc_algs); */ 2271 2274 err_aes_authenc_alg: 2272 2272 - for (j = 0; j < i; j++) 2273 2273 - crypto_unregister_aead(&aes_authenc_algs[j]); 2275 2275 + crypto_unregister_aeads(aes_authenc_algs, i); 2274 2276 crypto_unregister_skcipher(&aes_xts_alg); 2275 2277 #endif 2276 2278 err_aes_xts_alg: ··· 2277 2281 err_aes_gcm_alg: 2278 2282 i = ARRAY_SIZE(aes_algs); 2279 2283 err_aes_algs: 2280 2280 - for (j = 0; j < i; j++) 2281 2281 - crypto_unregister_skcipher(&aes_algs[j]); 2284 2284 + crypto_unregister_skciphers(aes_algs, i); 2282 2285 2283 2286 return err; 2284 2287 }

+10 -17

drivers/crypto/atmel-sha.c

reviewed

··· 2418 2418 2419 2419 static void atmel_sha_unregister_algs(struct atmel_sha_dev *dd) 2420 2420 { 2421 2421 - int i; 2422 2422 - 2423 2421 if (dd->caps.has_hmac) 2424 2424 - for (i = 0; i < ARRAY_SIZE(sha_hmac_algs); i++) 2425 2425 - crypto_unregister_ahash(&sha_hmac_algs[i]); 2422 2422 + crypto_unregister_ahashes(sha_hmac_algs, 2423 2423 + ARRAY_SIZE(sha_hmac_algs)); 2426 2424 2427 2427 - for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) 2428 2428 - crypto_unregister_ahash(&sha_1_256_algs[i]); 2425 2425 + crypto_unregister_ahashes(sha_1_256_algs, ARRAY_SIZE(sha_1_256_algs)); 2429 2426 2430 2427 if (dd->caps.has_sha224) 2431 2428 crypto_unregister_ahash(&sha_224_alg); 2432 2429 2433 2433 - if (dd->caps.has_sha_384_512) { 2434 2434 - for (i = 0; i < ARRAY_SIZE(sha_384_512_algs); i++) 2435 2435 - crypto_unregister_ahash(&sha_384_512_algs[i]); 2436 2436 - } 2430 2430 + if (dd->caps.has_sha_384_512) 2431 2431 + crypto_unregister_ahashes(sha_384_512_algs, 2432 2432 + ARRAY_SIZE(sha_384_512_algs)); 2437 2433 } 2438 2434 2439 2435 static int atmel_sha_register_algs(struct atmel_sha_dev *dd) 2440 2436 { 2441 2441 - int err, i, j; 2437 2437 + int err, i; 2442 2438 2443 2439 for (i = 0; i < ARRAY_SIZE(sha_1_256_algs); i++) { 2444 2440 atmel_sha_alg_init(&sha_1_256_algs[i]); ··· 2476 2480 2477 2481 /*i = ARRAY_SIZE(sha_hmac_algs);*/ 2478 2482 err_sha_hmac_algs: 2479 2479 - for (j = 0; j < i; j++) 2480 2480 - crypto_unregister_ahash(&sha_hmac_algs[j]); 2483 2483 + crypto_unregister_ahashes(sha_hmac_algs, i); 2481 2484 i = ARRAY_SIZE(sha_384_512_algs); 2482 2485 err_sha_384_512_algs: 2483 2483 - for (j = 0; j < i; j++) 2484 2484 - crypto_unregister_ahash(&sha_384_512_algs[j]); 2486 2486 + crypto_unregister_ahashes(sha_384_512_algs, i); 2485 2487 crypto_unregister_ahash(&sha_224_alg); 2486 2488 err_sha_224_algs: 2487 2489 i = ARRAY_SIZE(sha_1_256_algs); 2488 2490 err_sha_1_256_algs: 2489 2489 - for (j = 0; j < i; j++) 2490 2490 - crypto_unregister_ahash(&sha_1_256_algs[j]); 2491 2491 + crypto_unregister_ahashes(sha_1_256_algs, i); 2491 2492 2492 2493 return err; 2493 2494 }

+6 -19

drivers/crypto/atmel-tdes.c

reviewed

··· 897 897 return IRQ_NONE; 898 898 } 899 899 900 900 - static void atmel_tdes_unregister_algs(struct atmel_tdes_dev *dd) 901 901 - { 902 902 - int i; 903 903 - 904 904 - for (i = 0; i < ARRAY_SIZE(tdes_algs); i++) 905 905 - crypto_unregister_skcipher(&tdes_algs[i]); 906 906 - } 907 907 - 908 900 static int atmel_tdes_register_algs(struct atmel_tdes_dev *dd) 909 901 { 910 910 - int err, i, j; 902 902 + int err, i; 911 903 912 904 for (i = 0; i < ARRAY_SIZE(tdes_algs); i++) { 913 905 atmel_tdes_skcipher_alg_init(&tdes_algs[i]); 914 906 915 907 err = crypto_register_skcipher(&tdes_algs[i]); 916 916 - if (err) 917 917 - goto err_tdes_algs; 908 908 + if (err) { 909 909 + crypto_unregister_skciphers(tdes_algs, i); 910 910 + return err; 911 911 + } 918 912 } 919 913 920 914 return 0; 921 921 - 922 922 - err_tdes_algs: 923 923 - for (j = 0; j < i; j++) 924 924 - crypto_unregister_skcipher(&tdes_algs[j]); 925 925 - 926 926 - return err; 927 915 } 928 916 929 917 static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd) 930 918 { 931 931 - 932 919 dd->caps.has_dma = 0; 933 920 934 921 /* keep only major version number */ ··· 1048 1061 list_del(&tdes_dd->list); 1049 1062 spin_unlock(&atmel_tdes.lock); 1050 1063 1051 1051 - atmel_tdes_unregister_algs(tdes_dd); 1064 1064 + crypto_unregister_skciphers(tdes_algs, ARRAY_SIZE(tdes_algs)); 1052 1065 1053 1066 tasklet_kill(&tdes_dd->done_task); 1054 1067 tasklet_kill(&tdes_dd->queue_task);

+15 -12

drivers/crypto/caam/caamalg_qi2.c

reviewed

··· 4814 4814 { 4815 4815 struct device *dev = priv->dev; 4816 4816 struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); 4817 4817 - int err; 4817 4817 + struct dpaa2_caam_priv_per_cpu *ppriv; 4818 4818 + int i, err; 4818 4819 4819 4820 if (DPSECI_VER(priv->major_ver, priv->minor_ver) > DPSECI_VER(5, 3)) { 4820 4821 err = dpseci_reset(priv->mc_io, 0, ls_dev->mc_handle); 4821 4822 if (err) 4822 4823 dev_err(dev, "dpseci_reset() failed\n"); 4823 4824 } 4825 4825 + 4826 4826 + for_each_cpu(i, priv->clean_mask) { 4827 4827 + ppriv = per_cpu_ptr(priv->ppriv, i); 4828 4828 + free_netdev(ppriv->net_dev); 4829 4829 + } 4830 4830 + free_cpumask_var(priv->clean_mask); 4824 4831 4825 4832 dpaa2_dpseci_congestion_free(priv); 4826 4833 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); ··· 5014 5007 struct device *dev = &ls_dev->dev; 5015 5008 struct dpaa2_caam_priv *priv; 5016 5009 struct dpaa2_caam_priv_per_cpu *ppriv; 5017 5017 - cpumask_var_t clean_mask; 5018 5010 int err, cpu; 5019 5011 u8 i; 5020 5012 5021 5013 err = -ENOMEM; 5022 5022 - if (!zalloc_cpumask_var(&clean_mask, GFP_KERNEL)) 5023 5023 - goto err_cpumask; 5024 5024 - 5025 5014 priv = dev_get_drvdata(dev); 5015 5015 + 5016 5016 + if (!zalloc_cpumask_var(&priv->clean_mask, GFP_KERNEL)) 5017 5017 + goto err_cpumask; 5026 5018 5027 5019 priv->dev = dev; 5028 5020 priv->dpsec_id = ls_dev->obj_desc.id; ··· 5124 5118 err = -ENOMEM; 5125 5119 goto err_alloc_netdev; 5126 5120 } 5127 5127 - cpumask_set_cpu(cpu, clean_mask); 5121 5121 + cpumask_set_cpu(cpu, priv->clean_mask); 5128 5122 ppriv->net_dev->dev = *dev; 5129 5123 5130 5124 netif_napi_add_tx_weight(ppriv->net_dev, &ppriv->napi, ··· 5132 5126 DPAA2_CAAM_NAPI_WEIGHT); 5133 5127 } 5134 5128 5135 5135 - err = 0; 5136 5136 - goto free_cpumask; 5129 5129 + return 0; 5137 5130 5138 5131 err_alloc_netdev: 5139 5139 - free_dpaa2_pcpu_netdev(priv, clean_mask); 5132 5132 + free_dpaa2_pcpu_netdev(priv, priv->clean_mask); 5140 5133 err_get_rx_queue: 5141 5134 dpaa2_dpseci_congestion_free(priv); 5142 5135 err_get_vers: 5143 5136 dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); 5144 5137 err_open: 5145 5145 - free_cpumask: 5146 5146 - free_cpumask_var(clean_mask); 5138 5138 + free_cpumask_var(priv->clean_mask); 5147 5139 err_cpumask: 5148 5140 return err; 5149 5141 } ··· 5186 5182 ppriv = per_cpu_ptr(priv->ppriv, i); 5187 5183 napi_disable(&ppriv->napi); 5188 5184 netif_napi_del(&ppriv->napi); 5189 5189 - free_netdev(ppriv->net_dev); 5190 5185 } 5191 5186 5192 5187 return 0;

+2

drivers/crypto/caam/caamalg_qi2.h

reviewed

··· 42 42 * @mc_io: pointer to MC portal's I/O object 43 43 * @domain: IOMMU domain 44 44 * @ppriv: per CPU pointers to privata data 45 45 + * @clean_mask: CPU mask of CPUs that have allocated netdevs 45 46 */ 46 47 struct dpaa2_caam_priv { 47 48 int dpsec_id; ··· 66 65 67 66 struct dpaa2_caam_priv_per_cpu __percpu *ppriv; 68 67 struct dentry *dfs_root; 68 68 + cpumask_var_t clean_mask; 69 69 }; 70 70 71 71 /**

+2 -1

drivers/crypto/cavium/cpt/cptvf_main.c

reviewed

··· 180 180 181 181 hlist_for_each_entry_safe(chunk, node, &cqinfo->queue[i].chead, 182 182 nextchunk) { 183 183 - dma_free_coherent(&pdev->dev, chunk->size, 183 183 + dma_free_coherent(&pdev->dev, 184 184 + chunk->size + CPT_NEXT_CHUNK_PTR_SIZE, 184 185 chunk->head, 185 186 chunk->dma_addr); 186 187 chunk->head = NULL;

+1 -1

drivers/crypto/ccp/ccp-ops.c

reviewed

··· 642 642 struct ccp_data dst; 643 643 struct ccp_data aad; 644 644 struct ccp_op op; 645 645 - } *wa __cleanup(kfree) = kzalloc(sizeof *wa, GFP_KERNEL); 645 645 + } *wa __free(kfree) = kzalloc(sizeof(*wa), GFP_KERNEL); 646 646 unsigned int dm_offset; 647 647 unsigned int authsize; 648 648 unsigned int jobid;

+3

drivers/crypto/ccp/hsti.c

reviewed

··· 30 30 31 31 security_attribute_show(fused_part) 32 32 static DEVICE_ATTR_RO(fused_part); 33 33 + security_attribute_show(boot_integrity) 34 34 + static DEVICE_ATTR_RO(boot_integrity); 33 35 security_attribute_show(debug_lock_on) 34 36 static DEVICE_ATTR_RO(debug_lock_on); 35 37 security_attribute_show(tsme_status) ··· 49 47 50 48 static struct attribute *psp_security_attrs[] = { 51 49 &dev_attr_fused_part.attr, 50 50 + &dev_attr_boot_integrity.attr, 52 51 &dev_attr_debug_lock_on.attr, 53 52 &dev_attr_tsme_status.attr, 54 53 &dev_attr_anti_rollback_status.attr,

+1 -1

drivers/crypto/ccp/psp-dev.h

reviewed

··· 36 36 rsvd1 :3, 37 37 security_reporting :1, 38 38 fused_part :1, 39 39 - rsvd2 :1, 39 39 + boot_integrity :1, 40 40 debug_lock_on :1, 41 41 rsvd3 :2, 42 42 tsme_status :1,

+1 -1

drivers/crypto/ccp/sev-dev-tsm.c

reviewed

··· 228 228 229 229 if (is_pci_tsm_pf0(pdev)) 230 230 return tio_pf0_probe(pdev, sev); 231 231 - return 0; 231 231 + return NULL; 232 232 } 233 233 234 234 static void dsm_remove(struct pci_tsm *tsm)

+24 -35

drivers/crypto/ccp/sev-dev.c

reviewed

··· 127 127 #define NV_LENGTH (32 * 1024) 128 128 static void *sev_init_ex_buffer; 129 129 130 130 - /* 131 131 - * SEV_DATA_RANGE_LIST: 132 132 - * Array containing range of pages that firmware transitions to HV-fixed 133 133 - * page state. 134 134 - */ 135 135 - static struct sev_data_range_list *snp_range_list; 136 136 - 137 130 static void __sev_firmware_shutdown(struct sev_device *sev, bool panic); 138 131 139 132 static int snp_shutdown_on_panic(struct notifier_block *nb, ··· 1354 1361 1355 1362 static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) 1356 1363 { 1364 1364 + struct sev_data_range_list *snp_range_list __free(kfree) = NULL; 1357 1365 struct psp_device *psp = psp_master; 1358 1366 struct sev_data_snp_init_ex data; 1359 1367 struct sev_device *sev; ··· 2372 2378 static int sev_ioctl_do_snp_platform_status(struct sev_issue_cmd *argp) 2373 2379 { 2374 2380 struct sev_device *sev = psp_master->sev_data; 2375 2375 - bool shutdown_required = false; 2376 2381 struct sev_data_snp_addr buf; 2377 2382 struct page *status_page; 2378 2378 - int ret, error; 2379 2383 void *data; 2384 2384 + int ret; 2380 2385 2381 2386 if (!argp->data) 2382 2387 return -EINVAL; ··· 2386 2393 2387 2394 data = page_address(status_page); 2388 2395 2389 2389 - if (!sev->snp_initialized) { 2390 2390 - ret = snp_move_to_init_state(argp, &shutdown_required); 2391 2391 - if (ret) 2392 2392 - goto cleanup; 2393 2393 - } 2394 2394 - 2395 2396 /* 2396 2396 - * Firmware expects status page to be in firmware-owned state, otherwise 2397 2397 - * it will report firmware error code INVALID_PAGE_STATE (0x1A). 2397 2397 + * SNP_PLATFORM_STATUS can be executed in any SNP state. But if executed 2398 2398 + * when SNP has been initialized, the status page must be firmware-owned. 2398 2399 */ 2399 2399 - if (rmp_mark_pages_firmware(__pa(data), 1, true)) { 2400 2400 - ret = -EFAULT; 2401 2401 - goto cleanup; 2400 2400 + if (sev->snp_initialized) { 2401 2401 + /* 2402 2402 + * Firmware expects the status page to be in Firmware state, 2403 2403 + * otherwise it will report an error INVALID_PAGE_STATE. 2404 2404 + */ 2405 2405 + if (rmp_mark_pages_firmware(__pa(data), 1, true)) { 2406 2406 + ret = -EFAULT; 2407 2407 + goto cleanup; 2408 2408 + } 2402 2409 } 2403 2410 2404 2411 buf.address = __psp_pa(data); 2405 2412 ret = __sev_do_cmd_locked(SEV_CMD_SNP_PLATFORM_STATUS, &buf, &argp->error); 2406 2413 2407 2407 - /* 2408 2408 - * Status page will be transitioned to Reclaim state upon success, or 2409 2409 - * left in Firmware state in failure. Use snp_reclaim_pages() to 2410 2410 - * transition either case back to Hypervisor-owned state. 2411 2411 - */ 2412 2412 - if (snp_reclaim_pages(__pa(data), 1, true)) 2413 2413 - return -EFAULT; 2414 2414 + if (sev->snp_initialized) { 2415 2415 + /* 2416 2416 + * The status page will be in Reclaim state on success, or left 2417 2417 + * in Firmware state on failure. Use snp_reclaim_pages() to 2418 2418 + * transition either case back to Hypervisor-owned state. 2419 2419 + */ 2420 2420 + if (snp_reclaim_pages(__pa(data), 1, true)) { 2421 2421 + snp_leak_pages(__page_to_pfn(status_page), 1); 2422 2422 + return -EFAULT; 2423 2423 + } 2424 2424 + } 2414 2425 2415 2426 if (ret) 2416 2427 goto cleanup; ··· 2424 2427 ret = -EFAULT; 2425 2428 2426 2429 cleanup: 2427 2427 - if (shutdown_required) 2428 2428 - __sev_snp_shutdown_locked(&error, false); 2429 2429 - 2430 2430 __free_pages(status_page, 0); 2431 2431 return ret; 2432 2432 } ··· 2772 2778 get_order(NV_LENGTH), 2773 2779 true); 2774 2780 sev_init_ex_buffer = NULL; 2775 2775 - } 2776 2776 - 2777 2777 - if (snp_range_list) { 2778 2778 - kfree(snp_range_list); 2779 2779 - snp_range_list = NULL; 2780 2781 } 2781 2782 2782 2783 __sev_snp_shutdown_locked(&error, panic);

+2

drivers/crypto/hisilicon/Kconfig

reviewed

··· 57 57 depends on UACCE || UACCE=n 58 58 depends on ACPI 59 59 select CRYPTO_DEV_HISI_QM 60 60 + select CRYPTO_DEFLATE 61 61 + select CRYPTO_LZ4 60 62 help 61 63 Support for HiSilicon ZIP Driver 62 64

+2 -3

drivers/crypto/hisilicon/hpre/hpre.h

reviewed

··· 94 94 __le64 key; 95 95 __le64 in; 96 96 __le64 out; 97 97 - __le16 tag; 98 98 - __le16 resv2; 99 99 - #define _HPRE_SQE_ALIGN_EXT 7 97 97 + __le64 tag; 98 98 + #define _HPRE_SQE_ALIGN_EXT 6 100 99 __le32 rsvd1[_HPRE_SQE_ALIGN_EXT]; 101 100 }; 102 101

+223 -195

drivers/crypto/hisilicon/hpre/hpre_crypto.c

reviewed

··· 93 93 94 94 char *g; /* m */ 95 95 dma_addr_t dma_g; 96 96 + struct crypto_kpp *soft_tfm; 96 97 }; 97 98 98 99 struct hpre_ecdh_ctx { ··· 104 103 /* low address: x->y */ 105 104 unsigned char *g; 106 105 dma_addr_t dma_g; 106 106 + struct crypto_kpp *soft_tfm; 107 107 }; 108 108 109 109 struct hpre_ctx { 110 110 struct hisi_qp *qp; 111 111 struct device *dev; 112 112 - struct hpre_asym_request **req_list; 113 112 struct hpre *hpre; 114 114 - spinlock_t req_lock; 115 113 unsigned int key_sz; 116 114 bool crt_g2_mode; 117 117 - struct idr req_idr; 118 115 union { 119 116 struct hpre_rsa_ctx rsa; 120 117 struct hpre_dh_ctx dh; ··· 122 123 unsigned int curve_id; 123 124 /* for high performance core */ 124 125 u8 enable_hpcore; 126 126 + bool fallback; 125 127 }; 126 128 127 129 struct hpre_asym_request { ··· 136 136 struct kpp_request *ecdh; 137 137 } areq; 138 138 int err; 139 139 - int req_id; 140 139 hpre_cb cb; 141 140 struct timespec64 req_time; 142 141 }; ··· 150 151 return (hpre_align_sz() - 1) & ~(crypto_tfm_ctx_alignment() - 1); 151 152 } 152 153 153 153 - static int hpre_alloc_req_id(struct hpre_ctx *ctx) 154 154 - { 155 155 - unsigned long flags; 156 156 - int id; 157 157 - 158 158 - spin_lock_irqsave(&ctx->req_lock, flags); 159 159 - id = idr_alloc(&ctx->req_idr, NULL, 0, ctx->qp->sq_depth, GFP_ATOMIC); 160 160 - spin_unlock_irqrestore(&ctx->req_lock, flags); 161 161 - 162 162 - return id; 163 163 - } 164 164 - 165 165 - static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id) 166 166 - { 167 167 - unsigned long flags; 168 168 - 169 169 - spin_lock_irqsave(&ctx->req_lock, flags); 170 170 - idr_remove(&ctx->req_idr, req_id); 171 171 - spin_unlock_irqrestore(&ctx->req_lock, flags); 172 172 - } 173 173 - 174 174 - static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req) 175 175 - { 176 176 - struct hpre_ctx *ctx; 177 177 - struct hpre_dfx *dfx; 178 178 - int id; 179 179 - 180 180 - ctx = hpre_req->ctx; 181 181 - id = hpre_alloc_req_id(ctx); 182 182 - if (unlikely(id < 0)) 183 183 - return -EINVAL; 184 184 - 185 185 - ctx->req_list[id] = hpre_req; 186 186 - hpre_req->req_id = id; 187 187 - 188 188 - dfx = ctx->hpre->debug.dfx; 189 189 - if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) 190 190 - ktime_get_ts64(&hpre_req->req_time); 191 191 - 192 192 - return id; 193 193 - } 194 194 - 195 195 - static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) 154 154 + static void hpre_dfx_add_req_time(struct hpre_asym_request *hpre_req) 196 155 { 197 156 struct hpre_ctx *ctx = hpre_req->ctx; 198 198 - int id = hpre_req->req_id; 157 157 + struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 199 158 200 200 - if (hpre_req->req_id >= 0) { 201 201 - hpre_req->req_id = HPRE_INVLD_REQ_ID; 202 202 - ctx->req_list[id] = NULL; 203 203 - hpre_free_req_id(ctx, id); 204 204 - } 205 205 - } 206 206 - 207 207 - static struct hisi_qp *hpre_get_qp_and_start(u8 type) 208 208 - { 209 209 - struct hisi_qp *qp; 210 210 - int ret; 211 211 - 212 212 - qp = hpre_create_qp(type); 213 213 - if (!qp) { 214 214 - pr_err("Can not create hpre qp!\n"); 215 215 - return ERR_PTR(-ENODEV); 216 216 - } 217 217 - 218 218 - ret = hisi_qm_start_qp(qp, 0); 219 219 - if (ret < 0) { 220 220 - hisi_qm_free_qps(&qp, 1); 221 221 - pci_err(qp->qm->pdev, "Can not start qp!\n"); 222 222 - return ERR_PTR(-EINVAL); 223 223 - } 224 224 - 225 225 - return qp; 159 159 + if (atomic64_read(&dfx[HPRE_OVERTIME_THRHLD].value)) 160 160 + ktime_get_ts64(&hpre_req->req_time); 226 161 } 227 162 228 163 static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, ··· 273 340 static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, 274 341 void **kreq) 275 342 { 276 276 - struct hpre_asym_request *req; 277 343 unsigned int err, done, alg; 278 278 - int id; 279 344 280 345 #define HPRE_NO_HW_ERR 0 281 346 #define HPRE_HW_TASK_DONE 3 282 347 #define HREE_HW_ERR_MASK GENMASK(10, 0) 283 348 #define HREE_SQE_DONE_MASK GENMASK(1, 0) 284 349 #define HREE_ALG_TYPE_MASK GENMASK(4, 0) 285 285 - id = (int)le16_to_cpu(sqe->tag); 286 286 - req = ctx->req_list[id]; 287 287 - hpre_rm_req_from_ctx(req); 288 288 - *kreq = req; 350 350 + *kreq = (void *)le64_to_cpu(sqe->tag); 289 351 290 352 err = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_ALG_BITS) & 291 353 HREE_HW_ERR_MASK; 292 292 - 293 354 done = (le32_to_cpu(sqe->dw0) >> HPRE_SQE_DONE_SHIFT) & 294 355 HREE_SQE_DONE_MASK; 295 295 - 296 356 if (likely(err == HPRE_NO_HW_ERR && done == HPRE_HW_TASK_DONE)) 297 357 return 0; 298 358 ··· 296 370 return -EINVAL; 297 371 } 298 372 299 299 - static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) 300 300 - { 301 301 - struct hpre *hpre; 302 302 - 303 303 - if (!ctx || !qp || qlen < 0) 304 304 - return -EINVAL; 305 305 - 306 306 - spin_lock_init(&ctx->req_lock); 307 307 - ctx->qp = qp; 308 308 - ctx->dev = &qp->qm->pdev->dev; 309 309 - 310 310 - hpre = container_of(ctx->qp->qm, struct hpre, qm); 311 311 - ctx->hpre = hpre; 312 312 - ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL); 313 313 - if (!ctx->req_list) 314 314 - return -ENOMEM; 315 315 - ctx->key_sz = 0; 316 316 - ctx->crt_g2_mode = false; 317 317 - idr_init(&ctx->req_idr); 318 318 - 319 319 - return 0; 320 320 - } 321 321 - 322 373 static void hpre_ctx_clear(struct hpre_ctx *ctx, bool is_clear_all) 323 374 { 324 324 - if (is_clear_all) { 325 325 - idr_destroy(&ctx->req_idr); 326 326 - kfree(ctx->req_list); 375 375 + if (is_clear_all) 327 376 hisi_qm_free_qps(&ctx->qp, 1); 328 328 - } 329 377 330 378 ctx->crt_g2_mode = false; 331 379 ctx->key_sz = 0; ··· 367 467 368 468 static void hpre_alg_cb(struct hisi_qp *qp, void *resp) 369 469 { 370 370 - struct hpre_ctx *ctx = qp->qp_ctx; 371 371 - struct hpre_dfx *dfx = ctx->hpre->debug.dfx; 470 470 + struct hpre_asym_request *h_req; 372 471 struct hpre_sqe *sqe = resp; 373 373 - struct hpre_asym_request *req = ctx->req_list[le16_to_cpu(sqe->tag)]; 374 472 375 375 - if (unlikely(!req)) { 376 376 - atomic64_inc(&dfx[HPRE_INVALID_REQ_CNT].value); 473 473 + h_req = (struct hpre_asym_request *)le64_to_cpu(sqe->tag); 474 474 + if (unlikely(!h_req)) { 475 475 + pr_err("Failed to get request, and qp_id is %u\n", qp->qp_id); 377 476 return; 378 477 } 379 478 380 380 - req->cb(ctx, resp); 381 381 - } 382 382 - 383 383 - static void hpre_stop_qp_and_put(struct hisi_qp *qp) 384 384 - { 385 385 - hisi_qm_stop_qp(qp); 386 386 - hisi_qm_free_qps(&qp, 1); 479 479 + h_req->cb(h_req->ctx, resp); 387 480 } 388 481 389 482 static int hpre_ctx_init(struct hpre_ctx *ctx, u8 type) 390 483 { 391 484 struct hisi_qp *qp; 392 392 - int ret; 485 485 + struct hpre *hpre; 393 486 394 394 - qp = hpre_get_qp_and_start(type); 395 395 - if (IS_ERR(qp)) 396 396 - return PTR_ERR(qp); 487 487 + qp = hpre_create_qp(type); 488 488 + if (!qp) { 489 489 + ctx->qp = NULL; 490 490 + return -ENODEV; 491 491 + } 397 492 398 398 - qp->qp_ctx = ctx; 399 493 qp->req_cb = hpre_alg_cb; 494 494 + ctx->qp = qp; 495 495 + ctx->dev = &qp->qm->pdev->dev; 496 496 + hpre = container_of(ctx->qp->qm, struct hpre, qm); 497 497 + ctx->hpre = hpre; 498 498 + ctx->key_sz = 0; 499 499 + ctx->crt_g2_mode = false; 400 500 401 401 - ret = hpre_ctx_set(ctx, qp, qp->sq_depth); 402 402 - if (ret) 403 403 - hpre_stop_qp_and_put(qp); 404 404 - 405 405 - return ret; 501 501 + return 0; 406 502 } 407 503 408 504 static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) 409 505 { 410 506 struct hpre_asym_request *h_req; 411 507 struct hpre_sqe *msg; 412 412 - int req_id; 413 508 void *tmp; 414 509 415 510 if (is_rsa) { ··· 444 549 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 445 550 h_req->ctx = ctx; 446 551 447 447 - req_id = hpre_add_req_to_ctx(h_req); 448 448 - if (req_id < 0) 449 449 - return -EBUSY; 450 450 - 451 451 - msg->tag = cpu_to_le16((u16)req_id); 552 552 + hpre_dfx_add_req_time(h_req); 553 553 + msg->tag = cpu_to_le64((uintptr_t)h_req); 452 554 453 555 return 0; 454 556 } ··· 458 566 459 567 do { 460 568 atomic64_inc(&dfx[HPRE_SEND_CNT].value); 461 461 - spin_lock_bh(&ctx->req_lock); 462 569 ret = hisi_qp_send(ctx->qp, msg); 463 463 - spin_unlock_bh(&ctx->req_lock); 464 570 if (ret != -EBUSY) 465 571 break; 466 572 atomic64_inc(&dfx[HPRE_SEND_BUSY_CNT].value); ··· 509 619 return -EINPROGRESS; 510 620 511 621 clear_all: 512 512 - hpre_rm_req_from_ctx(hpre_req); 513 622 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 514 623 515 624 return ret; 625 625 + } 626 626 + 627 627 + static struct kpp_request *hpre_dh_prepare_fb_req(struct kpp_request *req) 628 628 + { 629 629 + struct kpp_request *fb_req = kpp_request_ctx(req); 630 630 + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 631 631 + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 632 632 + 633 633 + kpp_request_set_tfm(fb_req, ctx->dh.soft_tfm); 634 634 + kpp_request_set_callback(fb_req, req->base.flags, req->base.complete, req->base.data); 635 635 + kpp_request_set_input(fb_req, req->src, req->src_len); 636 636 + kpp_request_set_output(fb_req, req->dst, req->dst_len); 637 637 + 638 638 + return fb_req; 639 639 + } 640 640 + 641 641 + static int hpre_dh_generate_public_key(struct kpp_request *req) 642 642 + { 643 643 + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 644 644 + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 645 645 + struct kpp_request *fb_req; 646 646 + 647 647 + if (ctx->fallback) { 648 648 + fb_req = hpre_dh_prepare_fb_req(req); 649 649 + return crypto_kpp_generate_public_key(fb_req); 650 650 + } 651 651 + 652 652 + return hpre_dh_compute_value(req); 653 653 + } 654 654 + 655 655 + static int hpre_dh_compute_shared_secret(struct kpp_request *req) 656 656 + { 657 657 + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 658 658 + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 659 659 + struct kpp_request *fb_req; 660 660 + 661 661 + if (ctx->fallback) { 662 662 + fb_req = hpre_dh_prepare_fb_req(req); 663 663 + return crypto_kpp_compute_shared_secret(fb_req); 664 664 + } 665 665 + 666 666 + return hpre_dh_compute_value(req); 516 667 } 517 668 518 669 static int hpre_is_dh_params_length_valid(unsigned int key_sz) ··· 581 650 { 582 651 struct device *dev = ctx->dev; 583 652 unsigned int sz; 584 584 - 585 585 - if (params->p_size > HPRE_DH_MAX_P_SZ) 586 586 - return -EINVAL; 587 587 - 588 588 - if (hpre_is_dh_params_length_valid(params->p_size << 589 589 - HPRE_BITS_2_BYTES_SHIFT)) 590 590 - return -EINVAL; 591 653 592 654 sz = ctx->key_sz = params->p_size; 593 655 ctx->dh.xa_p = dma_alloc_coherent(dev, sz << 1, ··· 614 690 struct device *dev = ctx->dev; 615 691 unsigned int sz = ctx->key_sz; 616 692 617 617 - if (is_clear_all) 618 618 - hisi_qm_stop_qp(ctx->qp); 693 693 + if (!ctx->qp) 694 694 + return; 619 695 620 696 if (ctx->dh.g) { 621 697 dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g); ··· 642 718 if (crypto_dh_decode_key(buf, len, &params) < 0) 643 719 return -EINVAL; 644 720 721 721 + if (!ctx->qp) 722 722 + goto set_soft_secret; 723 723 + 724 724 + if (hpre_is_dh_params_length_valid(params.p_size << 725 725 + HPRE_BITS_2_BYTES_SHIFT)) 726 726 + goto set_soft_secret; 727 727 + 645 728 /* Free old secret if any */ 646 729 hpre_dh_clear_ctx(ctx, false); 647 730 ··· 659 728 memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key, 660 729 params.key_size); 661 730 731 731 + ctx->fallback = false; 662 732 return 0; 663 733 664 734 err_clear_ctx: 665 735 hpre_dh_clear_ctx(ctx, false); 666 736 return ret; 737 737 + set_soft_secret: 738 738 + ctx->fallback = true; 739 739 + return crypto_kpp_set_secret(ctx->dh.soft_tfm, buf, len); 667 740 } 668 741 669 742 static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm) 670 743 { 671 744 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 745 745 + 746 746 + if (ctx->fallback) 747 747 + return crypto_kpp_maxsize(ctx->dh.soft_tfm); 672 748 673 749 return ctx->key_sz; 674 750 } ··· 683 745 static int hpre_dh_init_tfm(struct crypto_kpp *tfm) 684 746 { 685 747 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 748 748 + const char *alg = kpp_alg_name(tfm); 749 749 + unsigned int reqsize; 750 750 + int ret; 686 751 687 687 - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 752 752 + ctx->dh.soft_tfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); 753 753 + if (IS_ERR(ctx->dh.soft_tfm)) { 754 754 + pr_err("Failed to alloc dh tfm!\n"); 755 755 + return PTR_ERR(ctx->dh.soft_tfm); 756 756 + } 688 757 689 689 - return hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); 758 758 + crypto_kpp_set_flags(ctx->dh.soft_tfm, crypto_kpp_get_flags(tfm)); 759 759 + 760 760 + reqsize = max(sizeof(struct hpre_asym_request) + hpre_align_pd(), 761 761 + sizeof(struct kpp_request) + crypto_kpp_reqsize(ctx->dh.soft_tfm)); 762 762 + kpp_set_reqsize(tfm, reqsize); 763 763 + 764 764 + ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); 765 765 + if (ret && ret != -ENODEV) { 766 766 + crypto_free_kpp(ctx->dh.soft_tfm); 767 767 + return ret; 768 768 + } else if (ret == -ENODEV) { 769 769 + ctx->fallback = true; 770 770 + } 771 771 + 772 772 + return 0; 690 773 } 691 774 692 775 static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) ··· 715 756 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 716 757 717 758 hpre_dh_clear_ctx(ctx, true); 759 759 + crypto_free_kpp(ctx->dh.soft_tfm); 718 760 } 719 761 720 762 static void hpre_rsa_drop_leading_zeros(const char **ptr, size_t *len) ··· 755 795 struct hpre_sqe *msg = &hpre_req->req; 756 796 int ret; 757 797 758 758 - /* For 512 and 1536 bits key size, use soft tfm instead */ 759 759 - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 760 760 - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { 798 798 + /* For unsupported key size and unavailable devices, use soft tfm instead */ 799 799 + if (ctx->fallback) { 761 800 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); 762 801 ret = crypto_akcipher_encrypt(req); 763 802 akcipher_request_set_tfm(req, tfm); ··· 787 828 return -EINPROGRESS; 788 829 789 830 clear_all: 790 790 - hpre_rm_req_from_ctx(hpre_req); 791 831 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 792 832 793 833 return ret; ··· 801 843 struct hpre_sqe *msg = &hpre_req->req; 802 844 int ret; 803 845 804 804 - /* For 512 and 1536 bits key size, use soft tfm instead */ 805 805 - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 806 806 - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) { 846 846 + /* For unsupported key size and unavailable devices, use soft tfm instead */ 847 847 + if (ctx->fallback) { 807 848 akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); 808 849 ret = crypto_akcipher_decrypt(req); 809 850 akcipher_request_set_tfm(req, tfm); ··· 840 883 return -EINPROGRESS; 841 884 842 885 clear_all: 843 843 - hpre_rm_req_from_ctx(hpre_req); 844 886 hpre_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 845 887 846 888 return ret; ··· 855 899 ctx->key_sz = vlen; 856 900 857 901 /* if invalid key size provided, we use software tfm */ 858 858 - if (!hpre_rsa_key_size_is_support(ctx->key_sz)) 902 902 + if (!hpre_rsa_key_size_is_support(ctx->key_sz)) { 903 903 + ctx->fallback = true; 859 904 return 0; 905 905 + } 860 906 861 907 ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1, 862 908 &ctx->rsa.dma_pubkey, ··· 993 1035 unsigned int half_key_sz = ctx->key_sz >> 1; 994 1036 struct device *dev = ctx->dev; 995 1037 996 996 - if (is_clear_all) 997 997 - hisi_qm_stop_qp(ctx->qp); 1038 1038 + if (!ctx->qp) 1039 1039 + return; 998 1040 999 1041 if (ctx->rsa.pubkey) { 1000 1042 dma_free_coherent(dev, ctx->key_sz << 1, ··· 1075 1117 goto free; 1076 1118 } 1077 1119 1120 1120 + ctx->fallback = false; 1078 1121 return 0; 1079 1122 1080 1123 free: ··· 1093 1134 if (ret) 1094 1135 return ret; 1095 1136 1137 1137 + if (!ctx->qp) 1138 1138 + return 0; 1139 1139 + 1096 1140 return hpre_rsa_setkey(ctx, key, keylen, false); 1097 1141 } 1098 1142 ··· 1109 1147 if (ret) 1110 1148 return ret; 1111 1149 1150 1150 + if (!ctx->qp) 1151 1151 + return 0; 1152 1152 + 1112 1153 return hpre_rsa_setkey(ctx, key, keylen, true); 1113 1154 } 1114 1155 ··· 1119 1154 { 1120 1155 struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); 1121 1156 1122 1122 - /* For 512 and 1536 bits key size, use soft tfm instead */ 1123 1123 - if (ctx->key_sz == HPRE_RSA_512BITS_KSZ || 1124 1124 - ctx->key_sz == HPRE_RSA_1536BITS_KSZ) 1157 1157 + /* For unsupported key size and unavailable devices, use soft tfm instead */ 1158 1158 + if (ctx->fallback) 1125 1159 return crypto_akcipher_maxsize(ctx->rsa.soft_tfm); 1126 1160 1127 1161 return ctx->key_sz; ··· 1141 1177 hpre_align_pd()); 1142 1178 1143 1179 ret = hpre_ctx_init(ctx, HPRE_V2_ALG_TYPE); 1144 1144 - if (ret) 1180 1180 + if (ret && ret != -ENODEV) { 1145 1181 crypto_free_akcipher(ctx->rsa.soft_tfm); 1182 1182 + return ret; 1183 1183 + } else if (ret == -ENODEV) { 1184 1184 + ctx->fallback = true; 1185 1185 + } 1146 1186 1147 1147 - return ret; 1187 1187 + return 0; 1148 1188 } 1149 1189 1150 1190 static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) ··· 1174 1206 struct device *dev = ctx->dev; 1175 1207 unsigned int sz = ctx->key_sz; 1176 1208 unsigned int shift = sz << 1; 1177 1177 - 1178 1178 - if (is_clear_all) 1179 1179 - hisi_qm_stop_qp(ctx->qp); 1180 1209 1181 1210 if (ctx->ecdh.p) { 1182 1211 /* ecdh: p->a->k->b */ ··· 1311 1346 return 0; 1312 1347 } 1313 1348 1314 1314 - static bool hpre_key_is_zero(char *key, unsigned short key_sz) 1349 1349 + static bool hpre_key_is_zero(const char *key, unsigned short key_sz) 1315 1350 { 1316 1351 int i; 1317 1352 ··· 1351 1386 char key[HPRE_ECC_MAX_KSZ]; 1352 1387 struct ecdh params; 1353 1388 int ret; 1389 1389 + 1390 1390 + if (ctx->fallback) 1391 1391 + return crypto_kpp_set_secret(ctx->ecdh.soft_tfm, buf, len); 1354 1392 1355 1393 if (crypto_ecdh_decode_key(buf, len, &params) < 0) { 1356 1394 dev_err(dev, "failed to decode ecdh key!\n"); ··· 1456 1488 { 1457 1489 struct hpre_asym_request *h_req; 1458 1490 struct hpre_sqe *msg; 1459 1459 - int req_id; 1460 1491 void *tmp; 1461 1492 1462 1493 if (req->dst_len < ctx->key_sz << 1) { ··· 1477 1510 msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; 1478 1511 h_req->ctx = ctx; 1479 1512 1480 1480 - req_id = hpre_add_req_to_ctx(h_req); 1481 1481 - if (req_id < 0) 1482 1482 - return -EBUSY; 1483 1483 - 1484 1484 - msg->tag = cpu_to_le16((u16)req_id); 1513 1513 + hpre_dfx_add_req_time(h_req); 1514 1514 + msg->tag = cpu_to_le64((uintptr_t)h_req); 1485 1515 return 0; 1486 1516 } 1487 1517 ··· 1576 1612 return -EINPROGRESS; 1577 1613 1578 1614 clear_all: 1579 1579 - hpre_rm_req_from_ctx(hpre_req); 1580 1615 hpre_ecdh_hw_data_clr_all(ctx, hpre_req, req->dst, req->src); 1581 1616 return ret; 1617 1617 + } 1618 1618 + 1619 1619 + static int hpre_ecdh_generate_public_key(struct kpp_request *req) 1620 1620 + { 1621 1621 + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 1622 1622 + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1623 1623 + int ret; 1624 1624 + 1625 1625 + if (ctx->fallback) { 1626 1626 + kpp_request_set_tfm(req, ctx->ecdh.soft_tfm); 1627 1627 + ret = crypto_kpp_generate_public_key(req); 1628 1628 + kpp_request_set_tfm(req, tfm); 1629 1629 + return ret; 1630 1630 + } 1631 1631 + 1632 1632 + return hpre_ecdh_compute_value(req); 1633 1633 + } 1634 1634 + 1635 1635 + static int hpre_ecdh_compute_shared_secret(struct kpp_request *req) 1636 1636 + { 1637 1637 + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); 1638 1638 + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1639 1639 + int ret; 1640 1640 + 1641 1641 + if (ctx->fallback) { 1642 1642 + kpp_request_set_tfm(req, ctx->ecdh.soft_tfm); 1643 1643 + ret = crypto_kpp_compute_shared_secret(req); 1644 1644 + kpp_request_set_tfm(req, tfm); 1645 1645 + return ret; 1646 1646 + } 1647 1647 + 1648 1648 + return hpre_ecdh_compute_value(req); 1582 1649 } 1583 1650 1584 1651 static unsigned int hpre_ecdh_max_size(struct crypto_kpp *tfm) 1585 1652 { 1586 1653 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1587 1654 1655 1655 + if (ctx->fallback) 1656 1656 + return crypto_kpp_maxsize(ctx->ecdh.soft_tfm); 1657 1657 + 1588 1658 /* max size is the pub_key_size, include x and y */ 1589 1659 return ctx->key_sz << 1; 1660 1660 + } 1661 1661 + 1662 1662 + static int hpre_ecdh_init_tfm(struct crypto_kpp *tfm) 1663 1663 + { 1664 1664 + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1665 1665 + const char *alg = kpp_alg_name(tfm); 1666 1666 + int ret; 1667 1667 + 1668 1668 + ret = hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1669 1669 + if (!ret) { 1670 1670 + kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1671 1671 + return 0; 1672 1672 + } else if (ret && ret != -ENODEV) { 1673 1673 + return ret; 1674 1674 + } 1675 1675 + 1676 1676 + ctx->ecdh.soft_tfm = crypto_alloc_kpp(alg, 0, CRYPTO_ALG_NEED_FALLBACK); 1677 1677 + if (IS_ERR(ctx->ecdh.soft_tfm)) { 1678 1678 + pr_err("Failed to alloc %s tfm!\n", alg); 1679 1679 + return PTR_ERR(ctx->ecdh.soft_tfm); 1680 1680 + } 1681 1681 + 1682 1682 + crypto_kpp_set_flags(ctx->ecdh.soft_tfm, crypto_kpp_get_flags(tfm)); 1683 1683 + ctx->fallback = true; 1684 1684 + 1685 1685 + return 0; 1590 1686 } 1591 1687 1592 1688 static int hpre_ecdh_nist_p192_init_tfm(struct crypto_kpp *tfm) ··· 1655 1631 1656 1632 ctx->curve_id = ECC_CURVE_NIST_P192; 1657 1633 1658 1658 - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1659 1659 - 1660 1660 - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1634 1634 + return hpre_ecdh_init_tfm(tfm); 1661 1635 } 1662 1636 1663 1637 static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) ··· 1665 1643 ctx->curve_id = ECC_CURVE_NIST_P256; 1666 1644 ctx->enable_hpcore = 1; 1667 1645 1668 1668 - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1669 1669 - 1670 1670 - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1646 1646 + return hpre_ecdh_init_tfm(tfm); 1671 1647 } 1672 1648 1673 1649 static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) ··· 1674 1654 1675 1655 ctx->curve_id = ECC_CURVE_NIST_P384; 1676 1656 1677 1677 - kpp_set_reqsize(tfm, sizeof(struct hpre_asym_request) + hpre_align_pd()); 1678 1678 - 1679 1679 - return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); 1657 1657 + return hpre_ecdh_init_tfm(tfm); 1680 1658 } 1681 1659 1682 1660 static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) 1683 1661 { 1684 1662 struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); 1663 1663 + 1664 1664 + if (ctx->fallback) { 1665 1665 + crypto_free_kpp(ctx->ecdh.soft_tfm); 1666 1666 + return; 1667 1667 + } 1685 1668 1686 1669 hpre_ecc_clear_ctx(ctx, true); 1687 1670 } ··· 1703 1680 .cra_name = "rsa", 1704 1681 .cra_driver_name = "hpre-rsa", 1705 1682 .cra_module = THIS_MODULE, 1683 1683 + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1706 1684 }, 1707 1685 }; 1708 1686 1709 1687 static struct kpp_alg dh = { 1710 1688 .set_secret = hpre_dh_set_secret, 1711 1711 - .generate_public_key = hpre_dh_compute_value, 1712 1712 - .compute_shared_secret = hpre_dh_compute_value, 1689 1689 + .generate_public_key = hpre_dh_generate_public_key, 1690 1690 + .compute_shared_secret = hpre_dh_compute_shared_secret, 1713 1691 .max_size = hpre_dh_max_size, 1714 1692 .init = hpre_dh_init_tfm, 1715 1693 .exit = hpre_dh_exit_tfm, ··· 1720 1696 .cra_name = "dh", 1721 1697 .cra_driver_name = "hpre-dh", 1722 1698 .cra_module = THIS_MODULE, 1699 1699 + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1723 1700 }, 1724 1701 }; 1725 1702 1726 1703 static struct kpp_alg ecdh_curves[] = { 1727 1704 { 1728 1705 .set_secret = hpre_ecdh_set_secret, 1729 1729 - .generate_public_key = hpre_ecdh_compute_value, 1730 1730 - .compute_shared_secret = hpre_ecdh_compute_value, 1706 1706 + .generate_public_key = hpre_ecdh_generate_public_key, 1707 1707 + .compute_shared_secret = hpre_ecdh_compute_shared_secret, 1731 1708 .max_size = hpre_ecdh_max_size, 1732 1709 .init = hpre_ecdh_nist_p192_init_tfm, 1733 1710 .exit = hpre_ecdh_exit_tfm, ··· 1738 1713 .cra_name = "ecdh-nist-p192", 1739 1714 .cra_driver_name = "hpre-ecdh-nist-p192", 1740 1715 .cra_module = THIS_MODULE, 1716 1716 + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1741 1717 }, 1742 1718 }, { 1743 1719 .set_secret = hpre_ecdh_set_secret, 1744 1744 - .generate_public_key = hpre_ecdh_compute_value, 1745 1745 - .compute_shared_secret = hpre_ecdh_compute_value, 1720 1720 + .generate_public_key = hpre_ecdh_generate_public_key, 1721 1721 + .compute_shared_secret = hpre_ecdh_compute_shared_secret, 1746 1722 .max_size = hpre_ecdh_max_size, 1747 1723 .init = hpre_ecdh_nist_p256_init_tfm, 1748 1724 .exit = hpre_ecdh_exit_tfm, ··· 1753 1727 .cra_name = "ecdh-nist-p256", 1754 1728 .cra_driver_name = "hpre-ecdh-nist-p256", 1755 1729 .cra_module = THIS_MODULE, 1730 1730 + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1756 1731 }, 1757 1732 }, { 1758 1733 .set_secret = hpre_ecdh_set_secret, 1759 1759 - .generate_public_key = hpre_ecdh_compute_value, 1760 1760 - .compute_shared_secret = hpre_ecdh_compute_value, 1734 1734 + .generate_public_key = hpre_ecdh_generate_public_key, 1735 1735 + .compute_shared_secret = hpre_ecdh_compute_shared_secret, 1761 1736 .max_size = hpre_ecdh_max_size, 1762 1737 .init = hpre_ecdh_nist_p384_init_tfm, 1763 1738 .exit = hpre_ecdh_exit_tfm, ··· 1768 1741 .cra_name = "ecdh-nist-p384", 1769 1742 .cra_driver_name = "hpre-ecdh-nist-p384", 1770 1743 .cra_module = THIS_MODULE, 1744 1744 + .cra_flags = CRYPTO_ALG_NEED_FALLBACK, 1771 1745 }, 1772 1746 } 1773 1747 };

+1 -1

drivers/crypto/hisilicon/hpre/hpre_main.c

reviewed

··· 465 465 * type: 0 - RSA/DH. algorithm supported in V2, 466 466 * 1 - ECC algorithm in V3. 467 467 */ 468 468 - ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, type, node, &qp); 468 468 + ret = hisi_qm_alloc_qps_node(&hpre_devices, 1, &type, node, &qp); 469 469 if (!ret) 470 470 return qp; 471 471

+274 -113

drivers/crypto/hisilicon/qm.c

reviewed

··· 31 31 /* mailbox */ 32 32 #define QM_MB_PING_ALL_VFS 0xffff 33 33 #define QM_MB_STATUS_MASK GENMASK(12, 9) 34 34 + #define QM_MB_BUSY_MASK BIT(13) 35 35 + #define QM_MB_MAX_WAIT_TIMEOUT USEC_PER_SEC 36 36 + #define QM_MB_MAX_STOP_TIMEOUT (5 * USEC_PER_SEC) 34 37 35 38 /* sqc shift */ 36 39 #define QM_SQ_HOP_NUM_SHIFT 0 ··· 191 188 #define QM_IFC_INT_DISABLE BIT(0) 192 189 #define QM_IFC_INT_STATUS_MASK BIT(0) 193 190 #define QM_IFC_INT_SET_MASK BIT(0) 194 194 - #define QM_WAIT_DST_ACK 10 195 195 - #define QM_MAX_PF_WAIT_COUNT 10 191 191 + #define QM_WAIT_DST_ACK 1000 192 192 + #define QM_MAX_PF_WAIT_COUNT 20 196 193 #define QM_MAX_VF_WAIT_COUNT 40 197 194 #define QM_VF_RESET_WAIT_US 20000 198 195 #define QM_VF_RESET_WAIT_CNT 3000 ··· 585 582 mailbox->rsvd = 0; 586 583 } 587 584 588 588 - /* return 0 mailbox ready, -ETIMEDOUT hardware timeout */ 589 589 - int hisi_qm_wait_mb_ready(struct hisi_qm *qm) 585 585 + /* 586 586 + * The mailbox is 128 bits and requires a single read/write operation. 587 587 + * Since there is no general 128-bit IO memory access API in the current 588 588 + * ARM64 architecture, this needs to be implemented in the driver. 589 589 + */ 590 590 + static struct qm_mailbox qm_mb_read(struct hisi_qm *qm) 590 591 { 591 591 - u32 val; 592 592 + struct qm_mailbox mailbox = {0}; 592 593 593 593 - return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE, 594 594 - val, !((val >> QM_MB_BUSY_SHIFT) & 595 595 - 0x1), POLL_PERIOD, POLL_TIMEOUT); 594 594 + #if IS_ENABLED(CONFIG_ARM64) 595 595 + const void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE; 596 596 + unsigned long tmp0, tmp1; 597 597 + 598 598 + asm volatile("ldp %0, %1, %3\n" 599 599 + "stp %0, %1, %2\n" 600 600 + : "=&r" (tmp0), 601 601 + "=&r" (tmp1), 602 602 + "+Q" (mailbox) 603 603 + : "Q" (*((char __iomem *)fun_base)) 604 604 + : "memory"); 605 605 + #endif 606 606 + 607 607 + return mailbox; 596 608 } 597 597 - EXPORT_SYMBOL_GPL(hisi_qm_wait_mb_ready); 598 609 599 610 /* 128 bit should be written to hardware at one time to trigger a mailbox */ 600 611 static void qm_mb_write(struct hisi_qm *qm, const void *src) 601 612 { 613 613 + #if IS_ENABLED(CONFIG_ARM64) 602 614 void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE; 603 603 - 604 604 - #if IS_ENABLED(CONFIG_ARM64) 605 605 - unsigned long tmp0 = 0, tmp1 = 0; 606 606 - #endif 607 607 - 608 608 - if (!IS_ENABLED(CONFIG_ARM64)) { 609 609 - memcpy_toio(fun_base, src, 16); 610 610 - dma_wmb(); 611 611 - return; 612 612 - } 613 613 - 614 614 - #if IS_ENABLED(CONFIG_ARM64) 615 615 + unsigned long tmp0, tmp1; 616 616 + /* 617 617 + * The dmb oshst instruction ensures that the data in the 618 618 + * mailbox is written before it is sent to the hardware. 619 619 + */ 615 620 asm volatile("ldp %0, %1, %3\n" 616 616 - "stp %0, %1, %2\n" 617 621 "dmb oshst\n" 622 622 + "stp %0, %1, %2\n" 618 623 : "=&r" (tmp0), 619 624 "=&r" (tmp1), 620 625 "+Q" (*((char __iomem *)fun_base)) ··· 631 620 #endif 632 621 } 633 622 634 634 - static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox) 623 623 + int hisi_qm_wait_mb_ready(struct hisi_qm *qm) 635 624 { 625 625 + struct qm_mailbox mailbox = {0}; 636 626 int ret; 637 637 - u32 val; 638 627 639 639 - if (unlikely(hisi_qm_wait_mb_ready(qm))) { 628 628 + ret = read_poll_timeout(qm_mb_read, mailbox, 629 629 + !(le16_to_cpu(mailbox.w0) & QM_MB_BUSY_MASK), 630 630 + POLL_PERIOD, POLL_TIMEOUT, 631 631 + true, qm); 632 632 + if (ret) 640 633 dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n"); 641 641 - ret = -EBUSY; 642 642 - goto mb_busy; 634 634 + 635 635 + return ret; 636 636 + } 637 637 + EXPORT_SYMBOL_GPL(hisi_qm_wait_mb_ready); 638 638 + 639 639 + static int qm_wait_mb_finish(struct hisi_qm *qm, struct qm_mailbox *mailbox, u32 wait_timeout) 640 640 + { 641 641 + struct device *dev = &qm->pdev->dev; 642 642 + int ret; 643 643 + 644 644 + ret = read_poll_timeout(qm_mb_read, *mailbox, 645 645 + !(le16_to_cpu(mailbox->w0) & QM_MB_BUSY_MASK), 646 646 + POLL_PERIOD, wait_timeout, 647 647 + true, qm); 648 648 + if (ret) { 649 649 + dev_err(dev, "QM mailbox operation timeout!\n"); 650 650 + return ret; 643 651 } 644 652 645 645 - qm_mb_write(qm, mailbox); 646 646 - 647 647 - if (unlikely(hisi_qm_wait_mb_ready(qm))) { 648 648 - dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n"); 649 649 - ret = -ETIMEDOUT; 650 650 - goto mb_busy; 651 651 - } 652 652 - 653 653 - val = readl(qm->io_base + QM_MB_CMD_SEND_BASE); 654 654 - if (val & QM_MB_STATUS_MASK) { 655 655 - dev_err(&qm->pdev->dev, "QM mailbox operation failed!\n"); 656 656 - ret = -EIO; 657 657 - goto mb_busy; 653 653 + if (le16_to_cpu(mailbox->w0) & QM_MB_STATUS_MASK) { 654 654 + dev_err(dev, "QM mailbox operation failed!\n"); 655 655 + return -EIO; 658 656 } 659 657 660 658 return 0; 659 659 + } 661 660 662 662 - mb_busy: 661 661 + static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox, u32 wait_timeout) 662 662 + { 663 663 + int ret; 664 664 + 665 665 + ret = hisi_qm_wait_mb_ready(qm); 666 666 + if (ret) 667 667 + goto mb_err_cnt_increase; 668 668 + 669 669 + qm_mb_write(qm, mailbox); 670 670 + 671 671 + ret = qm_wait_mb_finish(qm, mailbox, wait_timeout); 672 672 + if (ret) 673 673 + goto mb_err_cnt_increase; 674 674 + 675 675 + return 0; 676 676 + 677 677 + mb_err_cnt_increase: 663 678 atomic64_inc(&qm->debug.dfx.mb_err_cnt); 664 679 return ret; 665 680 } ··· 694 657 bool op) 695 658 { 696 659 struct qm_mailbox mailbox; 660 660 + u32 wait_timeout; 697 661 int ret; 662 662 + 663 663 + if (cmd == QM_MB_CMD_STOP_QP || cmd == QM_MB_CMD_FLUSH_QM) 664 664 + wait_timeout = QM_MB_MAX_STOP_TIMEOUT; 665 665 + else 666 666 + wait_timeout = QM_MB_MAX_WAIT_TIMEOUT; 667 667 + 668 668 + /* No need to judge if master OOO is blocked. */ 669 669 + if (qm_check_dev_error(qm)) { 670 670 + dev_err(&qm->pdev->dev, "QM mailbox operation failed since qm is stop!\n"); 671 671 + return -EIO; 672 672 + } 698 673 699 674 qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op); 700 675 701 676 mutex_lock(&qm->mailbox_lock); 702 702 - ret = qm_mb_nolock(qm, &mailbox); 677 677 + ret = qm_mb_nolock(qm, &mailbox, wait_timeout); 703 678 mutex_unlock(&qm->mailbox_lock); 704 679 705 680 return ret; 706 681 } 707 682 EXPORT_SYMBOL_GPL(hisi_qm_mb); 683 683 + 684 684 + int hisi_qm_mb_read(struct hisi_qm *qm, u64 *base, u8 cmd, u16 queue) 685 685 + { 686 686 + struct qm_mailbox mailbox; 687 687 + int ret; 688 688 + 689 689 + qm_mb_pre_init(&mailbox, cmd, 0, queue, 1); 690 690 + mutex_lock(&qm->mailbox_lock); 691 691 + ret = qm_mb_nolock(qm, &mailbox, QM_MB_MAX_WAIT_TIMEOUT); 692 692 + mutex_unlock(&qm->mailbox_lock); 693 693 + if (ret) 694 694 + return ret; 695 695 + 696 696 + *base = le32_to_cpu(mailbox.base_l) | 697 697 + ((u64)le32_to_cpu(mailbox.base_h) << 32); 698 698 + 699 699 + return 0; 700 700 + } 701 701 + EXPORT_SYMBOL_GPL(hisi_qm_mb_read); 708 702 709 703 /* op 0: set xqc information to hardware, 1: get xqc information from hardware. */ 710 704 int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op) ··· 783 715 memcpy(tmp_xqc, xqc, size); 784 716 785 717 qm_mb_pre_init(&mailbox, cmd, xqc_dma, qp_id, op); 786 786 - ret = qm_mb_nolock(qm, &mailbox); 718 718 + ret = qm_mb_nolock(qm, &mailbox, QM_MB_MAX_WAIT_TIMEOUT); 787 719 if (!ret && op) 788 720 memcpy(xqc, tmp_xqc, size); 789 721 ··· 1453 1385 u64 sqc_vft; 1454 1386 int ret; 1455 1387 1456 1456 - ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1); 1388 1388 + ret = hisi_qm_mb_read(qm, &sqc_vft, QM_MB_CMD_SQC_VFT_V2, 0); 1457 1389 if (ret) 1458 1390 return ret; 1459 1391 1460 1460 - sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) | 1461 1461 - ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32); 1462 1392 *base = QM_SQC_VFT_BASE_MASK_V2 & (sqc_vft >> QM_SQC_VFT_BASE_SHIFT_V2); 1463 1393 *number = (QM_SQC_VFT_NUM_MASK_V2 & 1464 1394 (sqc_vft >> QM_SQC_VFT_NUM_SHIFT_V2)) + 1; ··· 1594 1528 } 1595 1529 1596 1530 return ACC_ERR_RECOVERED; 1597 1597 - } 1598 1598 - 1599 1599 - static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num) 1600 1600 - { 1601 1601 - struct qm_mailbox mailbox; 1602 1602 - int ret; 1603 1603 - 1604 1604 - qm_mb_pre_init(&mailbox, QM_MB_CMD_DST, 0, fun_num, 0); 1605 1605 - mutex_lock(&qm->mailbox_lock); 1606 1606 - ret = qm_mb_nolock(qm, &mailbox); 1607 1607 - if (ret) 1608 1608 - goto err_unlock; 1609 1609 - 1610 1610 - *msg = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) | 1611 1611 - ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32); 1612 1612 - 1613 1613 - err_unlock: 1614 1614 - mutex_unlock(&qm->mailbox_lock); 1615 1615 - return ret; 1616 1531 } 1617 1532 1618 1533 static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask) ··· 1911 1864 1912 1865 qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, msg, fun_num, 0); 1913 1866 mutex_lock(&qm->mailbox_lock); 1914 1914 - return qm_mb_nolock(qm, &mailbox); 1867 1867 + return qm_mb_nolock(qm, &mailbox, QM_MB_MAX_WAIT_TIMEOUT); 1915 1868 } 1916 1869 1917 1870 static void qm_set_ifc_end_v3(struct hisi_qm *qm) ··· 1924 1877 u64 msg; 1925 1878 int ret; 1926 1879 1927 1927 - ret = qm_get_mb_cmd(qm, &msg, fun_num); 1880 1880 + ret = hisi_qm_mb_read(qm, &msg, QM_MB_CMD_DST, fun_num); 1928 1881 if (ret) 1929 1882 return ret; 1930 1883 ··· 2049 2002 *addr = 0; 2050 2003 } 2051 2004 2052 2052 - static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) 2005 2005 + static struct hisi_qp *find_shareable_qp(struct hisi_qm *qm, u8 alg_type, bool is_in_kernel) 2006 2006 + { 2007 2007 + struct device *dev = &qm->pdev->dev; 2008 2008 + struct hisi_qp *share_qp = NULL; 2009 2009 + struct hisi_qp *qp; 2010 2010 + u32 ref_count = ~0; 2011 2011 + int i; 2012 2012 + 2013 2013 + if (!is_in_kernel) 2014 2014 + goto queues_busy; 2015 2015 + 2016 2016 + for (i = 0; i < qm->qp_num; i++) { 2017 2017 + qp = &qm->qp_array[i]; 2018 2018 + if (qp->is_in_kernel && qp->alg_type == alg_type && qp->ref_count < ref_count) { 2019 2019 + ref_count = qp->ref_count; 2020 2020 + share_qp = qp; 2021 2021 + } 2022 2022 + } 2023 2023 + 2024 2024 + if (share_qp) { 2025 2025 + share_qp->ref_count++; 2026 2026 + return share_qp; 2027 2027 + } 2028 2028 + 2029 2029 + queues_busy: 2030 2030 + dev_info_ratelimited(dev, "All %u queues of QM are busy and no shareable queue\n", 2031 2031 + qm->qp_num); 2032 2032 + atomic64_inc(&qm->debug.dfx.create_qp_err_cnt); 2033 2033 + return ERR_PTR(-EBUSY); 2034 2034 + } 2035 2035 + 2036 2036 + static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type, bool is_in_kernel) 2053 2037 { 2054 2038 struct device *dev = &qm->pdev->dev; 2055 2039 struct hisi_qp *qp; ··· 2091 2013 return ERR_PTR(-EPERM); 2092 2014 } 2093 2015 2094 2094 - if (qm->qp_in_used == qm->qp_num) { 2095 2095 - dev_info_ratelimited(dev, "All %u queues of QM are busy!\n", 2096 2096 - qm->qp_num); 2097 2097 - atomic64_inc(&qm->debug.dfx.create_qp_err_cnt); 2098 2098 - return ERR_PTR(-EBUSY); 2099 2099 - } 2016 2016 + /* Try to find a shareable queue when all queues are busy */ 2017 2017 + if (qm->qp_in_used == qm->qp_num) 2018 2018 + return find_shareable_qp(qm, alg_type, is_in_kernel); 2100 2019 2101 2020 qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC); 2102 2021 if (qp_id < 0) { ··· 2109 2034 2110 2035 qp->event_cb = NULL; 2111 2036 qp->req_cb = NULL; 2112 2112 - qp->qp_id = qp_id; 2113 2037 qp->alg_type = alg_type; 2114 2114 - qp->is_in_kernel = true; 2038 2038 + qp->is_in_kernel = is_in_kernel; 2115 2039 qm->qp_in_used++; 2040 2040 + qp->ref_count = 1; 2116 2041 2117 2042 return qp; 2118 2043 } ··· 2134 2059 return ERR_PTR(ret); 2135 2060 2136 2061 down_write(&qm->qps_lock); 2137 2137 - qp = qm_create_qp_nolock(qm, alg_type); 2062 2062 + qp = qm_create_qp_nolock(qm, alg_type, false); 2138 2063 up_write(&qm->qps_lock); 2139 2064 2140 2065 if (IS_ERR(qp)) ··· 2294 2219 for (i = 0; i < qp_used; i++) { 2295 2220 pos = (i + cur_head) % sq_depth; 2296 2221 qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); 2222 2222 + qm_cq_head_update(qp); 2297 2223 atomic_dec(&qp->qp_status.used); 2298 2224 } 2299 2225 } ··· 2444 2368 int hisi_qp_send(struct hisi_qp *qp, const void *msg) 2445 2369 { 2446 2370 struct hisi_qp_status *qp_status = &qp->qp_status; 2447 2447 - u16 sq_tail = qp_status->sq_tail; 2448 2448 - u16 sq_tail_next = (sq_tail + 1) % qp->sq_depth; 2449 2449 - void *sqe = qm_get_avail_sqe(qp); 2371 2371 + u16 sq_tail, sq_tail_next; 2372 2372 + void *sqe; 2450 2373 2374 2374 + spin_lock_bh(&qp->qp_lock); 2451 2375 if (unlikely(atomic_read(&qp->qp_status.flags) == QP_STOP || 2452 2376 atomic_read(&qp->qm->status.flags) == QM_STOP || 2453 2377 qp->is_resetting)) { 2378 2378 + spin_unlock_bh(&qp->qp_lock); 2454 2379 dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); 2455 2380 return -EAGAIN; 2456 2381 } 2457 2382 2458 2458 - if (!sqe) 2383 2383 + sqe = qm_get_avail_sqe(qp); 2384 2384 + if (!sqe) { 2385 2385 + spin_unlock_bh(&qp->qp_lock); 2459 2386 return -EBUSY; 2387 2387 + } 2460 2388 2389 2389 + sq_tail = qp_status->sq_tail; 2390 2390 + sq_tail_next = (sq_tail + 1) % qp->sq_depth; 2461 2391 memcpy(sqe, msg, qp->qm->sqe_size); 2392 2392 + qp->msg[sq_tail] = msg; 2462 2393 2463 2394 qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0); 2464 2395 atomic_inc(&qp->qp_status.used); 2465 2396 qp_status->sq_tail = sq_tail_next; 2397 2397 + spin_unlock_bh(&qp->qp_lock); 2466 2398 2467 2399 return 0; 2468 2400 } ··· 2533 2449 qp->uacce_q = q; 2534 2450 qp->event_cb = qm_qp_event_notifier; 2535 2451 qp->pasid = arg; 2536 2536 - qp->is_in_kernel = false; 2537 2452 2538 2453 return 0; 2539 2454 } ··· 3002 2919 static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) 3003 2920 { 3004 2921 struct device *dev = &qm->pdev->dev; 3005 3005 - struct qm_dma *qdma; 2922 2922 + struct hisi_qp *qp; 3006 2923 int i; 3007 2924 3008 2925 for (i = num - 1; i >= 0; i--) { 3009 3009 - qdma = &qm->qp_array[i].qdma; 3010 3010 - dma_free_coherent(dev, qdma->size, qdma->va, qdma->dma); 2926 2926 + qp = &qm->qp_array[i]; 2927 2927 + dma_free_coherent(dev, qp->qdma.size, qp->qdma.va, qp->qdma.dma); 2928 2928 + kfree(qp->msg); 3011 2929 kfree(qm->poll_data[i].qp_finish_id); 3012 2930 } 3013 2931 ··· 3030 2946 return -ENOMEM; 3031 2947 3032 2948 qp = &qm->qp_array[id]; 2949 2949 + qp->msg = kmalloc_array(sq_depth, sizeof(void *), GFP_KERNEL); 2950 2950 + if (!qp->msg) 2951 2951 + goto err_free_qp_finish_id; 2952 2952 + 3033 2953 qp->qdma.va = dma_alloc_coherent(dev, dma_size, &qp->qdma.dma, 3034 2954 GFP_KERNEL); 3035 2955 if (!qp->qdma.va) 3036 3036 - goto err_free_qp_finish_id; 2956 2956 + goto err_free_qp_msg; 3037 2957 3038 2958 qp->sqe = qp->qdma.va; 3039 2959 qp->sqe_dma = qp->qdma.dma; ··· 3049 2961 qp->qm = qm; 3050 2962 qp->qp_id = id; 3051 2963 2964 2964 + spin_lock_init(&qp->qp_lock); 2965 2965 + spin_lock_init(&qp->backlog.lock); 2966 2966 + INIT_LIST_HEAD(&qp->backlog.list); 2967 2967 + 3052 2968 return 0; 3053 2969 2970 2970 + err_free_qp_msg: 2971 2971 + kfree(qp->msg); 3054 2972 err_free_qp_finish_id: 3055 2973 kfree(qm->poll_data[id].qp_finish_id); 3056 2974 return ret; ··· 3627 3533 } 3628 3534 EXPORT_SYMBOL_GPL(hisi_qm_dev_err_uninit); 3629 3535 3536 3536 + static void qm_release_qp_nolock(struct hisi_qp *qp) 3537 3537 + { 3538 3538 + struct hisi_qm *qm = qp->qm; 3539 3539 + 3540 3540 + if (--qp->ref_count) 3541 3541 + return; 3542 3542 + 3543 3543 + qm->qp_in_used--; 3544 3544 + idr_remove(&qm->qp_idr, qp->qp_id); 3545 3545 + } 3546 3546 + 3630 3547 /** 3631 3548 * hisi_qm_free_qps() - free multiple queue pairs. 3632 3549 * @qps: The queue pairs need to be freed. ··· 3650 3545 if (!qps || qp_num <= 0) 3651 3546 return; 3652 3547 3653 3653 - for (i = qp_num - 1; i >= 0; i--) 3654 3654 - hisi_qm_release_qp(qps[i]); 3548 3548 + down_write(&qps[0]->qm->qps_lock); 3549 3549 + 3550 3550 + for (i = qp_num - 1; i >= 0; i--) { 3551 3551 + if (qps[i]->ref_count == 1) 3552 3552 + qm_stop_qp_nolock(qps[i]); 3553 3553 + 3554 3554 + qm_release_qp_nolock(qps[i]); 3555 3555 + } 3556 3556 + 3557 3557 + up_write(&qps[0]->qm->qps_lock); 3558 3558 + qm_pm_put_sync(qps[0]->qm); 3655 3559 } 3656 3560 EXPORT_SYMBOL_GPL(hisi_qm_free_qps); 3561 3561 + 3562 3562 + static void qm_insert_sorted(struct list_head *head, struct hisi_qm_resource *res) 3563 3563 + { 3564 3564 + struct hisi_qm_resource *tmp; 3565 3565 + struct list_head *n = head; 3566 3566 + 3567 3567 + list_for_each_entry(tmp, head, list) { 3568 3568 + if (res->distance < tmp->distance) { 3569 3569 + n = &tmp->list; 3570 3570 + break; 3571 3571 + } 3572 3572 + } 3573 3573 + list_add_tail(&res->list, n); 3574 3574 + } 3657 3575 3658 3576 static void free_list(struct list_head *head) 3659 3577 { ··· 3688 3560 } 3689 3561 } 3690 3562 3563 3563 + static int qm_get_and_start_qp(struct hisi_qm *qm, int qp_num, struct hisi_qp **qps, u8 *alg_type) 3564 3564 + { 3565 3565 + int i, ret; 3566 3566 + 3567 3567 + ret = qm_pm_get_sync(qm); 3568 3568 + if (ret) 3569 3569 + return ret; 3570 3570 + 3571 3571 + down_write(&qm->qps_lock); 3572 3572 + for (i = 0; i < qp_num; i++) { 3573 3573 + qps[i] = qm_create_qp_nolock(qm, alg_type[i], true); 3574 3574 + if (IS_ERR(qps[i])) { 3575 3575 + ret = -ENODEV; 3576 3576 + goto stop_and_free; 3577 3577 + } 3578 3578 + 3579 3579 + if (qps[i]->ref_count != 1) 3580 3580 + continue; 3581 3581 + 3582 3582 + ret = qm_start_qp_nolock(qps[i], 0); 3583 3583 + if (ret) { 3584 3584 + qm_release_qp_nolock(qps[i]); 3585 3585 + goto stop_and_free; 3586 3586 + } 3587 3587 + } 3588 3588 + up_write(&qm->qps_lock); 3589 3589 + 3590 3590 + return 0; 3591 3591 + 3592 3592 + stop_and_free: 3593 3593 + for (i--; i >= 0; i--) { 3594 3594 + if (qps[i]->ref_count == 1) 3595 3595 + qm_stop_qp_nolock(qps[i]); 3596 3596 + 3597 3597 + qm_release_qp_nolock(qps[i]); 3598 3598 + } 3599 3599 + up_write(&qm->qps_lock); 3600 3600 + qm_pm_put_sync(qm); 3601 3601 + 3602 3602 + return ret; 3603 3603 + } 3604 3604 + 3691 3605 static int hisi_qm_sort_devices(int node, struct list_head *head, 3692 3606 struct hisi_qm_list *qm_list) 3693 3607 { 3694 3694 - struct hisi_qm_resource *res, *tmp; 3608 3608 + struct hisi_qm_resource *res; 3695 3609 struct hisi_qm *qm; 3696 3696 - struct list_head *n; 3697 3610 struct device *dev; 3698 3611 int dev_node; 3612 3612 + LIST_HEAD(non_full_list); 3613 3613 + LIST_HEAD(full_list); 3699 3614 3700 3615 list_for_each_entry(qm, &qm_list->list, list) { 3701 3616 dev = &qm->pdev->dev; ··· 3753 3582 3754 3583 res->qm = qm; 3755 3584 res->distance = node_distance(dev_node, node); 3756 3756 - n = head; 3757 3757 - list_for_each_entry(tmp, head, list) { 3758 3758 - if (res->distance < tmp->distance) { 3759 3759 - n = &tmp->list; 3760 3760 - break; 3761 3761 - } 3762 3762 - } 3763 3763 - list_add_tail(&res->list, n); 3585 3585 + 3586 3586 + if (qm->qp_in_used == qm->qp_num) 3587 3587 + qm_insert_sorted(&full_list, res); 3588 3588 + else 3589 3589 + qm_insert_sorted(&non_full_list, res); 3764 3590 } 3591 3591 + 3592 3592 + list_splice_tail(&non_full_list, head); 3593 3593 + list_splice_tail(&full_list, head); 3765 3594 3766 3595 return 0; 3767 3596 } ··· 3779 3608 * not meet the requirements will return error. 3780 3609 */ 3781 3610 int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, 3782 3782 - u8 alg_type, int node, struct hisi_qp **qps) 3611 3611 + u8 *alg_type, int node, struct hisi_qp **qps) 3783 3612 { 3784 3613 struct hisi_qm_resource *tmp; 3785 3614 int ret = -ENODEV; 3786 3615 LIST_HEAD(head); 3787 3787 - int i; 3788 3616 3789 3617 if (!qps || !qm_list || qp_num <= 0) 3790 3618 return -EINVAL; ··· 3795 3625 } 3796 3626 3797 3627 list_for_each_entry(tmp, &head, list) { 3798 3798 - for (i = 0; i < qp_num; i++) { 3799 3799 - qps[i] = hisi_qm_create_qp(tmp->qm, alg_type); 3800 3800 - if (IS_ERR(qps[i])) { 3801 3801 - hisi_qm_free_qps(qps, i); 3802 3802 - break; 3803 3803 - } 3804 3804 - } 3805 3805 - 3806 3806 - if (i == qp_num) { 3807 3807 - ret = 0; 3628 3628 + ret = qm_get_and_start_qp(tmp->qm, qp_num, qps, alg_type); 3629 3629 + if (!ret) 3808 3630 break; 3809 3809 - } 3810 3631 } 3811 3632 3812 3633 mutex_unlock(&qm_list->lock); 3813 3634 if (ret) 3814 3814 - pr_info("Failed to create qps, node[%d], alg[%u], qp[%d]!\n", 3815 3815 - node, alg_type, qp_num); 3635 3635 + pr_info("Failed to create qps, node[%d], qp[%d]!\n", 3636 3636 + node, qp_num); 3816 3637 3817 3638 err: 3818 3639 free_list(&head);

-7

drivers/crypto/hisilicon/sec2/sec.h

reviewed

··· 82 82 __u8 out_mac_buf[SEC_MAX_MAC_LEN]; 83 83 }; 84 84 85 85 - struct sec_instance_backlog { 86 86 - struct list_head list; 87 87 - spinlock_t lock; 88 88 - }; 89 89 - 90 85 /* SEC request of Crypto */ 91 86 struct sec_req { 92 87 union { ··· 107 112 bool use_pbuf; 108 113 109 114 struct list_head list; 110 110 - struct sec_instance_backlog *backlog; 111 115 struct sec_request_buf buf; 112 116 }; 113 117 ··· 166 172 spinlock_t id_lock; 167 173 struct hisi_acc_sgl_pool *c_in_pool; 168 174 struct hisi_acc_sgl_pool *c_out_pool; 169 169 - struct sec_instance_backlog backlog; 170 175 u16 send_head; 171 176 }; 172 177

+94 -69

drivers/crypto/hisilicon/sec2/sec_crypto.c

reviewed

··· 54 54 #define SEC_AUTH_CIPHER_V3 0x40 55 55 #define SEC_FLAG_OFFSET 7 56 56 #define SEC_FLAG_MASK 0x0780 57 57 - #define SEC_TYPE_MASK 0x0F 58 57 #define SEC_DONE_MASK 0x0001 59 58 #define SEC_ICV_MASK 0x000E 60 59 ··· 147 148 spin_unlock_bh(&qp_ctx->id_lock); 148 149 } 149 150 150 150 - static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) 151 151 + static void pre_parse_finished_bd(struct bd_status *status, void *resp) 151 152 { 152 153 struct sec_sqe *bd = resp; 153 154 ··· 157 158 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 158 159 status->tag = le16_to_cpu(bd->type2.tag); 159 160 status->err_type = bd->type2.error_type; 160 160 - 161 161 - return bd->type_cipher_auth & SEC_TYPE_MASK; 162 161 } 163 162 164 164 - static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) 163 163 + static void pre_parse_finished_bd3(struct bd_status *status, void *resp) 165 164 { 166 165 struct sec_sqe3 *bd3 = resp; 167 166 ··· 169 172 SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 170 173 status->tag = le64_to_cpu(bd3->tag); 171 174 status->err_type = bd3->error_type; 172 172 - 173 173 - return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK; 174 175 } 175 176 176 177 static int sec_cb_status_check(struct sec_req *req, ··· 239 244 struct sec_req *req, *tmp; 240 245 int ret; 241 246 242 242 - list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { 247 247 + list_for_each_entry_safe(req, tmp, &qp_ctx->qp->backlog.list, list) { 243 248 list_del(&req->list); 244 249 ctx->req_op->buf_unmap(ctx, req); 245 250 if (req->req_id >= 0) ··· 260 265 261 266 static void sec_alg_send_backlog(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) 262 267 { 268 268 + struct hisi_qp *qp = qp_ctx->qp; 263 269 struct sec_req *req, *tmp; 264 270 int ret; 265 271 266 266 - spin_lock_bh(&qp_ctx->backlog.lock); 267 267 - list_for_each_entry_safe(req, tmp, &qp_ctx->backlog.list, list) { 272 272 + spin_lock_bh(&qp->backlog.lock); 273 273 + list_for_each_entry_safe(req, tmp, &qp->backlog.list, list) { 268 274 ret = qp_send_message(req); 269 275 switch (ret) { 270 276 case -EINPROGRESS: ··· 283 287 } 284 288 285 289 unlock: 286 286 - spin_unlock_bh(&qp_ctx->backlog.lock); 290 290 + spin_unlock_bh(&qp->backlog.lock); 287 291 } 288 292 289 293 static void sec_req_cb(struct hisi_qp *qp, void *resp) 290 294 { 291 291 - struct sec_qp_ctx *qp_ctx = qp->qp_ctx; 292 292 - struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; 293 293 - u8 type_supported = qp_ctx->ctx->type_supported; 295 295 + const struct sec_sqe *sqe = qp->msg[qp->qp_status.cq_head]; 296 296 + struct sec_req *req = container_of(sqe, struct sec_req, sec_sqe); 297 297 + struct sec_ctx *ctx = req->ctx; 298 298 + struct sec_dfx *dfx = &ctx->sec->debug.dfx; 294 299 struct bd_status status; 295 295 - struct sec_ctx *ctx; 296 296 - struct sec_req *req; 297 300 int err; 298 298 - u8 type; 299 301 300 300 - if (type_supported == SEC_BD_TYPE2) { 301 301 - type = pre_parse_finished_bd(&status, resp); 302 302 - req = qp_ctx->req_list[status.tag]; 303 303 - } else { 304 304 - type = pre_parse_finished_bd3(&status, resp); 305 305 - req = (void *)(uintptr_t)status.tag; 306 306 - } 307 307 - 308 308 - if (unlikely(type != type_supported)) { 309 309 - atomic64_inc(&dfx->err_bd_cnt); 310 310 - pr_err("err bd type [%u]\n", type); 311 311 - return; 312 312 - } 313 313 - 314 314 - if (unlikely(!req)) { 315 315 - atomic64_inc(&dfx->invalid_req_cnt); 316 316 - atomic_inc(&qp->qp_status.used); 317 317 - return; 318 318 - } 302 302 + pre_parse_finished_bd(&status, resp); 319 303 320 304 req->err_type = status.err_type; 321 321 - ctx = req->ctx; 322 305 err = sec_cb_status_check(req, &status); 323 306 if (err) 324 307 atomic64_inc(&dfx->done_flag_cnt); ··· 305 330 atomic64_inc(&dfx->recv_cnt); 306 331 307 332 ctx->req_op->buf_unmap(ctx, req); 333 333 + ctx->req_op->callback(ctx, req, err); 334 334 + } 308 335 336 336 + static void sec_req_cb3(struct hisi_qp *qp, void *resp) 337 337 + { 338 338 + struct bd_status status; 339 339 + struct sec_ctx *ctx; 340 340 + struct sec_dfx *dfx; 341 341 + struct sec_req *req; 342 342 + int err; 343 343 + 344 344 + pre_parse_finished_bd3(&status, resp); 345 345 + 346 346 + req = (void *)(uintptr_t)status.tag; 347 347 + req->err_type = status.err_type; 348 348 + ctx = req->ctx; 349 349 + dfx = &ctx->sec->debug.dfx; 350 350 + 351 351 + err = sec_cb_status_check(req, &status); 352 352 + if (err) 353 353 + atomic64_inc(&dfx->done_flag_cnt); 354 354 + 355 355 + atomic64_inc(&dfx->recv_cnt); 356 356 + 357 357 + ctx->req_op->buf_unmap(ctx, req); 309 358 ctx->req_op->callback(ctx, req, err); 310 359 } 311 360 ··· 347 348 348 349 static int sec_alg_try_enqueue(struct sec_req *req) 349 350 { 351 351 + struct hisi_qp *qp = req->qp_ctx->qp; 352 352 + 350 353 /* Check if any request is already backlogged */ 351 351 - if (!list_empty(&req->backlog->list)) 354 354 + if (!list_empty(&qp->backlog.list)) 352 355 return -EBUSY; 353 356 354 357 /* Try to enqueue to HW ring */ ··· 360 359 361 360 static int sec_alg_send_message_maybacklog(struct sec_req *req) 362 361 { 362 362 + struct hisi_qp *qp = req->qp_ctx->qp; 363 363 int ret; 364 364 365 365 ret = sec_alg_try_enqueue(req); 366 366 if (ret != -EBUSY) 367 367 return ret; 368 368 369 369 - spin_lock_bh(&req->backlog->lock); 369 369 + spin_lock_bh(&qp->backlog.lock); 370 370 ret = sec_alg_try_enqueue(req); 371 371 if (ret == -EBUSY) 372 372 - list_add_tail(&req->list, &req->backlog->list); 373 373 - spin_unlock_bh(&req->backlog->lock); 372 372 + list_add_tail(&req->list, &qp->backlog.list); 373 373 + spin_unlock_bh(&qp->backlog.lock); 374 374 375 375 return ret; 376 376 } ··· 626 624 627 625 qp_ctx = &ctx->qp_ctx[qp_ctx_id]; 628 626 qp = ctx->qps[qp_ctx_id]; 629 629 - qp->req_type = 0; 630 630 - qp->qp_ctx = qp_ctx; 631 627 qp_ctx->qp = qp; 632 628 qp_ctx->ctx = ctx; 633 629 634 634 - qp->req_cb = sec_req_cb; 630 630 + if (ctx->type_supported == SEC_BD_TYPE3) 631 631 + qp->req_cb = sec_req_cb3; 632 632 + else 633 633 + qp->req_cb = sec_req_cb; 635 634 636 635 spin_lock_init(&qp_ctx->req_lock); 637 636 idr_init(&qp_ctx->req_idr); 638 638 - spin_lock_init(&qp_ctx->backlog.lock); 639 637 spin_lock_init(&qp_ctx->id_lock); 640 640 - INIT_LIST_HEAD(&qp_ctx->backlog.list); 641 638 qp_ctx->send_head = 0; 642 639 643 640 ret = sec_alloc_qp_ctx_resource(ctx, qp_ctx); 644 641 if (ret) 645 642 goto err_destroy_idr; 646 643 647 647 - ret = hisi_qm_start_qp(qp, 0); 648 648 - if (ret < 0) 649 649 - goto err_resource_free; 650 650 - 651 644 return 0; 652 645 653 653 - err_resource_free: 654 654 - sec_free_qp_ctx_resource(ctx, qp_ctx); 655 646 err_destroy_idr: 656 647 idr_destroy(&qp_ctx->req_idr); 657 648 return ret; ··· 653 658 static void sec_release_qp_ctx(struct sec_ctx *ctx, 654 659 struct sec_qp_ctx *qp_ctx) 655 660 { 656 656 - hisi_qm_stop_qp(qp_ctx->qp); 657 661 sec_free_qp_ctx_resource(ctx, qp_ctx); 658 662 idr_destroy(&qp_ctx->req_idr); 659 663 } ··· 663 669 int i, ret; 664 670 665 671 ctx->qps = sec_create_qps(); 666 666 - if (!ctx->qps) { 667 667 - pr_err("Can not create sec qps!\n"); 672 672 + if (!ctx->qps) 668 673 return -ENODEV; 669 669 - } 670 674 671 675 sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); 672 676 ctx->sec = sec; ··· 700 708 { 701 709 int i; 702 710 711 711 + if (!ctx->qps) 712 712 + return; 713 713 + 703 714 for (i = 0; i < ctx->sec->ctx_q_num; i++) 704 715 sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 705 716 ··· 713 718 static int sec_cipher_init(struct sec_ctx *ctx) 714 719 { 715 720 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 721 721 + 722 722 + if (!ctx->qps) 723 723 + return 0; 716 724 717 725 c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 718 726 &c_ctx->c_key_dma, GFP_KERNEL); ··· 728 730 static void sec_cipher_uninit(struct sec_ctx *ctx) 729 731 { 730 732 struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 733 733 + 734 734 + if (!ctx->qps) 735 735 + return; 731 736 732 737 memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); 733 738 dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, ··· 752 751 static void sec_auth_uninit(struct sec_ctx *ctx) 753 752 { 754 753 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 754 754 + 755 755 + if (!ctx->qps) 756 756 + return; 755 757 756 758 memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE); 757 759 dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, ··· 793 789 } 794 790 795 791 ret = sec_ctx_base_init(ctx); 796 796 - if (ret) 792 792 + if (ret && ret != -ENODEV) 797 793 return ret; 798 794 799 795 ret = sec_cipher_init(ctx); ··· 902 898 struct device *dev = ctx->dev; 903 899 int ret; 904 900 901 901 + if (!ctx->qps) 902 902 + goto set_soft_key; 903 903 + 905 904 if (c_mode == SEC_CMODE_XTS) { 906 905 ret = xts_verify_key(tfm, key, keylen); 907 906 if (ret) { ··· 935 928 } 936 929 937 930 memcpy(c_ctx->c_key, key, keylen); 938 938 - if (c_ctx->fbtfm) { 939 939 - ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); 940 940 - if (ret) { 941 941 - dev_err(dev, "failed to set fallback skcipher key!\n"); 942 942 - return ret; 943 943 - } 931 931 + 932 932 + set_soft_key: 933 933 + ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); 934 934 + if (ret) { 935 935 + dev_err(dev, "failed to set fallback skcipher key!\n"); 936 936 + return ret; 944 937 } 938 938 + 945 939 return 0; 946 940 } 947 941 ··· 1405 1397 struct device *dev = ctx->dev; 1406 1398 struct crypto_authenc_keys keys; 1407 1399 int ret; 1400 1400 + 1401 1401 + if (!ctx->qps) 1402 1402 + return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); 1408 1403 1409 1404 ctx->a_ctx.a_alg = a_alg; 1410 1405 ctx->c_ctx.c_alg = c_alg; ··· 1963 1952 } while (req->req_id < 0 && ++i < ctx->sec->ctx_q_num); 1964 1953 1965 1954 req->qp_ctx = qp_ctx; 1966 1966 - req->backlog = &qp_ctx->backlog; 1967 1955 1968 1956 return 0; 1969 1957 } ··· 2065 2055 if (ret) 2066 2056 return ret; 2067 2057 2058 2058 + if (!ctx->qps) 2059 2059 + return 0; 2060 2060 + 2068 2061 if (ctx->sec->qm.ver < QM_HW_V3) { 2069 2062 ctx->type_supported = SEC_BD_TYPE2; 2070 2063 ctx->req_op = &sec_skcipher_req_ops; ··· 2076 2063 ctx->req_op = &sec_skcipher_req_ops_v3; 2077 2064 } 2078 2065 2079 2079 - return ret; 2066 2066 + return 0; 2080 2067 } 2081 2068 2082 2069 static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) ··· 2144 2131 int ret; 2145 2132 2146 2133 ret = sec_aead_init(tfm); 2147 2147 - if (ret) { 2134 2134 + if (ret && ret != -ENODEV) { 2148 2135 pr_err("hisi_sec2: aead init error!\n"); 2149 2136 return ret; 2150 2137 } ··· 2186 2173 int ret; 2187 2174 2188 2175 ret = sec_aead_init(tfm); 2189 2189 - if (ret) { 2176 2176 + if (ret && ret != -ENODEV) { 2190 2177 dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); 2191 2178 return ret; 2192 2179 } ··· 2331 2318 bool need_fallback = false; 2332 2319 int ret; 2333 2320 2321 2321 + if (!ctx->qps) 2322 2322 + goto soft_crypto; 2323 2323 + 2334 2324 if (!sk_req->cryptlen) { 2335 2325 if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) 2336 2326 return -EINVAL; ··· 2351 2335 return -EINVAL; 2352 2336 2353 2337 if (unlikely(ctx->c_ctx.fallback || need_fallback)) 2354 2354 - return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); 2338 2338 + goto soft_crypto; 2355 2339 2356 2340 return ctx->req_op->process(ctx, req); 2341 2341 + 2342 2342 + soft_crypto: 2343 2343 + return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); 2357 2344 } 2358 2345 2359 2346 static int sec_skcipher_encrypt(struct skcipher_request *sk_req) ··· 2564 2545 bool need_fallback = false; 2565 2546 int ret; 2566 2547 2548 2548 + if (!ctx->qps) 2549 2549 + goto soft_crypto; 2550 2550 + 2567 2551 req->flag = a_req->base.flags; 2568 2552 req->aead_req.aead_req = a_req; 2569 2553 req->c_req.encrypt = encrypt; ··· 2577 2555 ret = sec_aead_param_check(ctx, req, &need_fallback); 2578 2556 if (unlikely(ret)) { 2579 2557 if (need_fallback) 2580 2580 - return sec_aead_soft_crypto(ctx, a_req, encrypt); 2558 2558 + goto soft_crypto; 2581 2559 return -EINVAL; 2582 2560 } 2583 2561 2584 2562 return ctx->req_op->process(ctx, req); 2563 2563 + 2564 2564 + soft_crypto: 2565 2565 + return sec_aead_soft_crypto(ctx, a_req, encrypt); 2585 2566 } 2586 2567 2587 2568 static int sec_aead_encrypt(struct aead_request *a_req)

+16 -5

drivers/crypto/hisilicon/sec2/sec_main.c

reviewed

··· 417 417 int node = cpu_to_node(raw_smp_processor_id()); 418 418 u32 ctx_num = ctx_q_num; 419 419 struct hisi_qp **qps; 420 420 + u8 *type; 420 421 int ret; 421 422 422 423 qps = kcalloc(ctx_num, sizeof(struct hisi_qp *), GFP_KERNEL); 423 424 if (!qps) 424 425 return NULL; 425 426 426 426 - ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, 0, node, qps); 427 427 - if (!ret) 428 428 - return qps; 427 427 + /* The type of SEC is all 0, so just allocated by kcalloc */ 428 428 + type = kcalloc(ctx_num, sizeof(u8), GFP_KERNEL); 429 429 + if (!type) { 430 430 + kfree(qps); 431 431 + return NULL; 432 432 + } 429 433 430 430 - kfree(qps); 431 431 - return NULL; 434 434 + ret = hisi_qm_alloc_qps_node(&sec_devices, ctx_num, type, node, qps); 435 435 + if (ret) { 436 436 + kfree(type); 437 437 + kfree(qps); 438 438 + return NULL; 439 439 + } 440 440 + 441 441 + kfree(type); 442 442 + return qps; 432 443 } 433 444 434 445 u64 sec_get_alg_bitmap(struct hisi_qm *qm, u32 high, u32 low)

+1 -1

drivers/crypto/hisilicon/sgl.c

reviewed

··· 260 260 return curr_hw_sgl; 261 261 262 262 err_unmap: 263 263 - dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); 263 263 + dma_unmap_sg(dev, sgl, sg_n, dir); 264 264 265 265 return ERR_PTR(ret); 266 266 }

+87 -36

drivers/crypto/hisilicon/trng/trng.c

reviewed

··· 40 40 #define SEED_SHIFT_24 24 41 41 #define SEED_SHIFT_16 16 42 42 #define SEED_SHIFT_8 8 43 43 + #define SW_MAX_RANDOM_BYTES 65520 43 44 44 45 struct hisi_trng_list { 45 46 struct mutex lock; ··· 54 53 struct list_head list; 55 54 struct hwrng rng; 56 55 u32 ver; 57 57 - bool is_used; 58 58 - struct mutex mutex; 56 56 + u32 ctx_num; 57 57 + /* The bytes of the random number generated since the last seeding. */ 58 58 + u32 random_bytes; 59 59 + struct mutex lock; 59 60 }; 60 61 61 62 struct hisi_trng_ctx { ··· 66 63 67 64 static atomic_t trng_active_devs; 68 65 static struct hisi_trng_list trng_devices; 66 66 + static int hisi_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait); 69 67 70 70 - static void hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) 68 68 + static int hisi_trng_set_seed(struct hisi_trng *trng, const u8 *seed) 71 69 { 72 70 u32 val, seed_reg, i; 71 71 + int ret; 72 72 + 73 73 + writel(0x0, trng->base + SW_DRBG_BLOCKS); 73 74 74 75 for (i = 0; i < SW_DRBG_SEED_SIZE; 75 76 i += SW_DRBG_SEED_SIZE / SW_DRBG_SEED_REGS_NUM) { ··· 85 78 seed_reg = (i >> SW_DRBG_NUM_SHIFT) % SW_DRBG_SEED_REGS_NUM; 86 79 writel(val, trng->base + SW_DRBG_SEED(seed_reg)); 87 80 } 81 81 + 82 82 + writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), 83 83 + trng->base + SW_DRBG_BLOCKS); 84 84 + writel(0x1, trng->base + SW_DRBG_INIT); 85 85 + ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, 86 86 + val, val & BIT(0), SLEEP_US, TIMEOUT_US); 87 87 + if (ret) { 88 88 + pr_err("failed to init trng(%d)\n", ret); 89 89 + return -EIO; 90 90 + } 91 91 + 92 92 + trng->random_bytes = 0; 93 93 + 94 94 + return 0; 88 95 } 89 96 90 97 static int hisi_trng_seed(struct crypto_rng *tfm, const u8 *seed, ··· 106 85 { 107 86 struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); 108 87 struct hisi_trng *trng = ctx->trng; 109 109 - u32 val = 0; 110 110 - int ret = 0; 88 88 + int ret; 111 89 112 90 if (slen < SW_DRBG_SEED_SIZE) { 113 91 pr_err("slen(%u) is not matched with trng(%d)\n", slen, ··· 114 94 return -EINVAL; 115 95 } 116 96 117 117 - writel(0x0, trng->base + SW_DRBG_BLOCKS); 118 118 - hisi_trng_set_seed(trng, seed); 119 119 - 120 120 - writel(SW_DRBG_BLOCKS_NUM | (0x1 << SW_DRBG_ENABLE_SHIFT), 121 121 - trng->base + SW_DRBG_BLOCKS); 122 122 - writel(0x1, trng->base + SW_DRBG_INIT); 123 123 - 124 124 - ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, 125 125 - val, val & BIT(0), SLEEP_US, TIMEOUT_US); 126 126 - if (ret) 127 127 - pr_err("fail to init trng(%d)\n", ret); 97 97 + mutex_lock(&trng->lock); 98 98 + ret = hisi_trng_set_seed(trng, seed); 99 99 + mutex_unlock(&trng->lock); 128 100 129 101 return ret; 130 102 } 131 103 132 132 - static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, 133 133 - unsigned int slen, u8 *dstn, unsigned int dlen) 104 104 + static int hisi_trng_reseed(struct hisi_trng *trng) 134 105 { 135 135 - struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); 136 136 - struct hisi_trng *trng = ctx->trng; 106 106 + u8 seed[SW_DRBG_SEED_SIZE]; 107 107 + int size; 108 108 + 109 109 + if (!trng->random_bytes) 110 110 + return 0; 111 111 + 112 112 + size = hisi_trng_read(&trng->rng, seed, SW_DRBG_SEED_SIZE, false); 113 113 + if (size != SW_DRBG_SEED_SIZE) 114 114 + return -EIO; 115 115 + 116 116 + return hisi_trng_set_seed(trng, seed); 117 117 + } 118 118 + 119 119 + static int hisi_trng_get_bytes(struct hisi_trng *trng, u8 *dstn, unsigned int dlen) 120 120 + { 137 121 u32 data[SW_DRBG_DATA_NUM]; 138 122 u32 currsize = 0; 139 123 u32 val = 0; 140 124 int ret; 141 125 u32 i; 142 126 143 143 - if (dlen > SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES || dlen == 0) { 144 144 - pr_err("dlen(%u) exceeds limit(%d)!\n", dlen, 145 145 - SW_DRBG_BLOCKS_NUM * SW_DRBG_BYTES); 146 146 - return -EINVAL; 147 147 - } 127 127 + ret = hisi_trng_reseed(trng); 128 128 + if (ret) 129 129 + return ret; 148 130 149 131 do { 150 132 ret = readl_relaxed_poll_timeout(trng->base + SW_DRBG_STATUS, 151 151 - val, val & BIT(1), SLEEP_US, TIMEOUT_US); 133 133 + val, val & BIT(1), SLEEP_US, TIMEOUT_US); 152 134 if (ret) { 153 153 - pr_err("fail to generate random number(%d)!\n", ret); 135 135 + pr_err("failed to generate random number(%d)!\n", ret); 154 136 break; 155 137 } 156 138 ··· 167 145 currsize = dlen; 168 146 } 169 147 148 148 + trng->random_bytes += SW_DRBG_BYTES; 170 149 writel(0x1, trng->base + SW_DRBG_GEN); 171 150 } while (currsize < dlen); 172 151 173 152 return ret; 174 153 } 175 154 155 155 + static int hisi_trng_generate(struct crypto_rng *tfm, const u8 *src, 156 156 + unsigned int slen, u8 *dstn, unsigned int dlen) 157 157 + { 158 158 + struct hisi_trng_ctx *ctx = crypto_rng_ctx(tfm); 159 159 + struct hisi_trng *trng = ctx->trng; 160 160 + unsigned int currsize = 0; 161 161 + unsigned int block_size; 162 162 + int ret; 163 163 + 164 164 + if (!dstn || !dlen) { 165 165 + pr_err("output is error, dlen %u!\n", dlen); 166 166 + return -EINVAL; 167 167 + } 168 168 + 169 169 + do { 170 170 + block_size = min_t(unsigned int, dlen - currsize, SW_MAX_RANDOM_BYTES); 171 171 + mutex_lock(&trng->lock); 172 172 + ret = hisi_trng_get_bytes(trng, dstn + currsize, block_size); 173 173 + mutex_unlock(&trng->lock); 174 174 + if (ret) 175 175 + return ret; 176 176 + currsize += block_size; 177 177 + } while (currsize < dlen); 178 178 + 179 179 + return 0; 180 180 + } 181 181 + 176 182 static int hisi_trng_init(struct crypto_tfm *tfm) 177 183 { 178 184 struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); 179 185 struct hisi_trng *trng; 180 180 - int ret = -EBUSY; 186 186 + u32 ctx_num = ~0; 181 187 182 188 mutex_lock(&trng_devices.lock); 183 189 list_for_each_entry(trng, &trng_devices.list, list) { 184 184 - if (!trng->is_used) { 185 185 - trng->is_used = true; 190 190 + if (trng->ctx_num < ctx_num) { 191 191 + ctx_num = trng->ctx_num; 186 192 ctx->trng = trng; 187 187 - ret = 0; 188 188 - break; 189 193 } 190 194 } 195 195 + ctx->trng->ctx_num++; 191 196 mutex_unlock(&trng_devices.lock); 192 197 193 193 - return ret; 198 198 + return 0; 194 199 } 195 200 196 201 static void hisi_trng_exit(struct crypto_tfm *tfm) ··· 225 176 struct hisi_trng_ctx *ctx = crypto_tfm_ctx(tfm); 226 177 227 178 mutex_lock(&trng_devices.lock); 228 228 - ctx->trng->is_used = false; 179 179 + ctx->trng->ctx_num--; 229 180 mutex_unlock(&trng_devices.lock); 230 181 } 231 182 ··· 287 238 int ret = -EBUSY; 288 239 289 240 mutex_lock(&trng_devices.lock); 290 290 - if (!trng->is_used) { 241 241 + if (!trng->ctx_num) { 291 242 list_del(&trng->list); 292 243 ret = 0; 293 244 } ··· 311 262 if (IS_ERR(trng->base)) 312 263 return PTR_ERR(trng->base); 313 264 314 314 - trng->is_used = false; 265 265 + trng->ctx_num = 0; 266 266 + trng->random_bytes = SW_MAX_RANDOM_BYTES; 267 267 + mutex_init(&trng->lock); 315 268 trng->ver = readl(trng->base + HISI_TRNG_VERSION); 316 269 if (!trng_devices.is_init) { 317 270 INIT_LIST_HEAD(&trng_devices.list);

+1 -1

drivers/crypto/hisilicon/zip/zip.h

reviewed

··· 99 99 ZIP_CORE5_BITMAP, 100 100 }; 101 101 102 102 - int zip_create_qps(struct hisi_qp **qps, int qp_num, int node); 102 102 + int zip_create_qps(struct hisi_qp **qps, int qp_num, int node, u8 *alg_type); 103 103 int hisi_zip_register_to_crypto(struct hisi_qm *qm); 104 104 void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); 105 105 bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg);

+138 -64

drivers/crypto/hisilicon/zip/zip_crypto.c

reviewed

··· 17 17 /* hisi_zip_sqe dw9 */ 18 18 #define HZIP_REQ_TYPE_M GENMASK(7, 0) 19 19 #define HZIP_ALG_TYPE_DEFLATE 0x01 20 20 + #define HZIP_ALG_TYPE_LZ4 0x04 20 21 #define HZIP_BUF_TYPE_M GENMASK(11, 8) 21 22 #define HZIP_SGL 0x1 23 23 + #define HZIP_WIN_SIZE_M GENMASK(15, 12) 24 24 + #define HZIP_16K_WINSZ 0x2 22 25 23 26 #define HZIP_ALG_PRIORITY 300 24 27 #define HZIP_SGL_SGE_NR 10 25 28 26 29 #define HZIP_ALG_DEFLATE GENMASK(5, 4) 30 30 + #define HZIP_ALG_LZ4 BIT(8) 27 31 28 32 static DEFINE_MUTEX(zip_algs_lock); 29 33 static unsigned int zip_available_devs; ··· 43 39 HZIP_CTX_Q_NUM 44 40 }; 45 41 42 42 + #define GET_REQ_FROM_SQE(sqe) ((u64)(sqe)->dw26 | (u64)(sqe)->dw27 << 32) 46 43 #define COMP_NAME_TO_TYPE(alg_name) \ 47 47 - (!strcmp((alg_name), "deflate") ? HZIP_ALG_TYPE_DEFLATE : 0) 44 44 + (!strcmp((alg_name), "deflate") ? HZIP_ALG_TYPE_DEFLATE : \ 45 45 + (!strcmp((alg_name), "lz4") ? HZIP_ALG_TYPE_LZ4 : 0)) 48 46 49 47 struct hisi_zip_req { 50 48 struct acomp_req *req; ··· 54 48 struct hisi_acc_hw_sgl *hw_dst; 55 49 dma_addr_t dma_src; 56 50 dma_addr_t dma_dst; 51 51 + struct hisi_zip_qp_ctx *qp_ctx; 57 52 u16 req_id; 58 53 }; 59 54 ··· 71 64 struct hisi_acc_sgl_pool *sgl_pool; 72 65 struct hisi_zip *zip_dev; 73 66 struct hisi_zip_ctx *ctx; 67 67 + u8 req_type; 74 68 }; 75 69 76 70 struct hisi_zip_sqe_ops { ··· 80 72 void (*fill_buf_size)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 81 73 void (*fill_buf_type)(struct hisi_zip_sqe *sqe, u8 buf_type); 82 74 void (*fill_req_type)(struct hisi_zip_sqe *sqe, u8 req_type); 75 75 + void (*fill_win_size)(struct hisi_zip_sqe *sqe, u8 win_size); 83 76 void (*fill_tag)(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req); 84 77 void (*fill_sqe_type)(struct hisi_zip_sqe *sqe, u8 sqe_type); 85 85 - u32 (*get_tag)(struct hisi_zip_sqe *sqe); 86 78 u32 (*get_status)(struct hisi_zip_sqe *sqe); 87 79 u32 (*get_dstlen)(struct hisi_zip_sqe *sqe); 88 80 }; ··· 90 82 struct hisi_zip_ctx { 91 83 struct hisi_zip_qp_ctx qp_ctx[HZIP_CTX_Q_NUM]; 92 84 const struct hisi_zip_sqe_ops *ops; 85 85 + bool fallback; 93 86 }; 94 87 95 88 static int sgl_sge_nr_set(const char *val, const struct kernel_param *kp) ··· 117 108 module_param_cb(sgl_sge_nr, &sgl_sge_nr_ops, &sgl_sge_nr, 0444); 118 109 MODULE_PARM_DESC(sgl_sge_nr, "Number of sge in sgl(1-255)"); 119 110 111 111 + static int hisi_zip_fallback_do_work(struct acomp_req *acomp_req, bool is_decompress) 112 112 + { 113 113 + ACOMP_FBREQ_ON_STACK(fbreq, acomp_req); 114 114 + int ret; 115 115 + 116 116 + if (!is_decompress) 117 117 + ret = crypto_acomp_compress(fbreq); 118 118 + else 119 119 + ret = crypto_acomp_decompress(fbreq); 120 120 + if (ret) { 121 121 + pr_err("failed to do fallback work, ret=%d\n", ret); 122 122 + return ret; 123 123 + } 124 124 + 125 125 + acomp_req->dlen = fbreq->dlen; 126 126 + return ret; 127 127 + } 128 128 + 120 129 static struct hisi_zip_req *hisi_zip_create_req(struct hisi_zip_qp_ctx *qp_ctx, 121 130 struct acomp_req *req) 122 131 { ··· 158 131 req_cache = q + req_id; 159 132 req_cache->req_id = req_id; 160 133 req_cache->req = req; 134 134 + req_cache->qp_ctx = qp_ctx; 161 135 162 136 return req_cache; 163 137 } ··· 207 179 sqe->dw9 = val; 208 180 } 209 181 182 182 + static void hisi_zip_fill_win_size(struct hisi_zip_sqe *sqe, u8 win_size) 183 183 + { 184 184 + u32 val; 185 185 + 186 186 + val = sqe->dw9 & ~HZIP_WIN_SIZE_M; 187 187 + val |= FIELD_PREP(HZIP_WIN_SIZE_M, win_size); 188 188 + sqe->dw9 = val; 189 189 + } 190 190 + 210 191 static void hisi_zip_fill_tag(struct hisi_zip_sqe *sqe, struct hisi_zip_req *req) 211 192 { 212 212 - sqe->dw26 = req->req_id; 193 193 + sqe->dw26 = lower_32_bits((u64)req); 194 194 + sqe->dw27 = upper_32_bits((u64)req); 213 195 } 214 196 215 197 static void hisi_zip_fill_sqe_type(struct hisi_zip_sqe *sqe, u8 sqe_type) ··· 242 204 ops->fill_buf_size(sqe, req); 243 205 ops->fill_buf_type(sqe, HZIP_SGL); 244 206 ops->fill_req_type(sqe, req_type); 207 207 + ops->fill_win_size(sqe, HZIP_16K_WINSZ); 245 208 ops->fill_tag(sqe, req); 246 209 ops->fill_sqe_type(sqe, ops->sqe_type); 247 210 } ··· 252 213 { 253 214 struct hisi_acc_sgl_pool *pool = qp_ctx->sgl_pool; 254 215 struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; 255 255 - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 256 216 struct acomp_req *a_req = req->req; 257 217 struct hisi_qp *qp = qp_ctx->qp; 258 218 struct device *dev = &qp->qm->pdev->dev; ··· 275 237 &req->dma_dst, DMA_FROM_DEVICE); 276 238 if (IS_ERR(req->hw_dst)) { 277 239 ret = PTR_ERR(req->hw_dst); 278 278 - dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", 240 240 + dev_err(dev, "failed to map the dst buffer to hw sgl (%d)!\n", 279 241 ret); 280 242 goto err_unmap_input; 281 243 } 282 244 283 283 - hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp->req_type, req); 245 245 + hisi_zip_fill_sqe(qp_ctx->ctx, &zip_sqe, qp_ctx->req_type, req); 284 246 285 247 /* send command to start a task */ 286 248 atomic64_inc(&dfx->send_cnt); 287 287 - spin_lock_bh(&req_q->req_lock); 288 249 ret = hisi_qp_send(qp, &zip_sqe); 289 289 - spin_unlock_bh(&req_q->req_lock); 290 250 if (unlikely(ret < 0)) { 291 251 atomic64_inc(&dfx->send_busy_cnt); 292 252 ret = -EAGAIN; ··· 301 265 return ret; 302 266 } 303 267 304 304 - static u32 hisi_zip_get_tag(struct hisi_zip_sqe *sqe) 305 305 - { 306 306 - return sqe->dw26; 307 307 - } 308 308 - 309 268 static u32 hisi_zip_get_status(struct hisi_zip_sqe *sqe) 310 269 { 311 270 return sqe->dw3 & HZIP_BD_STATUS_M; ··· 313 282 314 283 static void hisi_zip_acomp_cb(struct hisi_qp *qp, void *data) 315 284 { 316 316 - struct hisi_zip_qp_ctx *qp_ctx = qp->qp_ctx; 285 285 + struct hisi_zip_sqe *sqe = data; 286 286 + struct hisi_zip_req *req = (struct hisi_zip_req *)GET_REQ_FROM_SQE(sqe); 287 287 + struct hisi_zip_qp_ctx *qp_ctx = req->qp_ctx; 317 288 const struct hisi_zip_sqe_ops *ops = qp_ctx->ctx->ops; 318 289 struct hisi_zip_dfx *dfx = &qp_ctx->zip_dev->dfx; 319 319 - struct hisi_zip_req_q *req_q = &qp_ctx->req_q; 320 290 struct device *dev = &qp->qm->pdev->dev; 321 321 - struct hisi_zip_sqe *sqe = data; 322 322 - u32 tag = ops->get_tag(sqe); 323 323 - struct hisi_zip_req *req = req_q->q + tag; 324 291 struct acomp_req *acomp_req = req->req; 325 292 int err = 0; 326 293 u32 status; ··· 348 319 { 349 320 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 350 321 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_COMP]; 351 351 - struct device *dev = &qp_ctx->qp->qm->pdev->dev; 352 322 struct hisi_zip_req *req; 323 323 + struct device *dev; 353 324 int ret; 325 325 + 326 326 + if (ctx->fallback) 327 327 + return hisi_zip_fallback_do_work(acomp_req, 0); 328 328 + 329 329 + dev = &qp_ctx->qp->qm->pdev->dev; 354 330 355 331 req = hisi_zip_create_req(qp_ctx, acomp_req); 356 332 if (IS_ERR(req)) ··· 374 340 { 375 341 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(acomp_req->base.tfm); 376 342 struct hisi_zip_qp_ctx *qp_ctx = &ctx->qp_ctx[HZIP_QPC_DECOMP]; 377 377 - struct device *dev = &qp_ctx->qp->qm->pdev->dev; 378 343 struct hisi_zip_req *req; 344 344 + struct device *dev; 379 345 int ret; 346 346 + 347 347 + if (ctx->fallback) 348 348 + return hisi_zip_fallback_do_work(acomp_req, 1); 349 349 + 350 350 + dev = &qp_ctx->qp->qm->pdev->dev; 380 351 381 352 req = hisi_zip_create_req(qp_ctx, acomp_req); 382 353 if (IS_ERR(req)) ··· 397 358 return ret; 398 359 } 399 360 400 400 - static int hisi_zip_start_qp(struct hisi_qp *qp, struct hisi_zip_qp_ctx *qp_ctx, 401 401 - int alg_type, int req_type) 361 361 + static int hisi_zip_decompress(struct acomp_req *acomp_req) 402 362 { 403 403 - struct device *dev = &qp->qm->pdev->dev; 404 404 - int ret; 405 405 - 406 406 - qp->req_type = req_type; 407 407 - qp->alg_type = alg_type; 408 408 - qp->qp_ctx = qp_ctx; 409 409 - 410 410 - ret = hisi_qm_start_qp(qp, 0); 411 411 - if (ret < 0) { 412 412 - dev_err(dev, "failed to start qp (%d)!\n", ret); 413 413 - return ret; 414 414 - } 415 415 - 416 416 - qp_ctx->qp = qp; 417 417 - 418 418 - return 0; 419 419 - } 420 420 - 421 421 - static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *qp_ctx) 422 422 - { 423 423 - hisi_qm_stop_qp(qp_ctx->qp); 424 424 - hisi_qm_free_qps(&qp_ctx->qp, 1); 363 363 + return hisi_zip_fallback_do_work(acomp_req, 1); 425 364 } 426 365 427 366 static const struct hisi_zip_sqe_ops hisi_zip_ops = { ··· 408 391 .fill_buf_size = hisi_zip_fill_buf_size, 409 392 .fill_buf_type = hisi_zip_fill_buf_type, 410 393 .fill_req_type = hisi_zip_fill_req_type, 394 394 + .fill_win_size = hisi_zip_fill_win_size, 411 395 .fill_tag = hisi_zip_fill_tag, 412 396 .fill_sqe_type = hisi_zip_fill_sqe_type, 413 413 - .get_tag = hisi_zip_get_tag, 414 397 .get_status = hisi_zip_get_status, 415 398 .get_dstlen = hisi_zip_get_dstlen, 416 399 }; ··· 419 402 { 420 403 struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; 421 404 struct hisi_zip_qp_ctx *qp_ctx; 405 405 + u8 alg_type[HZIP_CTX_Q_NUM]; 422 406 struct hisi_zip *hisi_zip; 423 423 - int ret, i, j; 407 407 + int ret, i; 424 408 425 425 - ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node); 409 409 + /* alg_type = 0 for compress, 1 for decompress in hw sqe */ 410 410 + for (i = 0; i < HZIP_CTX_Q_NUM; i++) 411 411 + alg_type[i] = i; 412 412 + 413 413 + ret = zip_create_qps(qps, HZIP_CTX_Q_NUM, node, alg_type); 426 414 if (ret) { 427 415 pr_err("failed to create zip qps (%d)!\n", ret); 428 416 return -ENODEV; ··· 436 414 hisi_zip = container_of(qps[0]->qm, struct hisi_zip, qm); 437 415 438 416 for (i = 0; i < HZIP_CTX_Q_NUM; i++) { 439 439 - /* alg_type = 0 for compress, 1 for decompress in hw sqe */ 440 417 qp_ctx = &hisi_zip_ctx->qp_ctx[i]; 441 418 qp_ctx->ctx = hisi_zip_ctx; 442 442 - ret = hisi_zip_start_qp(qps[i], qp_ctx, i, req_type); 443 443 - if (ret) { 444 444 - for (j = i - 1; j >= 0; j--) 445 445 - hisi_qm_stop_qp(hisi_zip_ctx->qp_ctx[j].qp); 446 446 - 447 447 - hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); 448 448 - return ret; 449 449 - } 450 450 - 451 419 qp_ctx->zip_dev = hisi_zip; 420 420 + qp_ctx->req_type = req_type; 421 421 + qp_ctx->qp = qps[i]; 452 422 } 453 423 454 424 hisi_zip_ctx->ops = &hisi_zip_ops; ··· 450 436 451 437 static void hisi_zip_ctx_exit(struct hisi_zip_ctx *hisi_zip_ctx) 452 438 { 439 439 + struct hisi_qp *qps[HZIP_CTX_Q_NUM] = { NULL }; 453 440 int i; 454 441 455 442 for (i = 0; i < HZIP_CTX_Q_NUM; i++) 456 456 - hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); 443 443 + qps[i] = hisi_zip_ctx->qp_ctx[i].qp; 444 444 + 445 445 + hisi_qm_free_qps(qps, HZIP_CTX_Q_NUM); 457 446 } 458 447 459 448 static int hisi_zip_create_req_q(struct hisi_zip_ctx *ctx) ··· 566 549 ret = hisi_zip_ctx_init(ctx, COMP_NAME_TO_TYPE(alg_name), tfm->base.node); 567 550 if (ret) { 568 551 pr_err("failed to init ctx (%d)!\n", ret); 569 569 - return ret; 552 552 + goto switch_to_soft; 570 553 } 571 554 572 555 dev = &ctx->qp_ctx[0].qp->qm->pdev->dev; ··· 591 574 hisi_zip_release_req_q(ctx); 592 575 err_ctx_exit: 593 576 hisi_zip_ctx_exit(ctx); 594 594 - return ret; 577 577 + switch_to_soft: 578 578 + ctx->fallback = true; 579 579 + return 0; 595 580 } 596 581 597 582 static void hisi_zip_acomp_exit(struct crypto_acomp *tfm) 598 583 { 599 584 struct hisi_zip_ctx *ctx = crypto_tfm_ctx(&tfm->base); 600 585 601 601 - hisi_zip_set_acomp_cb(ctx, NULL); 586 586 + if (ctx->fallback) 587 587 + return; 588 588 + 602 589 hisi_zip_release_sgl_pool(ctx); 603 590 hisi_zip_release_req_q(ctx); 604 591 hisi_zip_ctx_exit(ctx); ··· 616 595 .base = { 617 596 .cra_name = "deflate", 618 597 .cra_driver_name = "hisi-deflate-acomp", 619 619 - .cra_flags = CRYPTO_ALG_ASYNC, 598 598 + .cra_flags = CRYPTO_ALG_ASYNC | 599 599 + CRYPTO_ALG_NEED_FALLBACK, 620 600 .cra_module = THIS_MODULE, 621 601 .cra_priority = HZIP_ALG_PRIORITY, 622 602 .cra_ctxsize = sizeof(struct hisi_zip_ctx), ··· 646 624 crypto_unregister_acomp(&hisi_zip_acomp_deflate); 647 625 } 648 626 627 627 + static struct acomp_alg hisi_zip_acomp_lz4 = { 628 628 + .init = hisi_zip_acomp_init, 629 629 + .exit = hisi_zip_acomp_exit, 630 630 + .compress = hisi_zip_acompress, 631 631 + .decompress = hisi_zip_decompress, 632 632 + .base = { 633 633 + .cra_name = "lz4", 634 634 + .cra_driver_name = "hisi-lz4-acomp", 635 635 + .cra_flags = CRYPTO_ALG_ASYNC | 636 636 + CRYPTO_ALG_NEED_FALLBACK, 637 637 + .cra_module = THIS_MODULE, 638 638 + .cra_priority = HZIP_ALG_PRIORITY, 639 639 + .cra_ctxsize = sizeof(struct hisi_zip_ctx), 640 640 + } 641 641 + }; 642 642 + 643 643 + static int hisi_zip_register_lz4(struct hisi_qm *qm) 644 644 + { 645 645 + int ret; 646 646 + 647 647 + if (!hisi_zip_alg_support(qm, HZIP_ALG_LZ4)) 648 648 + return 0; 649 649 + 650 650 + ret = crypto_register_acomp(&hisi_zip_acomp_lz4); 651 651 + if (ret) 652 652 + dev_err(&qm->pdev->dev, "failed to register to LZ4 (%d)!\n", ret); 653 653 + 654 654 + return ret; 655 655 + } 656 656 + 657 657 + static void hisi_zip_unregister_lz4(struct hisi_qm *qm) 658 658 + { 659 659 + if (!hisi_zip_alg_support(qm, HZIP_ALG_LZ4)) 660 660 + return; 661 661 + 662 662 + crypto_unregister_acomp(&hisi_zip_acomp_lz4); 663 663 + } 664 664 + 649 665 int hisi_zip_register_to_crypto(struct hisi_qm *qm) 650 666 { 651 667 int ret = 0; 652 668 653 669 mutex_lock(&zip_algs_lock); 654 654 - if (zip_available_devs++) 670 670 + if (zip_available_devs) { 671 671 + zip_available_devs++; 655 672 goto unlock; 673 673 + } 656 674 657 675 ret = hisi_zip_register_deflate(qm); 658 676 if (ret) 659 659 - zip_available_devs--; 677 677 + goto unlock; 660 678 679 679 + ret = hisi_zip_register_lz4(qm); 680 680 + if (ret) 681 681 + goto unreg_deflate; 682 682 + 683 683 + zip_available_devs++; 684 684 + mutex_unlock(&zip_algs_lock); 685 685 + 686 686 + return 0; 687 687 + 688 688 + unreg_deflate: 689 689 + hisi_zip_unregister_deflate(qm); 661 690 unlock: 662 691 mutex_unlock(&zip_algs_lock); 663 692 return ret; ··· 721 648 goto unlock; 722 649 723 650 hisi_zip_unregister_deflate(qm); 651 651 + hisi_zip_unregister_lz4(qm); 724 652 725 653 unlock: 726 654 mutex_unlock(&zip_algs_lock);

+2 -2

drivers/crypto/hisilicon/zip/zip_main.c

reviewed

··· 446 446 }; 447 447 MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids); 448 448 449 449 - int zip_create_qps(struct hisi_qp **qps, int qp_num, int node) 449 449 + int zip_create_qps(struct hisi_qp **qps, int qp_num, int node, u8 *alg_type) 450 450 { 451 451 if (node == NUMA_NO_NODE) 452 452 node = cpu_to_node(raw_smp_processor_id()); 453 453 454 454 - return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps); 454 454 + return hisi_qm_alloc_qps_node(&zip_devices, qp_num, alg_type, node, qps); 455 455 } 456 456 457 457 bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg)

+7 -14

drivers/crypto/img-hash.c

reviewed

··· 870 870 871 871 for (i = 0; i < ARRAY_SIZE(img_algs); i++) { 872 872 err = crypto_register_ahash(&img_algs[i]); 873 873 - if (err) 874 874 - goto err_reg; 873 873 + if (err) { 874 874 + crypto_unregister_ahashes(img_algs, i); 875 875 + return err; 876 876 + } 875 877 } 878 878 + 876 879 return 0; 877 877 - 878 878 - err_reg: 879 879 - for (; i--; ) 880 880 - crypto_unregister_ahash(&img_algs[i]); 881 881 - 882 882 - return err; 883 880 } 884 881 885 885 - static int img_unregister_algs(struct img_hash_dev *hdev) 882 882 + static void img_unregister_algs(struct img_hash_dev *hdev) 886 883 { 887 887 - int i; 888 888 - 889 889 - for (i = 0; i < ARRAY_SIZE(img_algs); i++) 890 890 - crypto_unregister_ahash(&img_algs[i]); 891 891 - return 0; 884 884 + crypto_unregister_ahashes(img_algs, ARRAY_SIZE(img_algs)); 892 885 } 893 886 894 887 static void img_hash_done_task(unsigned long data)

+54 -40

drivers/crypto/inside-secure/eip93/eip93-main.c

reviewed

··· 77 77 __raw_writel(mask, eip93->base + EIP93_REG_INT_CLR); 78 78 } 79 79 80 80 - static void eip93_unregister_algs(unsigned int i) 80 80 + static int eip93_algo_is_supported(u32 alg_flags, u32 supported_algo_flags) 81 81 + { 82 82 + if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && 83 83 + !(supported_algo_flags & EIP93_PE_OPTION_TDES)) 84 84 + return 0; 85 85 + 86 86 + if (IS_AES(alg_flags) && 87 87 + !(supported_algo_flags & EIP93_PE_OPTION_AES)) 88 88 + return 0; 89 89 + 90 90 + if (IS_HASH_MD5(alg_flags) && 91 91 + !(supported_algo_flags & EIP93_PE_OPTION_MD5)) 92 92 + return 0; 93 93 + 94 94 + if (IS_HASH_SHA1(alg_flags) && 95 95 + !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) 96 96 + return 0; 97 97 + 98 98 + if (IS_HASH_SHA224(alg_flags) && 99 99 + !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) 100 100 + return 0; 101 101 + 102 102 + if (IS_HASH_SHA256(alg_flags) && 103 103 + !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) 104 104 + return 0; 105 105 + 106 106 + return 1; 107 107 + } 108 108 + 109 109 + static void eip93_unregister_algs(u32 supported_algo_flags, unsigned int i) 81 110 { 82 111 unsigned int j; 83 112 84 113 for (j = 0; j < i; j++) { 114 114 + if (!eip93_algo_is_supported(eip93_algs[j]->flags, 115 115 + supported_algo_flags)) 116 116 + continue; 117 117 + 85 118 switch (eip93_algs[j]->type) { 86 119 case EIP93_ALG_TYPE_SKCIPHER: 87 120 crypto_unregister_skcipher(&eip93_algs[j]->alg.skcipher); ··· 123 90 crypto_unregister_aead(&eip93_algs[j]->alg.aead); 124 91 break; 125 92 case EIP93_ALG_TYPE_HASH: 126 126 - crypto_unregister_ahash(&eip93_algs[i]->alg.ahash); 93 93 + crypto_unregister_ahash(&eip93_algs[j]->alg.ahash); 127 94 break; 128 95 } 129 96 } ··· 139 106 140 107 eip93_algs[i]->eip93 = eip93; 141 108 142 142 - if ((IS_DES(alg_flags) || IS_3DES(alg_flags)) && 143 143 - !(supported_algo_flags & EIP93_PE_OPTION_TDES)) 109 109 + if (!eip93_algo_is_supported(alg_flags, supported_algo_flags)) 144 110 continue; 145 111 146 146 - if (IS_AES(alg_flags)) { 147 147 - if (!(supported_algo_flags & EIP93_PE_OPTION_AES)) 148 148 - continue; 112 112 + if (IS_AES(alg_flags) && !IS_HMAC(alg_flags)) { 113 113 + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) 114 114 + eip93_algs[i]->alg.skcipher.max_keysize = 115 115 + AES_KEYSIZE_128; 149 116 150 150 - if (!IS_HMAC(alg_flags)) { 151 151 - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY128) 152 152 - eip93_algs[i]->alg.skcipher.max_keysize = 153 153 - AES_KEYSIZE_128; 117 117 + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) 118 118 + eip93_algs[i]->alg.skcipher.max_keysize = 119 119 + AES_KEYSIZE_192; 154 120 155 155 - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY192) 156 156 - eip93_algs[i]->alg.skcipher.max_keysize = 157 157 - AES_KEYSIZE_192; 121 121 + if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) 122 122 + eip93_algs[i]->alg.skcipher.max_keysize = 123 123 + AES_KEYSIZE_256; 158 124 159 159 - if (supported_algo_flags & EIP93_PE_OPTION_AES_KEY256) 160 160 - eip93_algs[i]->alg.skcipher.max_keysize = 161 161 - AES_KEYSIZE_256; 162 162 - 163 163 - if (IS_RFC3686(alg_flags)) 164 164 - eip93_algs[i]->alg.skcipher.max_keysize += 165 165 - CTR_RFC3686_NONCE_SIZE; 166 166 - } 125 125 + if (IS_RFC3686(alg_flags)) 126 126 + eip93_algs[i]->alg.skcipher.max_keysize += 127 127 + CTR_RFC3686_NONCE_SIZE; 167 128 } 168 168 - 169 169 - if (IS_HASH_MD5(alg_flags) && 170 170 - !(supported_algo_flags & EIP93_PE_OPTION_MD5)) 171 171 - continue; 172 172 - 173 173 - if (IS_HASH_SHA1(alg_flags) && 174 174 - !(supported_algo_flags & EIP93_PE_OPTION_SHA_1)) 175 175 - continue; 176 176 - 177 177 - if (IS_HASH_SHA224(alg_flags) && 178 178 - !(supported_algo_flags & EIP93_PE_OPTION_SHA_224)) 179 179 - continue; 180 180 - 181 181 - if (IS_HASH_SHA256(alg_flags) && 182 182 - !(supported_algo_flags & EIP93_PE_OPTION_SHA_256)) 183 183 - continue; 184 129 185 130 switch (eip93_algs[i]->type) { 186 131 case EIP93_ALG_TYPE_SKCIPHER: ··· 178 167 return 0; 179 168 180 169 fail: 181 181 - eip93_unregister_algs(i); 170 170 + eip93_unregister_algs(supported_algo_flags, i); 182 171 183 172 return ret; 184 173 } ··· 480 469 static void eip93_crypto_remove(struct platform_device *pdev) 481 470 { 482 471 struct eip93_device *eip93 = platform_get_drvdata(pdev); 472 472 + u32 algo_flags; 483 473 484 484 - eip93_unregister_algs(ARRAY_SIZE(eip93_algs)); 474 474 + algo_flags = readl(eip93->base + EIP93_REG_PE_OPTION_1); 475 475 + 476 476 + eip93_unregister_algs(algo_flags, ARRAY_SIZE(eip93_algs)); 485 477 eip93_cleanup(eip93); 486 478 } 487 479

+13 -23

drivers/crypto/intel/iaa/iaa_crypto_main.c

reviewed

··· 5 5 #include <linux/kernel.h> 6 6 #include <linux/module.h> 7 7 #include <linux/pci.h> 8 8 + #include <linux/sysfs.h> 8 9 #include <linux/device.h> 9 10 #include <linux/iommu.h> 10 11 #include <uapi/linux/idxd.h> ··· 97 96 98 97 static ssize_t verify_compress_show(struct device_driver *driver, char *buf) 99 98 { 100 100 - return sprintf(buf, "%d\n", iaa_verify_compress); 99 99 + return sysfs_emit(buf, "%d\n", iaa_verify_compress); 101 100 } 102 101 103 102 static ssize_t verify_compress_store(struct device_driver *driver, ··· 189 188 int ret = 0; 190 189 191 190 if (!async_mode && !use_irq) 192 192 - ret = sprintf(buf, "%s\n", "sync"); 191 191 + ret = sysfs_emit(buf, "%s\n", "sync"); 193 192 else if (async_mode && !use_irq) 194 194 - ret = sprintf(buf, "%s\n", "async"); 193 193 + ret = sysfs_emit(buf, "%s\n", "async"); 195 194 else if (async_mode && use_irq) 196 196 - ret = sprintf(buf, "%s\n", "async_irq"); 195 195 + ret = sysfs_emit(buf, "%s\n", "async_irq"); 197 196 198 197 return ret; 199 198 } ··· 222 221 223 222 static int find_empty_iaa_compression_mode(void) 224 223 { 225 225 - int i = -EINVAL; 224 224 + int i; 226 225 227 227 - for (i = 0; i < IAA_COMP_MODES_MAX; i++) { 228 228 - if (iaa_compression_modes[i]) 229 229 - continue; 230 230 - break; 231 231 - } 226 226 + for (i = 0; i < IAA_COMP_MODES_MAX; i++) 227 227 + if (!iaa_compression_modes[i]) 228 228 + return i; 232 229 233 233 - return i; 230 230 + return -EINVAL; 234 231 } 235 232 236 233 static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx) ··· 543 544 544 545 static int init_iaa_device(struct iaa_device *iaa_device, struct iaa_wq *iaa_wq) 545 546 { 546 546 - int ret = 0; 547 547 - 548 548 - ret = init_device_compression_modes(iaa_device, iaa_wq->wq); 549 549 - if (ret) 550 550 - return ret; 551 551 - 552 552 - return ret; 547 547 + return init_device_compression_modes(iaa_device, iaa_wq->wq); 553 548 } 554 549 555 550 static void del_iaa_device(struct iaa_device *iaa_device) ··· 1697 1704 return ret; 1698 1705 } 1699 1706 1700 1700 - static int iaa_unregister_compression_device(void) 1707 1707 + static void iaa_unregister_compression_device(void) 1701 1708 { 1702 1709 if (iaa_crypto_registered) 1703 1710 crypto_unregister_acomp(&iaa_acomp_fixed_deflate); 1704 1704 - 1705 1705 - return 0; 1706 1711 } 1707 1712 1708 1713 static int iaa_crypto_probe(struct idxd_dev *idxd_dev) ··· 1916 1925 1917 1926 static void __exit iaa_crypto_cleanup_module(void) 1918 1927 { 1919 1919 - if (iaa_unregister_compression_device()) 1920 1920 - pr_debug("IAA compression device unregister failed\n"); 1928 1928 + iaa_unregister_compression_device(); 1921 1929 1922 1930 iaa_crypto_debugfs_cleanup(); 1923 1931 driver_remove_file(&iaa_crypto_driver.drv,

+3

drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c

reviewed

··· 3 3 #include <linux/iopoll.h> 4 4 #include <adf_accel_devices.h> 5 5 #include <adf_admin.h> 6 6 + #include <adf_bank_state.h> 6 7 #include <adf_cfg.h> 7 8 #include <adf_cfg_services.h> 8 9 #include <adf_clock.h> ··· 460 459 hw_data->get_ring_to_svc_map = adf_gen4_get_ring_to_svc_map; 461 460 hw_data->disable_iov = adf_disable_sriov; 462 461 hw_data->ring_pair_reset = adf_gen4_ring_pair_reset; 462 462 + hw_data->bank_state_save = adf_bank_state_save; 463 463 + hw_data->bank_state_restore = adf_bank_state_restore; 463 464 hw_data->enable_pm = adf_gen4_enable_pm; 464 465 hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; 465 466 hw_data->dev_config = adf_gen4_dev_config;

+10

drivers/crypto/intel/qat/qat_common/adf_pfvf_pf_proto.c

reviewed

··· 148 148 blk_byte = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, req.data); 149 149 byte_max = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX; 150 150 break; 151 151 + default: 152 152 + dev_err(&GET_DEV(vf_info->accel_dev), 153 153 + "Invalid BlockMsg type 0x%.4x received from VF%u\n", 154 154 + req.type, vf_info->vf_nr); 155 155 + resp.type = ADF_PF2VF_MSGTYPE_BLKMSG_RESP; 156 156 + resp.data = FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, 157 157 + ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR) | 158 158 + FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, 159 159 + ADF_PF2VF_UNSPECIFIED_ERROR); 160 160 + return resp; 151 161 } 152 162 153 163 /* Is this a request for CRC or data? */

+6 -6

drivers/crypto/intel/qat/qat_common/qat_asym_algs.c

reviewed

··· 255 255 qat_req->areq.dh = req; 256 256 msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; 257 257 msg->pke_hdr.comn_req_flags = 258 258 - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, 259 259 - QAT_COMN_CD_FLD_TYPE_64BIT_ADR); 258 258 + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, 259 259 + QAT_COMN_PTR_TYPE_FLAT); 260 260 261 261 /* 262 262 * If no source is provided use g as base ··· 731 731 qat_req->areq.rsa = req; 732 732 msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; 733 733 msg->pke_hdr.comn_req_flags = 734 734 - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, 735 735 - QAT_COMN_CD_FLD_TYPE_64BIT_ADR); 734 734 + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, 735 735 + QAT_COMN_PTR_TYPE_FLAT); 736 736 737 737 qat_req->in.rsa.enc.e = ctx->dma_e; 738 738 qat_req->in.rsa.enc.n = ctx->dma_n; ··· 867 867 qat_req->areq.rsa = req; 868 868 msg->pke_hdr.service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_PKE; 869 869 msg->pke_hdr.comn_req_flags = 870 870 - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_PTR_TYPE_FLAT, 871 871 - QAT_COMN_CD_FLD_TYPE_64BIT_ADR); 870 870 + ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, 871 871 + QAT_COMN_PTR_TYPE_FLAT); 872 872 873 873 if (ctx->crt_mode) { 874 874 qat_req->in.rsa.dec_crt.p = ctx->dma_p;

+1 -7

drivers/crypto/marvell/cesa/cesa.c

reviewed

··· 38 38 mv_cesa_dequeue_req_locked(struct mv_cesa_engine *engine, 39 39 struct crypto_async_request **backlog) 40 40 { 41 41 - struct crypto_async_request *req; 42 42 - 43 41 *backlog = crypto_get_backlog(&engine->queue); 44 44 - req = crypto_dequeue_request(&engine->queue); 45 42 46 46 - if (!req) 47 47 - return NULL; 48 48 - 49 49 - return req; 43 43 + return crypto_dequeue_request(&engine->queue); 50 44 } 51 45 52 46 static void mv_cesa_rearm_engine(struct mv_cesa_engine *engine)

+1 -1

drivers/crypto/marvell/octeontx/otx_cptpf_ucode.c

reviewed

··· 1326 1326 int del_grp_idx = -1; 1327 1327 int ucode_idx = 0; 1328 1328 1329 1329 - if (strlen(buf) > OTX_CPT_UCODE_NAME_LENGTH) 1329 1329 + if (count >= OTX_CPT_UCODE_NAME_LENGTH) 1330 1330 return -EINVAL; 1331 1331 1332 1332 eng_grps = container_of(attr, struct otx_cpt_eng_grps, ucode_load_attr);

+2 -1

drivers/crypto/marvell/octeontx/otx_cptvf_main.c

reviewed

··· 168 168 chunk = list_first_entry(&cqinfo->queue[i].chead, 169 169 struct otx_cpt_cmd_chunk, nextchunk); 170 170 171 171 - dma_free_coherent(&pdev->dev, chunk->size, 171 171 + dma_free_coherent(&pdev->dev, 172 172 + chunk->size + OTX_CPT_NEXT_CHUNK_PTR_SIZE, 172 173 chunk->head, 173 174 chunk->dma_addr); 174 175 chunk->head = NULL;

+3 -2

drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c

reviewed

··· 2 2 /* Copyright (C) 2020 Marvell. */ 3 3 4 4 #include <linux/firmware.h> 5 5 + #include <linux/sysfs.h> 5 6 #include "otx2_cpt_hw_types.h" 6 7 #include "otx2_cpt_common.h" 7 8 #include "otx2_cpt_devlink.h" ··· 508 507 { 509 508 struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); 510 509 511 511 - return sprintf(buf, "%d\n", cptpf->sso_pf_func_ovrd); 510 510 + return sysfs_emit(buf, "%d\n", cptpf->sso_pf_func_ovrd); 512 511 } 513 512 514 513 static ssize_t sso_pf_func_ovrd_store(struct device *dev, ··· 534 533 { 535 534 struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev); 536 535 537 537 - return sprintf(buf, "%d\n", cptpf->kvf_limits); 536 536 + return sysfs_emit(buf, "%d\n", cptpf->kvf_limits); 538 537 } 539 538 540 539 static ssize_t kvf_limits_store(struct device *dev,

+2 -5

drivers/crypto/nx/nx-common-powernv.c

reviewed

··· 908 908 { 909 909 int chip_id, vasid, ret = 0; 910 910 int ct_842 = 0, ct_gzip = 0; 911 911 - struct device_node *dn; 912 911 913 912 chip_id = of_get_ibm_chip_id(pn); 914 913 if (chip_id < 0) { ··· 921 922 return -EINVAL; 922 923 } 923 924 924 924 - for_each_child_of_node(pn, dn) { 925 925 + for_each_child_of_node_scoped(pn, dn) { 925 926 ret = find_nx_device_tree(dn, chip_id, vasid, NX_CT_842, 926 927 "ibm,p9-nx-842", &ct_842); 927 928 ··· 929 930 ret = find_nx_device_tree(dn, chip_id, vasid, 930 931 NX_CT_GZIP, "ibm,p9-nx-gzip", &ct_gzip); 931 932 932 932 - if (ret) { 933 933 - of_node_put(dn); 933 933 + if (ret) 934 934 return ret; 935 935 - } 936 935 } 937 936 938 937 if (!ct_842 || !ct_gzip) {

+2 -1

drivers/crypto/omap-aes.c

reviewed

··· 32 32 #include <linux/pm_runtime.h> 33 33 #include <linux/scatterlist.h> 34 34 #include <linux/string.h> 35 35 + #include <linux/sysfs.h> 35 36 #include <linux/workqueue.h> 36 37 37 38 #include "omap-crypto.h" ··· 1043 1042 { 1044 1043 struct omap_aes_dev *dd = dev_get_drvdata(dev); 1045 1044 1046 1046 - return sprintf(buf, "%d\n", dd->engine->queue.max_qlen); 1045 1045 + return sysfs_emit(buf, "%d\n", dd->engine->queue.max_qlen); 1047 1046 } 1048 1047 1049 1048 static ssize_t queue_len_store(struct device *dev,

+3 -2

drivers/crypto/omap-sham.c

reviewed

··· 37 37 #include <linux/scatterlist.h> 38 38 #include <linux/slab.h> 39 39 #include <linux/string.h> 40 40 + #include <linux/sysfs.h> 40 41 #include <linux/workqueue.h> 41 42 42 43 #define MD5_DIGEST_SIZE 16 ··· 1974 1973 { 1975 1974 struct omap_sham_dev *dd = dev_get_drvdata(dev); 1976 1975 1977 1977 - return sprintf(buf, "%d\n", dd->fallback_sz); 1976 1976 + return sysfs_emit(buf, "%d\n", dd->fallback_sz); 1978 1977 } 1979 1978 1980 1979 static ssize_t fallback_store(struct device *dev, struct device_attribute *attr, ··· 2004 2003 { 2005 2004 struct omap_sham_dev *dd = dev_get_drvdata(dev); 2006 2005 2007 2007 - return sprintf(buf, "%d\n", dd->queue.max_qlen); 2006 2006 + return sysfs_emit(buf, "%d\n", dd->queue.max_qlen); 2008 2007 } 2009 2008 2010 2009 static ssize_t queue_len_store(struct device *dev,

+7 -2

drivers/crypto/starfive/jh7110-aes.c

reviewed

··· 669 669 return -ENOMEM; 670 670 671 671 if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, cryp->assoclen), 672 672 - rctx->adata, cryp->assoclen) != cryp->assoclen) 672 672 + rctx->adata, cryp->assoclen) != cryp->assoclen) { 673 673 + kfree(rctx->adata); 673 674 return -EINVAL; 675 675 + } 674 676 } 675 677 676 678 if (cryp->total_in) ··· 683 681 ctx->rctx = rctx; 684 682 685 683 ret = starfive_aes_hw_init(ctx); 686 686 - if (ret) 684 684 + if (ret) { 685 685 + if (cryp->assoclen) 686 686 + kfree(rctx->adata); 687 687 return ret; 688 688 + } 688 689 689 690 if (!cryp->assoclen) 690 691 goto write_text;

+3 -1

drivers/crypto/starfive/jh7110-cryp.h

reviewed

··· 216 216 217 217 struct scatterlist *in_sg; 218 218 struct scatterlist *out_sg; 219 219 - struct ahash_request ahash_fbk_req; 220 219 size_t total; 221 220 unsigned int blksize; 222 221 unsigned int digsize; 223 222 unsigned long in_sg_len; 224 223 unsigned char *adata; 225 224 u8 rsa_data[STARFIVE_RSA_MAX_KEYSZ] __aligned(sizeof(u32)); 225 225 + 226 226 + /* Must be last as it ends in a flexible-array member. */ 227 227 + struct ahash_request ahash_fbk_req; 226 228 }; 227 229 228 230 struct starfive_cryp_dev *starfive_cryp_find_dev(struct starfive_cryp_ctx *ctx);

+14 -15

drivers/crypto/stm32/stm32-cryp.c

reviewed

··· 21 21 #include <linux/interrupt.h> 22 22 #include <linux/kernel.h> 23 23 #include <linux/module.h> 24 24 + #include <linux/minmax.h> 24 25 #include <linux/of.h> 25 26 #include <linux/platform_device.h> 26 27 #include <linux/pm_runtime.h> ··· 1923 1922 u32 block[AES_BLOCK_32]; 1924 1923 1925 1924 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); 1926 1926 - memcpy_to_scatterwalk(&cryp->out_walk, block, min_t(size_t, cryp->hw_blocksize, 1927 1927 - cryp->payload_out)); 1928 1928 - cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, 1929 1929 - cryp->payload_out); 1925 1925 + memcpy_to_scatterwalk(&cryp->out_walk, block, min(cryp->hw_blocksize, 1926 1926 + cryp->payload_out)); 1927 1927 + cryp->payload_out -= min(cryp->hw_blocksize, cryp->payload_out); 1930 1928 } 1931 1929 1932 1930 static void stm32_cryp_irq_write_block(struct stm32_cryp *cryp) 1933 1931 { 1934 1932 u32 block[AES_BLOCK_32] = {0}; 1935 1933 1936 1936 - memcpy_from_scatterwalk(block, &cryp->in_walk, min_t(size_t, cryp->hw_blocksize, 1937 1937 - cryp->payload_in)); 1934 1934 + memcpy_from_scatterwalk(block, &cryp->in_walk, min(cryp->hw_blocksize, 1935 1935 + cryp->payload_in)); 1938 1936 writesl(cryp->regs + cryp->caps->din, block, cryp->hw_blocksize / sizeof(u32)); 1939 1939 - cryp->payload_in -= min_t(size_t, cryp->hw_blocksize, cryp->payload_in); 1937 1937 + cryp->payload_in -= min(cryp->hw_blocksize, cryp->payload_in); 1940 1938 } 1941 1939 1942 1940 static void stm32_cryp_irq_write_gcm_padded_data(struct stm32_cryp *cryp) ··· 1980 1980 */ 1981 1981 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); 1982 1982 1983 1983 - memcpy_to_scatterwalk(&cryp->out_walk, block, min_t(size_t, cryp->hw_blocksize, 1984 1984 - cryp->payload_out)); 1985 1985 - cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, 1986 1986 - cryp->payload_out); 1983 1983 + memcpy_to_scatterwalk(&cryp->out_walk, block, min(cryp->hw_blocksize, 1984 1984 + cryp->payload_out)); 1985 1985 + cryp->payload_out -= min(cryp->hw_blocksize, cryp->payload_out); 1987 1986 1988 1987 /* d) change mode back to AES GCM */ 1989 1988 cfg &= ~CR_ALGO_MASK; ··· 2077 2078 */ 2078 2079 readsl(cryp->regs + cryp->caps->dout, block, cryp->hw_blocksize / sizeof(u32)); 2079 2080 2080 2080 - memcpy_to_scatterwalk(&cryp->out_walk, block, min_t(size_t, cryp->hw_blocksize, 2081 2081 - cryp->payload_out)); 2082 2082 - cryp->payload_out -= min_t(size_t, cryp->hw_blocksize, cryp->payload_out); 2081 2081 + memcpy_to_scatterwalk(&cryp->out_walk, block, min(cryp->hw_blocksize, 2082 2082 + cryp->payload_out)); 2083 2083 + cryp->payload_out -= min(cryp->hw_blocksize, cryp->payload_out); 2083 2084 2084 2085 /* d) Load again CRYP_CSGCMCCMxR */ 2085 2086 for (i = 0; i < ARRAY_SIZE(cstmp2); i++) ··· 2157 2158 u32 block[AES_BLOCK_32] = {0}; 2158 2159 size_t written; 2159 2160 2160 2160 - written = min_t(size_t, AES_BLOCK_SIZE, cryp->header_in); 2161 2161 + written = min(AES_BLOCK_SIZE, cryp->header_in); 2161 2162 2162 2163 memcpy_from_scatterwalk(block, &cryp->in_walk, written); 2163 2164

+2 -4

drivers/crypto/stm32/stm32-hash.c

reviewed

··· 1115 1115 return -ENOMEM; 1116 1116 } 1117 1117 1118 1118 - if (state->bufcnt) 1119 1119 - memcpy(buf, rctx->hdev->xmit_buf, state->bufcnt); 1118 1118 + memcpy(buf, rctx->hdev->xmit_buf, state->bufcnt); 1120 1119 1121 1120 scatterwalk_map_and_copy(buf + state->bufcnt, sg, rctx->offset, 1122 1121 min(new_len, rctx->total) - state->bufcnt, 0); ··· 1299 1300 } 1300 1301 1301 1302 /* copy buffer in a temporary one that is used for sg alignment */ 1302 1302 - if (state->bufcnt) 1303 1303 - memcpy(hdev->xmit_buf, state->buffer, state->bufcnt); 1303 1303 + memcpy(hdev->xmit_buf, state->buffer, state->bufcnt); 1304 1304 1305 1305 ret = stm32_hash_align_sgs(req->src, nbytes, bs, init, final, rctx); 1306 1306 if (ret)

+1 -1

drivers/crypto/virtio/virtio_crypto_common.h

reviewed

··· 135 135 int cpu, node; 136 136 137 137 cpu = get_cpu(); 138 138 - node = topology_physical_package_id(cpu); 138 138 + node = cpu_to_node(cpu); 139 139 put_cpu(); 140 140 141 141 return node;

+5

drivers/crypto/virtio/virtio_crypto_core.c

reviewed

··· 75 75 struct data_queue *data_vq = (struct data_queue *)data; 76 76 struct virtqueue *vq = data_vq->vq; 77 77 struct virtio_crypto_request *vc_req; 78 78 + unsigned long flags; 78 79 unsigned int len; 79 80 81 81 + spin_lock_irqsave(&data_vq->lock, flags); 80 82 do { 81 83 virtqueue_disable_cb(vq); 82 84 while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) { 85 85 + spin_unlock_irqrestore(&data_vq->lock, flags); 83 86 if (vc_req->alg_cb) 84 87 vc_req->alg_cb(vc_req, len); 88 88 + spin_lock_irqsave(&data_vq->lock, flags); 85 89 } 86 90 } while (!virtqueue_enable_cb(vq)); 91 91 + spin_unlock_irqrestore(&data_vq->lock, flags); 87 92 } 88 93 89 94 static void virtcrypto_dataq_callback(struct virtqueue *vq)

-2

drivers/crypto/virtio/virtio_crypto_skcipher_algs.c

reviewed

··· 541 541 if (ret < 0) 542 542 return ret; 543 543 544 544 - virtqueue_kick(data_vq->vq); 545 545 - 546 544 return 0; 547 545 } 548 546

+798 -221

drivers/crypto/xilinx/zynqmp-aes-gcm.c

reviewed

··· 1 1 // SPDX-License-Identifier: GPL-2.0 2 2 /* 3 3 * Xilinx ZynqMP AES Driver. 4 4 - * Copyright (c) 2020 Xilinx Inc. 4 4 + * Copyright (C) 2020-2022 Xilinx Inc. 5 5 + * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. 5 6 */ 6 7 7 8 #include <crypto/aes.h> ··· 20 19 #include <linux/string.h> 21 20 22 21 #define ZYNQMP_DMA_BIT_MASK 32U 23 23 - 24 24 - #define ZYNQMP_AES_KEY_SIZE AES_KEYSIZE_256 25 25 - #define ZYNQMP_AES_AUTH_SIZE 16U 26 26 - #define ZYNQMP_KEY_SRC_SEL_KEY_LEN 1U 27 27 - #define ZYNQMP_AES_BLK_SIZE 1U 22 22 + #define VERSAL_DMA_BIT_MASK 64U 23 23 + #define XILINX_AES_AUTH_SIZE 16U 24 24 + #define XILINX_AES_BLK_SIZE 1U 28 25 #define ZYNQMP_AES_MIN_INPUT_BLK_SIZE 4U 29 26 #define ZYNQMP_AES_WORD_LEN 4U 30 27 31 31 - #define ZYNQMP_AES_GCM_TAG_MISMATCH_ERR 0x01 32 32 - #define ZYNQMP_AES_WRONG_KEY_SRC_ERR 0x13 33 33 - #define ZYNQMP_AES_PUF_NOT_PROGRAMMED 0xE300 28 28 + #define VERSAL_AES_QWORD_LEN 16U 29 29 + #define ZYNQMP_AES_GCM_TAG_MISMATCH_ERR 0x01 30 30 + #define ZYNQMP_AES_WRONG_KEY_SRC_ERR 0x13 31 31 + #define ZYNQMP_AES_PUF_NOT_PROGRAMMED 0xE300 32 32 + #define XILINX_KEY_MAGIC 0x3EA0 34 33 35 35 - enum zynqmp_aead_op { 36 36 - ZYNQMP_AES_DECRYPT = 0, 37 37 - ZYNQMP_AES_ENCRYPT 34 34 + enum xilinx_aead_op { 35 35 + XILINX_AES_DECRYPT = 0, 36 36 + XILINX_AES_ENCRYPT 38 37 }; 39 38 40 39 enum zynqmp_aead_keysrc { ··· 43 42 ZYNQMP_AES_PUF_KEY 44 43 }; 45 44 46 46 - struct zynqmp_aead_drv_ctx { 47 47 - union { 48 48 - struct aead_engine_alg aead; 49 49 - } alg; 45 45 + struct xilinx_aead_dev { 50 46 struct device *dev; 51 47 struct crypto_engine *engine; 48 48 + struct xilinx_aead_alg *aead_algs; 52 49 }; 50 50 + 51 51 + struct xilinx_aead_alg { 52 52 + struct xilinx_aead_dev *aead_dev; 53 53 + struct aead_engine_alg aead; 54 54 + int (*aes_aead_cipher)(struct aead_request *areq); 55 55 + u8 dma_bit_mask; 56 56 + }; 57 57 + 58 58 + struct xilinx_hwkey_info { 59 59 + u16 magic; 60 60 + u16 type; 61 61 + } __packed; 53 62 54 63 struct zynqmp_aead_hw_req { 55 64 u64 src; ··· 71 60 u64 keysrc; 72 61 }; 73 62 74 74 - struct zynqmp_aead_tfm_ctx { 63 63 + struct xilinx_aead_tfm_ctx { 75 64 struct device *dev; 76 76 - u8 key[ZYNQMP_AES_KEY_SIZE]; 77 77 - u8 *iv; 65 65 + dma_addr_t key_dma_addr; 66 66 + u8 *key; 78 67 u32 keylen; 79 68 u32 authsize; 80 80 - enum zynqmp_aead_keysrc keysrc; 69 69 + u8 keysrc; 81 70 struct crypto_aead *fbk_cipher; 82 71 }; 83 72 84 84 - struct zynqmp_aead_req_ctx { 85 85 - enum zynqmp_aead_op op; 73 73 + struct xilinx_aead_req_ctx { 74 74 + enum xilinx_aead_op op; 75 75 + }; 76 76 + 77 77 + static struct xilinx_aead_dev *aead_dev; 78 78 + 79 79 + enum versal_aead_keysrc { 80 80 + VERSAL_AES_BBRAM_KEY = 0, 81 81 + VERSAL_AES_BBRAM_RED_KEY, 82 82 + VERSAL_AES_BH_KEY, 83 83 + VERSAL_AES_BH_RED_KEY, 84 84 + VERSAL_AES_EFUSE_KEY, 85 85 + VERSAL_AES_EFUSE_RED_KEY, 86 86 + VERSAL_AES_EFUSE_USER_KEY_0, 87 87 + VERSAL_AES_EFUSE_USER_KEY_1, 88 88 + VERSAL_AES_EFUSE_USER_RED_KEY_0, 89 89 + VERSAL_AES_EFUSE_USER_RED_KEY_1, 90 90 + VERSAL_AES_KUP_KEY, 91 91 + VERSAL_AES_PUF_KEY, 92 92 + VERSAL_AES_USER_KEY_0, 93 93 + VERSAL_AES_USER_KEY_1, 94 94 + VERSAL_AES_USER_KEY_2, 95 95 + VERSAL_AES_USER_KEY_3, 96 96 + VERSAL_AES_USER_KEY_4, 97 97 + VERSAL_AES_USER_KEY_5, 98 98 + VERSAL_AES_USER_KEY_6, 99 99 + VERSAL_AES_USER_KEY_7, 100 100 + VERSAL_AES_EXPANDED_KEYS, 101 101 + VERSAL_AES_ALL_KEYS, 102 102 + }; 103 103 + 104 104 + enum versal_aead_op { 105 105 + VERSAL_AES_ENCRYPT = 0, 106 106 + VERSAL_AES_DECRYPT 107 107 + }; 108 108 + 109 109 + enum versal_aes_keysize { 110 110 + HW_AES_KEY_SIZE_128 = 0, 111 111 + HW_AES_KEY_SIZE_256 = 2, 112 112 + }; 113 113 + 114 114 + struct versal_init_ops { 115 115 + u64 iv; 116 116 + u32 op; 117 117 + u32 keysrc; 118 118 + u32 size; 119 119 + }; 120 120 + 121 121 + struct versal_in_params { 122 122 + u64 in_data_addr; 123 123 + u32 size; 124 124 + u32 is_last; 86 125 }; 87 126 88 127 static int zynqmp_aes_aead_cipher(struct aead_request *req) 89 128 { 90 129 struct crypto_aead *aead = crypto_aead_reqtfm(req); 91 91 - struct zynqmp_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 92 92 - struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req); 130 130 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 131 131 + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 132 132 + dma_addr_t dma_addr_data, dma_addr_hw_req; 93 133 struct device *dev = tfm_ctx->dev; 94 134 struct zynqmp_aead_hw_req *hwreq; 95 95 - dma_addr_t dma_addr_data, dma_addr_hw_req; 96 135 unsigned int data_size; 97 136 unsigned int status; 98 137 int ret; 99 138 size_t dma_size; 139 139 + void *dmabuf; 100 140 char *kbuf; 101 101 - int err; 102 141 103 103 - if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY) 104 104 - dma_size = req->cryptlen + ZYNQMP_AES_KEY_SIZE 105 105 - + GCM_AES_IV_SIZE; 106 106 - else 107 107 - dma_size = req->cryptlen + GCM_AES_IV_SIZE; 108 108 - 109 109 - kbuf = dma_alloc_coherent(dev, dma_size, &dma_addr_data, GFP_KERNEL); 142 142 + dma_size = req->cryptlen + XILINX_AES_AUTH_SIZE; 143 143 + kbuf = kmalloc(dma_size, GFP_KERNEL); 110 144 if (!kbuf) 111 145 return -ENOMEM; 112 146 113 113 - hwreq = dma_alloc_coherent(dev, sizeof(struct zynqmp_aead_hw_req), 114 114 - &dma_addr_hw_req, GFP_KERNEL); 115 115 - if (!hwreq) { 116 116 - dma_free_coherent(dev, dma_size, kbuf, dma_addr_data); 147 147 + dmabuf = kmalloc(sizeof(*hwreq) + GCM_AES_IV_SIZE, GFP_KERNEL); 148 148 + if (!dmabuf) { 149 149 + kfree(kbuf); 117 150 return -ENOMEM; 118 151 } 119 119 - 152 152 + hwreq = dmabuf; 120 153 data_size = req->cryptlen; 121 154 scatterwalk_map_and_copy(kbuf, req->src, 0, req->cryptlen, 0); 122 122 - memcpy(kbuf + data_size, req->iv, GCM_AES_IV_SIZE); 155 155 + memcpy(dmabuf + sizeof(struct zynqmp_aead_hw_req), req->iv, GCM_AES_IV_SIZE); 156 156 + dma_addr_data = dma_map_single(dev, kbuf, dma_size, DMA_BIDIRECTIONAL); 157 157 + if (unlikely(dma_mapping_error(dev, dma_addr_data))) { 158 158 + ret = -ENOMEM; 159 159 + goto freemem; 160 160 + } 123 161 124 162 hwreq->src = dma_addr_data; 125 163 hwreq->dst = dma_addr_data; 126 126 - hwreq->iv = hwreq->src + data_size; 127 164 hwreq->keysrc = tfm_ctx->keysrc; 128 165 hwreq->op = rq_ctx->op; 129 166 130 130 - if (hwreq->op == ZYNQMP_AES_ENCRYPT) 167 167 + if (hwreq->op == XILINX_AES_ENCRYPT) 131 168 hwreq->size = data_size; 132 169 else 133 133 - hwreq->size = data_size - ZYNQMP_AES_AUTH_SIZE; 170 170 + hwreq->size = data_size - XILINX_AES_AUTH_SIZE; 134 171 135 135 - if (hwreq->keysrc == ZYNQMP_AES_KUP_KEY) { 136 136 - memcpy(kbuf + data_size + GCM_AES_IV_SIZE, 137 137 - tfm_ctx->key, ZYNQMP_AES_KEY_SIZE); 138 138 - 139 139 - hwreq->key = hwreq->src + data_size + GCM_AES_IV_SIZE; 140 140 - } else { 172 172 + if (hwreq->keysrc == ZYNQMP_AES_KUP_KEY) 173 173 + hwreq->key = tfm_ctx->key_dma_addr; 174 174 + else 141 175 hwreq->key = 0; 176 176 + 177 177 + dma_addr_hw_req = dma_map_single(dev, dmabuf, sizeof(struct zynqmp_aead_hw_req) + 178 178 + GCM_AES_IV_SIZE, 179 179 + DMA_TO_DEVICE); 180 180 + if (unlikely(dma_mapping_error(dev, dma_addr_hw_req))) { 181 181 + ret = -ENOMEM; 182 182 + dma_unmap_single(dev, dma_addr_data, dma_size, DMA_BIDIRECTIONAL); 183 183 + goto freemem; 142 184 } 143 143 - 185 185 + hwreq->iv = dma_addr_hw_req + sizeof(struct zynqmp_aead_hw_req); 186 186 + dma_sync_single_for_device(dev, dma_addr_hw_req, sizeof(struct zynqmp_aead_hw_req) + 187 187 + GCM_AES_IV_SIZE, DMA_TO_DEVICE); 144 188 ret = zynqmp_pm_aes_engine(dma_addr_hw_req, &status); 145 145 - 189 189 + dma_unmap_single(dev, dma_addr_hw_req, sizeof(struct zynqmp_aead_hw_req) + GCM_AES_IV_SIZE, 190 190 + DMA_TO_DEVICE); 191 191 + dma_unmap_single(dev, dma_addr_data, dma_size, DMA_BIDIRECTIONAL); 146 192 if (ret) { 147 193 dev_err(dev, "ERROR: AES PM API failed\n"); 148 148 - err = ret; 149 194 } else if (status) { 150 195 switch (status) { 151 196 case ZYNQMP_AES_GCM_TAG_MISMATCH_ERR: 152 152 - dev_err(dev, "ERROR: Gcm Tag mismatch\n"); 197 197 + ret = -EBADMSG; 153 198 break; 154 199 case ZYNQMP_AES_WRONG_KEY_SRC_ERR: 200 200 + ret = -EINVAL; 155 201 dev_err(dev, "ERROR: Wrong KeySrc, enable secure mode\n"); 156 202 break; 157 203 case ZYNQMP_AES_PUF_NOT_PROGRAMMED: 204 204 + ret = -EINVAL; 158 205 dev_err(dev, "ERROR: PUF is not registered\n"); 159 206 break; 160 207 default: 161 161 - dev_err(dev, "ERROR: Unknown error\n"); 208 208 + ret = -EINVAL; 162 209 break; 163 210 } 164 164 - err = -status; 165 211 } else { 166 166 - if (hwreq->op == ZYNQMP_AES_ENCRYPT) 167 167 - data_size = data_size + ZYNQMP_AES_AUTH_SIZE; 212 212 + if (hwreq->op == XILINX_AES_ENCRYPT) 213 213 + data_size = data_size + crypto_aead_authsize(aead); 168 214 else 169 169 - data_size = data_size - ZYNQMP_AES_AUTH_SIZE; 215 215 + data_size = data_size - XILINX_AES_AUTH_SIZE; 170 216 171 217 sg_copy_from_buffer(req->dst, sg_nents(req->dst), 172 218 kbuf, data_size); 173 173 - err = 0; 219 219 + ret = 0; 174 220 } 175 221 176 176 - if (kbuf) { 177 177 - memzero_explicit(kbuf, dma_size); 178 178 - dma_free_coherent(dev, dma_size, kbuf, dma_addr_data); 179 179 - } 180 180 - if (hwreq) { 181 181 - memzero_explicit(hwreq, sizeof(struct zynqmp_aead_hw_req)); 182 182 - dma_free_coherent(dev, sizeof(struct zynqmp_aead_hw_req), 183 183 - hwreq, dma_addr_hw_req); 184 184 - } 185 185 - return err; 222 222 + freemem: 223 223 + memzero_explicit(kbuf, dma_size); 224 224 + kfree(kbuf); 225 225 + memzero_explicit(dmabuf, sizeof(struct zynqmp_aead_hw_req) + GCM_AES_IV_SIZE); 226 226 + kfree(dmabuf); 227 227 + 228 228 + return ret; 186 229 } 187 230 188 188 - static int zynqmp_fallback_check(struct zynqmp_aead_tfm_ctx *tfm_ctx, 231 231 + static int versal_aes_aead_cipher(struct aead_request *req) 232 232 + { 233 233 + struct crypto_aead *aead = crypto_aead_reqtfm(req); 234 234 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 235 235 + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 236 236 + dma_addr_t dma_addr_data, dma_addr_hw_req, dma_addr_in; 237 237 + u32 total_len = req->assoclen + req->cryptlen; 238 238 + struct device *dev = tfm_ctx->dev; 239 239 + struct versal_init_ops *hwreq; 240 240 + struct versal_in_params *in; 241 241 + u32 gcm_offset, out_len; 242 242 + size_t dmabuf_size; 243 243 + size_t kbuf_size; 244 244 + void *dmabuf; 245 245 + char *kbuf; 246 246 + int ret; 247 247 + 248 248 + kbuf_size = total_len + XILINX_AES_AUTH_SIZE; 249 249 + kbuf = kmalloc(kbuf_size, GFP_KERNEL); 250 250 + if (unlikely(!kbuf)) { 251 251 + ret = -ENOMEM; 252 252 + goto err; 253 253 + } 254 254 + dmabuf_size = sizeof(struct versal_init_ops) + 255 255 + sizeof(struct versal_in_params) + 256 256 + GCM_AES_IV_SIZE; 257 257 + dmabuf = kmalloc(dmabuf_size, GFP_KERNEL); 258 258 + if (unlikely(!dmabuf)) { 259 259 + ret = -ENOMEM; 260 260 + goto buf1_free; 261 261 + } 262 262 + 263 263 + dma_addr_hw_req = dma_map_single(dev, dmabuf, dmabuf_size, DMA_BIDIRECTIONAL); 264 264 + if (unlikely(dma_mapping_error(dev, dma_addr_hw_req))) { 265 265 + ret = -ENOMEM; 266 266 + goto buf2_free; 267 267 + } 268 268 + scatterwalk_map_and_copy(kbuf, req->src, 0, total_len, 0); 269 269 + dma_addr_data = dma_map_single(dev, kbuf, kbuf_size, DMA_BIDIRECTIONAL); 270 270 + if (unlikely(dma_mapping_error(dev, dma_addr_data))) { 271 271 + dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL); 272 272 + ret = -ENOMEM; 273 273 + goto buf2_free; 274 274 + } 275 275 + hwreq = dmabuf; 276 276 + in = dmabuf + sizeof(struct versal_init_ops); 277 277 + memcpy(dmabuf + sizeof(struct versal_init_ops) + 278 278 + sizeof(struct versal_in_params), req->iv, GCM_AES_IV_SIZE); 279 279 + hwreq->iv = dma_addr_hw_req + sizeof(struct versal_init_ops) + 280 280 + sizeof(struct versal_in_params); 281 281 + hwreq->keysrc = tfm_ctx->keysrc; 282 282 + dma_addr_in = dma_addr_hw_req + sizeof(struct versal_init_ops); 283 283 + if (rq_ctx->op == XILINX_AES_ENCRYPT) { 284 284 + hwreq->op = VERSAL_AES_ENCRYPT; 285 285 + out_len = total_len + crypto_aead_authsize(aead); 286 286 + in->size = req->cryptlen; 287 287 + } else { 288 288 + hwreq->op = VERSAL_AES_DECRYPT; 289 289 + out_len = total_len - XILINX_AES_AUTH_SIZE; 290 290 + in->size = req->cryptlen - XILINX_AES_AUTH_SIZE; 291 291 + } 292 292 + 293 293 + if (tfm_ctx->keylen == AES_KEYSIZE_128) 294 294 + hwreq->size = HW_AES_KEY_SIZE_128; 295 295 + else 296 296 + hwreq->size = HW_AES_KEY_SIZE_256; 297 297 + 298 298 + /* Request aes key write for volatile user keys */ 299 299 + if (hwreq->keysrc >= VERSAL_AES_USER_KEY_0 && hwreq->keysrc <= VERSAL_AES_USER_KEY_7) { 300 300 + ret = versal_pm_aes_key_write(hwreq->size, hwreq->keysrc, 301 301 + tfm_ctx->key_dma_addr); 302 302 + if (ret) 303 303 + goto unmap; 304 304 + } 305 305 + 306 306 + in->in_data_addr = dma_addr_data + req->assoclen; 307 307 + in->is_last = 1; 308 308 + gcm_offset = req->assoclen + in->size; 309 309 + dma_sync_single_for_device(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL); 310 310 + ret = versal_pm_aes_op_init(dma_addr_hw_req); 311 311 + if (ret) 312 312 + goto clearkey; 313 313 + 314 314 + if (req->assoclen > 0) { 315 315 + /* Currently GMAC is OFF by default */ 316 316 + ret = versal_pm_aes_update_aad(dma_addr_data, req->assoclen); 317 317 + if (ret) 318 318 + goto clearkey; 319 319 + } 320 320 + if (rq_ctx->op == XILINX_AES_ENCRYPT) { 321 321 + ret = versal_pm_aes_enc_update(dma_addr_in, 322 322 + dma_addr_data + req->assoclen); 323 323 + if (ret) 324 324 + goto clearkey; 325 325 + 326 326 + ret = versal_pm_aes_enc_final(dma_addr_data + gcm_offset); 327 327 + if (ret) 328 328 + goto clearkey; 329 329 + } else { 330 330 + ret = versal_pm_aes_dec_update(dma_addr_in, 331 331 + dma_addr_data + req->assoclen); 332 332 + if (ret) 333 333 + goto clearkey; 334 334 + 335 335 + ret = versal_pm_aes_dec_final(dma_addr_data + gcm_offset); 336 336 + if (ret) { 337 337 + ret = -EBADMSG; 338 338 + goto clearkey; 339 339 + } 340 340 + } 341 341 + dma_unmap_single(dev, dma_addr_data, kbuf_size, DMA_BIDIRECTIONAL); 342 342 + dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL); 343 343 + sg_copy_from_buffer(req->dst, sg_nents(req->dst), 344 344 + kbuf, out_len); 345 345 + dma_addr_data = 0; 346 346 + dma_addr_hw_req = 0; 347 347 + 348 348 + clearkey: 349 349 + if (hwreq->keysrc >= VERSAL_AES_USER_KEY_0 && hwreq->keysrc <= VERSAL_AES_USER_KEY_7) 350 350 + versal_pm_aes_key_zero(hwreq->keysrc); 351 351 + unmap: 352 352 + if (unlikely(dma_addr_data)) 353 353 + dma_unmap_single(dev, dma_addr_data, kbuf_size, DMA_BIDIRECTIONAL); 354 354 + if (unlikely(dma_addr_hw_req)) 355 355 + dma_unmap_single(dev, dma_addr_hw_req, dmabuf_size, DMA_BIDIRECTIONAL); 356 356 + buf2_free: 357 357 + memzero_explicit(dmabuf, dmabuf_size); 358 358 + kfree(dmabuf); 359 359 + buf1_free: 360 360 + memzero_explicit(kbuf, kbuf_size); 361 361 + kfree(kbuf); 362 362 + err: 363 363 + return ret; 364 364 + } 365 365 + 366 366 + static int zynqmp_fallback_check(struct xilinx_aead_tfm_ctx *tfm_ctx, 189 367 struct aead_request *req) 190 368 { 191 191 - int need_fallback = 0; 192 192 - struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req); 369 369 + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 193 370 194 194 - if (tfm_ctx->authsize != ZYNQMP_AES_AUTH_SIZE) 195 195 - need_fallback = 1; 371 371 + if (tfm_ctx->authsize != XILINX_AES_AUTH_SIZE && rq_ctx->op == XILINX_AES_DECRYPT) 372 372 + return 1; 196 373 197 197 - if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY && 198 198 - tfm_ctx->keylen != ZYNQMP_AES_KEY_SIZE) { 199 199 - need_fallback = 1; 200 200 - } 201 374 if (req->assoclen != 0 || 202 202 - req->cryptlen < ZYNQMP_AES_MIN_INPUT_BLK_SIZE) { 203 203 - need_fallback = 1; 204 204 - } 205 205 - if ((req->cryptlen % ZYNQMP_AES_WORD_LEN) != 0) 206 206 - need_fallback = 1; 375 375 + req->cryptlen < ZYNQMP_AES_MIN_INPUT_BLK_SIZE) 376 376 + return 1; 377 377 + if (tfm_ctx->keylen == AES_KEYSIZE_128 || 378 378 + tfm_ctx->keylen == AES_KEYSIZE_192) 379 379 + return 1; 207 380 208 208 - if (rq_ctx->op == ZYNQMP_AES_DECRYPT && 209 209 - req->cryptlen <= ZYNQMP_AES_AUTH_SIZE) { 210 210 - need_fallback = 1; 211 211 - } 212 212 - return need_fallback; 381 381 + if ((req->cryptlen % ZYNQMP_AES_WORD_LEN) != 0) 382 382 + return 1; 383 383 + 384 384 + if (rq_ctx->op == XILINX_AES_DECRYPT && 385 385 + req->cryptlen <= XILINX_AES_AUTH_SIZE) 386 386 + return 1; 387 387 + 388 388 + return 0; 213 389 } 214 390 215 215 - static int zynqmp_handle_aes_req(struct crypto_engine *engine, 216 216 - void *req) 391 391 + static int versal_fallback_check(struct xilinx_aead_tfm_ctx *tfm_ctx, 392 392 + struct aead_request *req) 393 393 + { 394 394 + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 395 395 + 396 396 + if (tfm_ctx->authsize != XILINX_AES_AUTH_SIZE && rq_ctx->op == XILINX_AES_DECRYPT) 397 397 + return 1; 398 398 + 399 399 + if (tfm_ctx->keylen == AES_KEYSIZE_192) 400 400 + return 1; 401 401 + 402 402 + if (req->cryptlen < ZYNQMP_AES_MIN_INPUT_BLK_SIZE || 403 403 + req->cryptlen % ZYNQMP_AES_WORD_LEN || 404 404 + req->assoclen % VERSAL_AES_QWORD_LEN) 405 405 + return 1; 406 406 + 407 407 + if (rq_ctx->op == XILINX_AES_DECRYPT && 408 408 + req->cryptlen <= XILINX_AES_AUTH_SIZE) 409 409 + return 1; 410 410 + 411 411 + return 0; 412 412 + } 413 413 + 414 414 + static int xilinx_handle_aes_req(struct crypto_engine *engine, void *req) 217 415 { 218 416 struct aead_request *areq = 219 417 container_of(req, struct aead_request, base); 220 418 struct crypto_aead *aead = crypto_aead_reqtfm(req); 221 221 - struct zynqmp_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 222 222 - struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(areq); 223 223 - struct aead_request *subreq = aead_request_ctx(req); 224 224 - int need_fallback; 419 419 + struct aead_alg *alg = crypto_aead_alg(aead); 420 420 + struct xilinx_aead_alg *drv_ctx; 225 421 int err; 226 422 227 227 - need_fallback = zynqmp_fallback_check(tfm_ctx, areq); 228 228 - 229 229 - if (need_fallback) { 230 230 - aead_request_set_tfm(subreq, tfm_ctx->fbk_cipher); 231 231 - 232 232 - aead_request_set_callback(subreq, areq->base.flags, 233 233 - NULL, NULL); 234 234 - aead_request_set_crypt(subreq, areq->src, areq->dst, 235 235 - areq->cryptlen, areq->iv); 236 236 - aead_request_set_ad(subreq, areq->assoclen); 237 237 - if (rq_ctx->op == ZYNQMP_AES_ENCRYPT) 238 238 - err = crypto_aead_encrypt(subreq); 239 239 - else 240 240 - err = crypto_aead_decrypt(subreq); 241 241 - } else { 242 242 - err = zynqmp_aes_aead_cipher(areq); 243 243 - } 244 244 - 423 423 + drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); 424 424 + err = drv_ctx->aes_aead_cipher(areq); 245 425 local_bh_disable(); 246 426 crypto_finalize_aead_request(engine, areq, err); 247 427 local_bh_enable(); ··· 444 242 unsigned int keylen) 445 243 { 446 244 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 447 447 - struct zynqmp_aead_tfm_ctx *tfm_ctx = 448 448 - (struct zynqmp_aead_tfm_ctx *)crypto_tfm_ctx(tfm); 449 449 - unsigned char keysrc; 245 245 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 246 246 + int err; 450 247 451 451 - if (keylen == ZYNQMP_KEY_SRC_SEL_KEY_LEN) { 452 452 - keysrc = *key; 453 453 - if (keysrc == ZYNQMP_AES_KUP_KEY || 454 454 - keysrc == ZYNQMP_AES_DEV_KEY || 455 455 - keysrc == ZYNQMP_AES_PUF_KEY) { 456 456 - tfm_ctx->keysrc = (enum zynqmp_aead_keysrc)keysrc; 457 457 - } else { 458 458 - tfm_ctx->keylen = keylen; 459 459 - } 460 460 - } else { 461 461 - tfm_ctx->keylen = keylen; 462 462 - if (keylen == ZYNQMP_AES_KEY_SIZE) { 463 463 - tfm_ctx->keysrc = ZYNQMP_AES_KUP_KEY; 464 464 - memcpy(tfm_ctx->key, key, keylen); 465 465 - } 248 248 + if (keylen == AES_KEYSIZE_256) { 249 249 + memcpy(tfm_ctx->key, key, keylen); 250 250 + dma_sync_single_for_device(tfm_ctx->dev, tfm_ctx->key_dma_addr, 251 251 + AES_KEYSIZE_256, 252 252 + DMA_TO_DEVICE); 466 253 } 467 254 468 255 tfm_ctx->fbk_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; 469 256 tfm_ctx->fbk_cipher->base.crt_flags |= (aead->base.crt_flags & 470 470 - CRYPTO_TFM_REQ_MASK); 257 257 + CRYPTO_TFM_REQ_MASK); 471 258 472 472 - return crypto_aead_setkey(tfm_ctx->fbk_cipher, key, keylen); 259 259 + err = crypto_aead_setkey(tfm_ctx->fbk_cipher, key, keylen); 260 260 + if (err) 261 261 + goto err; 262 262 + tfm_ctx->keylen = keylen; 263 263 + tfm_ctx->keysrc = ZYNQMP_AES_KUP_KEY; 264 264 + err: 265 265 + return err; 473 266 } 474 267 475 475 - static int zynqmp_aes_aead_setauthsize(struct crypto_aead *aead, 268 268 + static int zynqmp_paes_aead_setkey(struct crypto_aead *aead, const u8 *key, 269 269 + unsigned int keylen) 270 270 + { 271 271 + struct crypto_tfm *tfm = crypto_aead_tfm(aead); 272 272 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 273 273 + struct xilinx_hwkey_info hwkey; 274 274 + unsigned char keysrc; 275 275 + int err = -EINVAL; 276 276 + 277 277 + if (keylen != sizeof(struct xilinx_hwkey_info)) 278 278 + return -EINVAL; 279 279 + memcpy(&hwkey, key, sizeof(struct xilinx_hwkey_info)); 280 280 + if (hwkey.magic != XILINX_KEY_MAGIC) 281 281 + return -EINVAL; 282 282 + keysrc = hwkey.type; 283 283 + if (keysrc == ZYNQMP_AES_DEV_KEY || 284 284 + keysrc == ZYNQMP_AES_PUF_KEY) { 285 285 + tfm_ctx->keysrc = keysrc; 286 286 + tfm_ctx->keylen = sizeof(struct xilinx_hwkey_info); 287 287 + err = 0; 288 288 + } 289 289 + 290 290 + return err; 291 291 + } 292 292 + 293 293 + static int versal_aes_aead_setkey(struct crypto_aead *aead, const u8 *key, 294 294 + unsigned int keylen) 295 295 + { 296 296 + struct crypto_tfm *tfm = crypto_aead_tfm(aead); 297 297 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 298 298 + struct xilinx_hwkey_info hwkey; 299 299 + unsigned char keysrc; 300 300 + int err; 301 301 + 302 302 + tfm_ctx->keysrc = VERSAL_AES_USER_KEY_0; 303 303 + if (keylen == sizeof(struct xilinx_hwkey_info)) { 304 304 + memcpy(&hwkey, key, sizeof(struct xilinx_hwkey_info)); 305 305 + if (hwkey.magic != XILINX_KEY_MAGIC) 306 306 + return -EINVAL; 307 307 + 308 308 + keysrc = hwkey.type; 309 309 + if (keysrc >= VERSAL_AES_USER_KEY_1 && 310 310 + keysrc <= VERSAL_AES_USER_KEY_7) { 311 311 + tfm_ctx->keysrc = keysrc; 312 312 + tfm_ctx->keylen = sizeof(struct xilinx_hwkey_info); 313 313 + return 0; 314 314 + } 315 315 + return -EINVAL; 316 316 + } 317 317 + 318 318 + if (keylen == AES_KEYSIZE_256 || keylen == AES_KEYSIZE_128) { 319 319 + tfm_ctx->keylen = keylen; 320 320 + memcpy(tfm_ctx->key, key, keylen); 321 321 + dma_sync_single_for_device(tfm_ctx->dev, tfm_ctx->key_dma_addr, 322 322 + AES_KEYSIZE_256, 323 323 + DMA_TO_DEVICE); 324 324 + } 325 325 + 326 326 + tfm_ctx->fbk_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; 327 327 + tfm_ctx->fbk_cipher->base.crt_flags |= (aead->base.crt_flags & 328 328 + CRYPTO_TFM_REQ_MASK); 329 329 + err = crypto_aead_setkey(tfm_ctx->fbk_cipher, key, keylen); 330 330 + if (!err) 331 331 + tfm_ctx->keylen = keylen; 332 332 + 333 333 + return err; 334 334 + } 335 335 + 336 336 + static int versal_paes_aead_setkey(struct crypto_aead *aead, const u8 *key, 337 337 + unsigned int keylen) 338 338 + { 339 339 + struct crypto_tfm *tfm = crypto_aead_tfm(aead); 340 340 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 341 341 + struct xilinx_hwkey_info hwkey; 342 342 + unsigned char keysrc; 343 343 + int err = 0; 344 344 + 345 345 + if (keylen != sizeof(struct xilinx_hwkey_info)) 346 346 + return -EINVAL; 347 347 + 348 348 + memcpy(&hwkey, key, sizeof(struct xilinx_hwkey_info)); 349 349 + if (hwkey.magic != XILINX_KEY_MAGIC) 350 350 + return -EINVAL; 351 351 + 352 352 + keysrc = hwkey.type; 353 353 + 354 354 + switch (keysrc) { 355 355 + case VERSAL_AES_EFUSE_USER_KEY_0: 356 356 + case VERSAL_AES_EFUSE_USER_KEY_1: 357 357 + case VERSAL_AES_EFUSE_USER_RED_KEY_0: 358 358 + case VERSAL_AES_EFUSE_USER_RED_KEY_1: 359 359 + case VERSAL_AES_PUF_KEY: 360 360 + tfm_ctx->keysrc = keysrc; 361 361 + tfm_ctx->keylen = sizeof(struct xilinx_hwkey_info); 362 362 + break; 363 363 + default: 364 364 + err = -EINVAL; 365 365 + break; 366 366 + } 367 367 + 368 368 + return err; 369 369 + } 370 370 + 371 371 + static int xilinx_aes_aead_setauthsize(struct crypto_aead *aead, 476 372 unsigned int authsize) 477 373 { 478 374 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 479 479 - struct zynqmp_aead_tfm_ctx *tfm_ctx = 480 480 - (struct zynqmp_aead_tfm_ctx *)crypto_tfm_ctx(tfm); 375 375 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 481 376 482 377 tfm_ctx->authsize = authsize; 483 483 - return crypto_aead_setauthsize(tfm_ctx->fbk_cipher, authsize); 378 378 + return tfm_ctx->fbk_cipher ? crypto_aead_setauthsize(tfm_ctx->fbk_cipher, authsize) : 0; 379 379 + } 380 380 + 381 381 + static int xilinx_aes_fallback_crypt(struct aead_request *req, bool encrypt) 382 382 + { 383 383 + struct aead_request *subreq = aead_request_ctx(req); 384 384 + struct crypto_aead *aead = crypto_aead_reqtfm(req); 385 385 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 386 386 + 387 387 + aead_request_set_tfm(subreq, tfm_ctx->fbk_cipher); 388 388 + aead_request_set_callback(subreq, req->base.flags, NULL, NULL); 389 389 + aead_request_set_crypt(subreq, req->src, req->dst, 390 390 + req->cryptlen, req->iv); 391 391 + aead_request_set_ad(subreq, req->assoclen); 392 392 + 393 393 + return encrypt ? crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); 484 394 } 485 395 486 396 static int zynqmp_aes_aead_encrypt(struct aead_request *req) 487 397 { 488 488 - struct zynqmp_aead_drv_ctx *drv_ctx; 398 398 + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 489 399 struct crypto_aead *aead = crypto_aead_reqtfm(req); 400 400 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 490 401 struct aead_alg *alg = crypto_aead_alg(aead); 491 491 - struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req); 402 402 + struct xilinx_aead_alg *drv_ctx; 403 403 + int err; 492 404 493 493 - rq_ctx->op = ZYNQMP_AES_ENCRYPT; 494 494 - drv_ctx = container_of(alg, struct zynqmp_aead_drv_ctx, alg.aead.base); 405 405 + drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); 406 406 + if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY && 407 407 + tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info)) 408 408 + return -EINVAL; 495 409 496 496 - return crypto_transfer_aead_request_to_engine(drv_ctx->engine, req); 410 410 + rq_ctx->op = XILINX_AES_ENCRYPT; 411 411 + err = zynqmp_fallback_check(tfm_ctx, req); 412 412 + if (err && tfm_ctx->keysrc != ZYNQMP_AES_KUP_KEY) 413 413 + return -EOPNOTSUPP; 414 414 + 415 415 + if (err) 416 416 + return xilinx_aes_fallback_crypt(req, true); 417 417 + 418 418 + return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req); 419 419 + } 420 420 + 421 421 + static int versal_aes_aead_encrypt(struct aead_request *req) 422 422 + { 423 423 + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 424 424 + struct crypto_aead *aead = crypto_aead_reqtfm(req); 425 425 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 426 426 + struct aead_alg *alg = crypto_aead_alg(aead); 427 427 + struct xilinx_aead_alg *drv_ctx; 428 428 + int err; 429 429 + 430 430 + drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); 431 431 + rq_ctx->op = XILINX_AES_ENCRYPT; 432 432 + if (tfm_ctx->keysrc >= VERSAL_AES_USER_KEY_0 && 433 433 + tfm_ctx->keysrc <= VERSAL_AES_USER_KEY_7 && 434 434 + tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info)) 435 435 + return -EINVAL; 436 436 + err = versal_fallback_check(tfm_ctx, req); 437 437 + if (err && (tfm_ctx->keysrc < VERSAL_AES_USER_KEY_0 || 438 438 + tfm_ctx->keysrc > VERSAL_AES_USER_KEY_7)) 439 439 + return -EOPNOTSUPP; 440 440 + if (err) 441 441 + return xilinx_aes_fallback_crypt(req, true); 442 442 + 443 443 + return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req); 497 444 } 498 445 499 446 static int zynqmp_aes_aead_decrypt(struct aead_request *req) 500 447 { 501 501 - struct zynqmp_aead_drv_ctx *drv_ctx; 448 448 + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 502 449 struct crypto_aead *aead = crypto_aead_reqtfm(req); 450 450 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 503 451 struct aead_alg *alg = crypto_aead_alg(aead); 504 504 - struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req); 452 452 + struct xilinx_aead_alg *drv_ctx; 453 453 + int err; 505 454 506 506 - rq_ctx->op = ZYNQMP_AES_DECRYPT; 507 507 - drv_ctx = container_of(alg, struct zynqmp_aead_drv_ctx, alg.aead.base); 455 455 + rq_ctx->op = XILINX_AES_DECRYPT; 456 456 + drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); 457 457 + if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY && 458 458 + tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info)) 459 459 + return -EINVAL; 460 460 + err = zynqmp_fallback_check(tfm_ctx, req); 461 461 + if (err && tfm_ctx->keysrc != ZYNQMP_AES_KUP_KEY) 462 462 + return -EOPNOTSUPP; 463 463 + if (err) 464 464 + return xilinx_aes_fallback_crypt(req, false); 508 465 509 509 - return crypto_transfer_aead_request_to_engine(drv_ctx->engine, req); 466 466 + return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req); 510 467 } 511 468 512 512 - static int zynqmp_aes_aead_init(struct crypto_aead *aead) 469 469 + static int xilinx_paes_aead_init(struct crypto_aead *aead) 513 470 { 514 471 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 515 515 - struct zynqmp_aead_tfm_ctx *tfm_ctx = 516 516 - (struct zynqmp_aead_tfm_ctx *)crypto_tfm_ctx(tfm); 517 517 - struct zynqmp_aead_drv_ctx *drv_ctx; 472 472 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 473 473 + struct xilinx_aead_alg *drv_alg; 518 474 struct aead_alg *alg = crypto_aead_alg(aead); 519 475 520 520 - drv_ctx = container_of(alg, struct zynqmp_aead_drv_ctx, alg.aead.base); 521 521 - tfm_ctx->dev = drv_ctx->dev; 476 476 + drv_alg = container_of(alg, struct xilinx_aead_alg, aead.base); 477 477 + tfm_ctx->dev = drv_alg->aead_dev->dev; 478 478 + tfm_ctx->keylen = 0; 479 479 + tfm_ctx->key = NULL; 480 480 + tfm_ctx->fbk_cipher = NULL; 481 481 + crypto_aead_set_reqsize(aead, sizeof(struct xilinx_aead_req_ctx)); 522 482 523 523 - tfm_ctx->fbk_cipher = crypto_alloc_aead(drv_ctx->alg.aead.base.base.cra_name, 483 483 + return 0; 484 484 + } 485 485 + 486 486 + static int versal_aes_aead_decrypt(struct aead_request *req) 487 487 + { 488 488 + struct xilinx_aead_req_ctx *rq_ctx = aead_request_ctx(req); 489 489 + struct crypto_aead *aead = crypto_aead_reqtfm(req); 490 490 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(aead); 491 491 + struct aead_alg *alg = crypto_aead_alg(aead); 492 492 + struct xilinx_aead_alg *drv_ctx; 493 493 + int err; 494 494 + 495 495 + drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); 496 496 + rq_ctx->op = XILINX_AES_DECRYPT; 497 497 + if (tfm_ctx->keysrc >= VERSAL_AES_USER_KEY_0 && 498 498 + tfm_ctx->keysrc <= VERSAL_AES_USER_KEY_7 && 499 499 + tfm_ctx->keylen == sizeof(struct xilinx_hwkey_info)) 500 500 + return -EINVAL; 501 501 + 502 502 + err = versal_fallback_check(tfm_ctx, req); 503 503 + if (err && 504 504 + (tfm_ctx->keysrc < VERSAL_AES_USER_KEY_0 || 505 505 + tfm_ctx->keysrc > VERSAL_AES_USER_KEY_7)) 506 506 + return -EOPNOTSUPP; 507 507 + if (err) 508 508 + return xilinx_aes_fallback_crypt(req, false); 509 509 + 510 510 + return crypto_transfer_aead_request_to_engine(drv_ctx->aead_dev->engine, req); 511 511 + } 512 512 + 513 513 + static int xilinx_aes_aead_init(struct crypto_aead *aead) 514 514 + { 515 515 + struct crypto_tfm *tfm = crypto_aead_tfm(aead); 516 516 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 517 517 + struct xilinx_aead_alg *drv_ctx; 518 518 + struct aead_alg *alg = crypto_aead_alg(aead); 519 519 + 520 520 + drv_ctx = container_of(alg, struct xilinx_aead_alg, aead.base); 521 521 + tfm_ctx->dev = drv_ctx->aead_dev->dev; 522 522 + tfm_ctx->keylen = 0; 523 523 + 524 524 + tfm_ctx->fbk_cipher = crypto_alloc_aead(drv_ctx->aead.base.base.cra_name, 524 525 0, 525 526 CRYPTO_ALG_NEED_FALLBACK); 526 527 527 528 if (IS_ERR(tfm_ctx->fbk_cipher)) { 528 528 - pr_err("%s() Error: failed to allocate fallback for %s\n", 529 529 - __func__, drv_ctx->alg.aead.base.base.cra_name); 529 529 + dev_err(tfm_ctx->dev, "failed to allocate fallback for %s\n", 530 530 + drv_ctx->aead.base.base.cra_name); 530 531 return PTR_ERR(tfm_ctx->fbk_cipher); 531 532 } 532 532 - 533 533 + tfm_ctx->key = kmalloc(AES_KEYSIZE_256, GFP_KERNEL); 534 534 + if (!tfm_ctx->key) { 535 535 + crypto_free_aead(tfm_ctx->fbk_cipher); 536 536 + return -ENOMEM; 537 537 + } 538 538 + tfm_ctx->key_dma_addr = dma_map_single(tfm_ctx->dev, tfm_ctx->key, 539 539 + AES_KEYSIZE_256, 540 540 + DMA_TO_DEVICE); 541 541 + if (unlikely(dma_mapping_error(tfm_ctx->dev, tfm_ctx->key_dma_addr))) { 542 542 + kfree(tfm_ctx->key); 543 543 + crypto_free_aead(tfm_ctx->fbk_cipher); 544 544 + tfm_ctx->fbk_cipher = NULL; 545 545 + return -ENOMEM; 546 546 + } 533 547 crypto_aead_set_reqsize(aead, 534 534 - max(sizeof(struct zynqmp_aead_req_ctx), 548 548 + max(sizeof(struct xilinx_aead_req_ctx), 535 549 sizeof(struct aead_request) + 536 550 crypto_aead_reqsize(tfm_ctx->fbk_cipher))); 537 551 return 0; 538 552 } 539 553 540 540 - static void zynqmp_aes_aead_exit(struct crypto_aead *aead) 554 554 + static void xilinx_paes_aead_exit(struct crypto_aead *aead) 541 555 { 542 556 struct crypto_tfm *tfm = crypto_aead_tfm(aead); 543 543 - struct zynqmp_aead_tfm_ctx *tfm_ctx = 544 544 - (struct zynqmp_aead_tfm_ctx *)crypto_tfm_ctx(tfm); 557 557 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 545 558 559 559 + memzero_explicit(tfm_ctx, sizeof(struct xilinx_aead_tfm_ctx)); 560 560 + } 561 561 + 562 562 + static void xilinx_aes_aead_exit(struct crypto_aead *aead) 563 563 + { 564 564 + struct crypto_tfm *tfm = crypto_aead_tfm(aead); 565 565 + struct xilinx_aead_tfm_ctx *tfm_ctx = crypto_tfm_ctx(tfm); 566 566 + 567 567 + dma_unmap_single(tfm_ctx->dev, tfm_ctx->key_dma_addr, AES_KEYSIZE_256, DMA_TO_DEVICE); 568 568 + kfree(tfm_ctx->key); 546 569 if (tfm_ctx->fbk_cipher) { 547 570 crypto_free_aead(tfm_ctx->fbk_cipher); 548 571 tfm_ctx->fbk_cipher = NULL; 549 572 } 550 550 - memzero_explicit(tfm_ctx, sizeof(struct zynqmp_aead_tfm_ctx)); 573 573 + memzero_explicit(tfm_ctx, sizeof(struct xilinx_aead_tfm_ctx)); 551 574 } 552 575 553 553 - static struct zynqmp_aead_drv_ctx aes_drv_ctx = { 554 554 - .alg.aead.base = { 555 555 - .setkey = zynqmp_aes_aead_setkey, 556 556 - .setauthsize = zynqmp_aes_aead_setauthsize, 557 557 - .encrypt = zynqmp_aes_aead_encrypt, 558 558 - .decrypt = zynqmp_aes_aead_decrypt, 559 559 - .init = zynqmp_aes_aead_init, 560 560 - .exit = zynqmp_aes_aead_exit, 561 561 - .ivsize = GCM_AES_IV_SIZE, 562 562 - .maxauthsize = ZYNQMP_AES_AUTH_SIZE, 563 563 - .base = { 564 564 - .cra_name = "gcm(aes)", 565 565 - .cra_driver_name = "xilinx-zynqmp-aes-gcm", 566 566 - .cra_priority = 200, 567 567 - .cra_flags = CRYPTO_ALG_TYPE_AEAD | 568 568 - CRYPTO_ALG_ASYNC | 569 569 - CRYPTO_ALG_ALLOCATES_MEMORY | 570 570 - CRYPTO_ALG_KERN_DRIVER_ONLY | 571 571 - CRYPTO_ALG_NEED_FALLBACK, 572 572 - .cra_blocksize = ZYNQMP_AES_BLK_SIZE, 573 573 - .cra_ctxsize = sizeof(struct zynqmp_aead_tfm_ctx), 574 574 - .cra_module = THIS_MODULE, 575 575 - } 576 576 + static struct xilinx_aead_alg zynqmp_aes_algs[] = { 577 577 + { 578 578 + .aes_aead_cipher = zynqmp_aes_aead_cipher, 579 579 + .aead.base = { 580 580 + .setkey = zynqmp_aes_aead_setkey, 581 581 + .setauthsize = xilinx_aes_aead_setauthsize, 582 582 + .encrypt = zynqmp_aes_aead_encrypt, 583 583 + .decrypt = zynqmp_aes_aead_decrypt, 584 584 + .init = xilinx_aes_aead_init, 585 585 + .exit = xilinx_aes_aead_exit, 586 586 + .ivsize = GCM_AES_IV_SIZE, 587 587 + .maxauthsize = XILINX_AES_AUTH_SIZE, 588 588 + .base = { 589 589 + .cra_name = "gcm(aes)", 590 590 + .cra_driver_name = "xilinx-zynqmp-aes-gcm", 591 591 + .cra_priority = 200, 592 592 + .cra_flags = CRYPTO_ALG_TYPE_AEAD | 593 593 + CRYPTO_ALG_ASYNC | 594 594 + CRYPTO_ALG_ALLOCATES_MEMORY | 595 595 + CRYPTO_ALG_KERN_DRIVER_ONLY | 596 596 + CRYPTO_ALG_NEED_FALLBACK, 597 597 + .cra_blocksize = XILINX_AES_BLK_SIZE, 598 598 + .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx), 599 599 + .cra_module = THIS_MODULE, 600 600 + } 601 601 + }, 602 602 + .aead.op = { 603 603 + .do_one_request = xilinx_handle_aes_req, 604 604 + }, 605 605 + .dma_bit_mask = ZYNQMP_DMA_BIT_MASK, 576 606 }, 577 577 - .alg.aead.op = { 578 578 - .do_one_request = zynqmp_handle_aes_req, 607 607 + { 608 608 + .aes_aead_cipher = zynqmp_aes_aead_cipher, 609 609 + .aead.base = { 610 610 + .setkey = zynqmp_paes_aead_setkey, 611 611 + .setauthsize = xilinx_aes_aead_setauthsize, 612 612 + .encrypt = zynqmp_aes_aead_encrypt, 613 613 + .decrypt = zynqmp_aes_aead_decrypt, 614 614 + .init = xilinx_paes_aead_init, 615 615 + .exit = xilinx_paes_aead_exit, 616 616 + .ivsize = GCM_AES_IV_SIZE, 617 617 + .maxauthsize = XILINX_AES_AUTH_SIZE, 618 618 + .base = { 619 619 + .cra_name = "gcm(paes)", 620 620 + .cra_driver_name = "xilinx-zynqmp-paes-gcm", 621 621 + .cra_priority = 200, 622 622 + .cra_flags = CRYPTO_ALG_TYPE_AEAD | 623 623 + CRYPTO_ALG_ASYNC | 624 624 + CRYPTO_ALG_ALLOCATES_MEMORY | 625 625 + CRYPTO_ALG_KERN_DRIVER_ONLY, 626 626 + .cra_blocksize = XILINX_AES_BLK_SIZE, 627 627 + .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx), 628 628 + .cra_module = THIS_MODULE, 629 629 + } 630 630 + }, 631 631 + .aead.op = { 632 632 + .do_one_request = xilinx_handle_aes_req, 633 633 + }, 634 634 + .dma_bit_mask = ZYNQMP_DMA_BIT_MASK, 579 635 }, 636 636 + { /* sentinel */ } 580 637 }; 581 638 582 582 - static int zynqmp_aes_aead_probe(struct platform_device *pdev) 639 639 + static struct xilinx_aead_alg versal_aes_algs[] = { 640 640 + { 641 641 + .aes_aead_cipher = versal_aes_aead_cipher, 642 642 + .aead.base = { 643 643 + .setkey = versal_aes_aead_setkey, 644 644 + .setauthsize = xilinx_aes_aead_setauthsize, 645 645 + .encrypt = versal_aes_aead_encrypt, 646 646 + .decrypt = versal_aes_aead_decrypt, 647 647 + .init = xilinx_aes_aead_init, 648 648 + .exit = xilinx_aes_aead_exit, 649 649 + .ivsize = GCM_AES_IV_SIZE, 650 650 + .maxauthsize = XILINX_AES_AUTH_SIZE, 651 651 + .base = { 652 652 + .cra_name = "gcm(aes)", 653 653 + .cra_driver_name = "versal-aes-gcm", 654 654 + .cra_priority = 300, 655 655 + .cra_flags = CRYPTO_ALG_TYPE_AEAD | 656 656 + CRYPTO_ALG_ASYNC | 657 657 + CRYPTO_ALG_ALLOCATES_MEMORY | 658 658 + CRYPTO_ALG_KERN_DRIVER_ONLY | 659 659 + CRYPTO_ALG_NEED_FALLBACK, 660 660 + .cra_blocksize = XILINX_AES_BLK_SIZE, 661 661 + .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx), 662 662 + .cra_module = THIS_MODULE, 663 663 + } 664 664 + }, 665 665 + .aead.op = { 666 666 + .do_one_request = xilinx_handle_aes_req, 667 667 + }, 668 668 + .dma_bit_mask = VERSAL_DMA_BIT_MASK, 669 669 + }, 670 670 + { 671 671 + .aes_aead_cipher = versal_aes_aead_cipher, 672 672 + .aead.base = { 673 673 + .setkey = versal_paes_aead_setkey, 674 674 + .setauthsize = xilinx_aes_aead_setauthsize, 675 675 + .encrypt = versal_aes_aead_encrypt, 676 676 + .decrypt = versal_aes_aead_decrypt, 677 677 + .init = xilinx_paes_aead_init, 678 678 + .exit = xilinx_paes_aead_exit, 679 679 + .ivsize = GCM_AES_IV_SIZE, 680 680 + .maxauthsize = XILINX_AES_AUTH_SIZE, 681 681 + .base = { 682 682 + .cra_name = "gcm(paes)", 683 683 + .cra_driver_name = "versal-paes-gcm", 684 684 + .cra_priority = 300, 685 685 + .cra_flags = CRYPTO_ALG_TYPE_AEAD | 686 686 + CRYPTO_ALG_ASYNC | 687 687 + CRYPTO_ALG_ALLOCATES_MEMORY | 688 688 + CRYPTO_ALG_KERN_DRIVER_ONLY, 689 689 + .cra_blocksize = XILINX_AES_BLK_SIZE, 690 690 + .cra_ctxsize = sizeof(struct xilinx_aead_tfm_ctx), 691 691 + .cra_module = THIS_MODULE, 692 692 + } 693 693 + }, 694 694 + .aead.op = { 695 695 + .do_one_request = xilinx_handle_aes_req, 696 696 + }, 697 697 + .dma_bit_mask = VERSAL_DMA_BIT_MASK, 698 698 + }, 699 699 + { /* sentinel */ } 700 700 + }; 701 701 + 702 702 + static struct xlnx_feature aes_feature_map[] = { 703 703 + { 704 704 + .family = PM_ZYNQMP_FAMILY_CODE, 705 705 + .feature_id = PM_SECURE_AES, 706 706 + .data = zynqmp_aes_algs, 707 707 + }, 708 708 + { 709 709 + .family = PM_VERSAL_FAMILY_CODE, 710 710 + .feature_id = XSECURE_API_AES_OP_INIT, 711 711 + .data = versal_aes_algs, 712 712 + }, 713 713 + { /* sentinel */ } 714 714 + }; 715 715 + 716 716 + static int xilinx_aes_aead_probe(struct platform_device *pdev) 583 717 { 718 718 + struct xilinx_aead_alg *aead_algs; 584 719 struct device *dev = &pdev->dev; 585 720 int err; 721 721 + int i; 722 722 + 723 723 + /* Verify the hardware is present */ 724 724 + aead_algs = xlnx_get_crypto_dev_data(aes_feature_map); 725 725 + if (IS_ERR(aead_algs)) { 726 726 + dev_err(dev, "AES is not supported on the platform\n"); 727 727 + return PTR_ERR(aead_algs); 728 728 + } 586 729 587 730 /* ZynqMP AES driver supports only one instance */ 588 588 - if (!aes_drv_ctx.dev) 589 589 - aes_drv_ctx.dev = dev; 590 590 - else 731 731 + if (aead_dev) 591 732 return -ENODEV; 592 733 593 593 - err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(ZYNQMP_DMA_BIT_MASK)); 734 734 + aead_dev = devm_kzalloc(dev, sizeof(*aead_dev), GFP_KERNEL); 735 735 + if (!aead_dev) 736 736 + return -ENOMEM; 737 737 + aead_dev->dev = dev; 738 738 + aead_dev->aead_algs = aead_algs; 739 739 + platform_set_drvdata(pdev, aead_dev); 740 740 + err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(aead_algs[0].dma_bit_mask)); 594 741 if (err < 0) { 595 742 dev_err(dev, "No usable DMA configuration\n"); 596 743 return err; 597 744 } 598 745 599 599 - aes_drv_ctx.engine = crypto_engine_alloc_init(dev, 1); 600 600 - if (!aes_drv_ctx.engine) { 746 746 + aead_dev->engine = crypto_engine_alloc_init(dev, 1); 747 747 + if (!aead_dev->engine) { 601 748 dev_err(dev, "Cannot alloc AES engine\n"); 602 602 - err = -ENOMEM; 603 603 - goto err_engine; 749 749 + return -ENOMEM; 604 750 } 605 751 606 606 - err = crypto_engine_start(aes_drv_ctx.engine); 752 752 + err = crypto_engine_start(aead_dev->engine); 607 753 if (err) { 608 754 dev_err(dev, "Cannot start AES engine\n"); 609 609 - goto err_engine; 755 755 + goto err_engine_start; 610 756 } 611 757 612 612 - err = crypto_engine_register_aead(&aes_drv_ctx.alg.aead); 613 613 - if (err < 0) { 614 614 - dev_err(dev, "Failed to register AEAD alg.\n"); 615 615 - goto err_aead; 758 758 + for (i = 0; aead_dev->aead_algs[i].dma_bit_mask; i++) { 759 759 + aead_dev->aead_algs[i].aead_dev = aead_dev; 760 760 + err = crypto_engine_register_aead(&aead_dev->aead_algs[i].aead); 761 761 + if (err < 0) { 762 762 + dev_err(dev, "Failed to register AEAD alg %d.\n", i); 763 763 + goto err_alg_register; 764 764 + } 616 765 } 766 766 + 617 767 return 0; 618 768 619 619 - err_aead: 620 620 - crypto_engine_unregister_aead(&aes_drv_ctx.alg.aead); 621 621 - 622 622 - err_engine: 623 623 - if (aes_drv_ctx.engine) 624 624 - crypto_engine_exit(aes_drv_ctx.engine); 769 769 + err_alg_register: 770 770 + while (i > 0) 771 771 + crypto_engine_unregister_aead(&aead_dev->aead_algs[--i].aead); 772 772 + err_engine_start: 773 773 + crypto_engine_exit(aead_dev->engine); 625 774 626 775 return err; 627 776 } 628 777 629 629 - static void zynqmp_aes_aead_remove(struct platform_device *pdev) 778 778 + static void xilinx_aes_aead_remove(struct platform_device *pdev) 630 779 { 631 631 - crypto_engine_exit(aes_drv_ctx.engine); 632 632 - crypto_engine_unregister_aead(&aes_drv_ctx.alg.aead); 780 780 + aead_dev = platform_get_drvdata(pdev); 781 781 + crypto_engine_exit(aead_dev->engine); 782 782 + for (int i = 0; aead_dev->aead_algs[i].dma_bit_mask; i++) 783 783 + crypto_engine_unregister_aead(&aead_dev->aead_algs[i].aead); 784 784 + 785 785 + aead_dev = NULL; 633 786 } 634 787 635 635 - static const struct of_device_id zynqmp_aes_dt_ids[] = { 636 636 - { .compatible = "xlnx,zynqmp-aes" }, 637 637 - { /* sentinel */ } 638 638 - }; 639 639 - MODULE_DEVICE_TABLE(of, zynqmp_aes_dt_ids); 640 640 - 641 641 - static struct platform_driver zynqmp_aes_driver = { 642 642 - .probe = zynqmp_aes_aead_probe, 643 643 - .remove = zynqmp_aes_aead_remove, 788 788 + static struct platform_driver xilinx_aes_driver = { 789 789 + .probe = xilinx_aes_aead_probe, 790 790 + .remove = xilinx_aes_aead_remove, 644 791 .driver = { 645 792 .name = "zynqmp-aes", 646 646 - .of_match_table = zynqmp_aes_dt_ids, 647 793 }, 648 794 }; 649 795 650 650 - module_platform_driver(zynqmp_aes_driver); 651 651 - MODULE_DESCRIPTION("Xilinx ZynqMP AES Driver"); 796 796 + static struct platform_device *platform_dev; 797 797 + 798 798 + static int __init aes_driver_init(void) 799 799 + { 800 800 + int ret; 801 801 + 802 802 + ret = platform_driver_register(&xilinx_aes_driver); 803 803 + if (ret) 804 804 + return ret; 805 805 + 806 806 + platform_dev = platform_device_register_simple(xilinx_aes_driver.driver.name, 807 807 + 0, NULL, 0); 808 808 + if (IS_ERR(platform_dev)) { 809 809 + ret = PTR_ERR(platform_dev); 810 810 + platform_driver_unregister(&xilinx_aes_driver); 811 811 + } 812 812 + 813 813 + return ret; 814 814 + } 815 815 + 816 816 + static void __exit aes_driver_exit(void) 817 817 + { 818 818 + platform_device_unregister(platform_dev); 819 819 + platform_driver_unregister(&xilinx_aes_driver); 820 820 + } 821 821 + 822 822 + module_init(aes_driver_init); 823 823 + module_exit(aes_driver_exit); 824 824 + MODULE_DESCRIPTION("zynqmp aes-gcm hardware acceleration support."); 652 825 MODULE_LICENSE("GPL");

+1 -1

drivers/firmware/xilinx/Makefile

reviewed

··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 2 # Makefile for Xilinx firmwares 3 3 4 4 - obj-$(CONFIG_ZYNQMP_FIRMWARE) += zynqmp.o zynqmp-ufs.o 4 4 + obj-$(CONFIG_ZYNQMP_FIRMWARE) += zynqmp.o zynqmp-ufs.o zynqmp-crypto.o 5 5 obj-$(CONFIG_ZYNQMP_FIRMWARE_DEBUG) += zynqmp-debug.o

+238

drivers/firmware/xilinx/zynqmp-crypto.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* 3 3 + * Firmware layer for XilSecure APIs. 4 4 + * 5 5 + * Copyright (C) 2014-2022 Xilinx, Inc. 6 6 + * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. 7 7 + */ 8 8 + 9 9 + #include <linux/firmware/xlnx-zynqmp.h> 10 10 + #include <linux/module.h> 11 11 + 12 12 + /** 13 13 + * zynqmp_pm_aes_engine - Access AES hardware to encrypt/decrypt the data using 14 14 + * AES-GCM core. 15 15 + * @address: Address of the AesParams structure. 16 16 + * @out: Returned output value 17 17 + * 18 18 + * Return: Returns status, either success or error code. 19 19 + */ 20 20 + int zynqmp_pm_aes_engine(const u64 address, u32 *out) 21 21 + { 22 22 + u32 ret_payload[PAYLOAD_ARG_CNT]; 23 23 + int ret; 24 24 + 25 25 + if (!out) 26 26 + return -EINVAL; 27 27 + 28 28 + ret = zynqmp_pm_invoke_fn(PM_SECURE_AES, ret_payload, 2, upper_32_bits(address), 29 29 + lower_32_bits(address)); 30 30 + *out = ret_payload[1]; 31 31 + 32 32 + return ret; 33 33 + } 34 34 + EXPORT_SYMBOL_GPL(zynqmp_pm_aes_engine); 35 35 + 36 36 + /** 37 37 + * zynqmp_pm_sha_hash - Access the SHA engine to calculate the hash 38 38 + * @address: Address of the data/ Address of output buffer where 39 39 + * hash should be stored. 40 40 + * @size: Size of the data. 41 41 + * @flags: 42 42 + * BIT(0) - for initializing csudma driver and SHA3(Here address 43 43 + * and size inputs can be NULL). 44 44 + * BIT(1) - to call Sha3_Update API which can be called multiple 45 45 + * times when data is not contiguous. 46 46 + * BIT(2) - to get final hash of the whole updated data. 47 47 + * Hash will be overwritten at provided address with 48 48 + * 48 bytes. 49 49 + * 50 50 + * Return: Returns status, either success or error code. 51 51 + */ 52 52 + int zynqmp_pm_sha_hash(const u64 address, const u32 size, const u32 flags) 53 53 + { 54 54 + u32 lower_addr = lower_32_bits(address); 55 55 + u32 upper_addr = upper_32_bits(address); 56 56 + 57 57 + return zynqmp_pm_invoke_fn(PM_SECURE_SHA, NULL, 4, upper_addr, lower_addr, size, flags); 58 58 + } 59 59 + EXPORT_SYMBOL_GPL(zynqmp_pm_sha_hash); 60 60 + 61 61 + /** 62 62 + * xlnx_get_crypto_dev_data() - Get crypto dev data of platform 63 63 + * @feature_map: List of available feature map of all platform 64 64 + * 65 65 + * Return: Returns crypto dev data, either address crypto dev or ERR PTR 66 66 + */ 67 67 + void *xlnx_get_crypto_dev_data(struct xlnx_feature *feature_map) 68 68 + { 69 69 + struct xlnx_feature *feature; 70 70 + u32 pm_family_code; 71 71 + int ret; 72 72 + 73 73 + /* Get the Family code and sub family code of platform */ 74 74 + ret = zynqmp_pm_get_family_info(&pm_family_code); 75 75 + if (ret < 0) 76 76 + return ERR_PTR(ret); 77 77 + 78 78 + feature = feature_map; 79 79 + for (; feature->family; feature++) { 80 80 + if (feature->family == pm_family_code) { 81 81 + ret = zynqmp_pm_feature(feature->feature_id); 82 82 + if (ret < 0) 83 83 + return ERR_PTR(ret); 84 84 + 85 85 + return feature->data; 86 86 + } 87 87 + } 88 88 + return ERR_PTR(-ENODEV); 89 89 + } 90 90 + EXPORT_SYMBOL_GPL(xlnx_get_crypto_dev_data); 91 91 + 92 92 + /** 93 93 + * versal_pm_aes_key_write - Write AES key registers 94 94 + * @keylen: Size of the input key to be written 95 95 + * @keysrc: Key Source to be selected to which provided 96 96 + * key should be updated 97 97 + * @keyaddr: Address of a buffer which should contain the key 98 98 + * to be written 99 99 + * 100 100 + * This function provides support to write AES volatile user keys. 101 101 + * 102 102 + * Return: Returns status, either success or error+reason 103 103 + */ 104 104 + int versal_pm_aes_key_write(const u32 keylen, 105 105 + const u32 keysrc, const u64 keyaddr) 106 106 + { 107 107 + return zynqmp_pm_invoke_fn(XSECURE_API_AES_WRITE_KEY, NULL, 4, 108 108 + keylen, keysrc, 109 109 + lower_32_bits(keyaddr), 110 110 + upper_32_bits(keyaddr)); 111 111 + } 112 112 + EXPORT_SYMBOL_GPL(versal_pm_aes_key_write); 113 113 + 114 114 + /** 115 115 + * versal_pm_aes_key_zero - Zeroise AES User key registers 116 116 + * @keysrc: Key Source to be selected to which provided 117 117 + * key should be updated 118 118 + * 119 119 + * This function provides support to zeroise AES volatile user keys. 120 120 + * 121 121 + * Return: Returns status, either success or error+reason 122 122 + */ 123 123 + int versal_pm_aes_key_zero(const u32 keysrc) 124 124 + { 125 125 + return zynqmp_pm_invoke_fn(XSECURE_API_AES_KEY_ZERO, NULL, 1, keysrc); 126 126 + } 127 127 + EXPORT_SYMBOL_GPL(versal_pm_aes_key_zero); 128 128 + 129 129 + /** 130 130 + * versal_pm_aes_op_init - Init AES operation 131 131 + * @hw_req: AES op init structure address 132 132 + * 133 133 + * This function provides support to init AES operation. 134 134 + * 135 135 + * Return: Returns status, either success or error+reason 136 136 + */ 137 137 + int versal_pm_aes_op_init(const u64 hw_req) 138 138 + { 139 139 + return zynqmp_pm_invoke_fn(XSECURE_API_AES_OP_INIT, NULL, 2, 140 140 + lower_32_bits(hw_req), 141 141 + upper_32_bits(hw_req)); 142 142 + } 143 143 + EXPORT_SYMBOL_GPL(versal_pm_aes_op_init); 144 144 + 145 145 + /** 146 146 + * versal_pm_aes_update_aad - AES update aad 147 147 + * @aad_addr: AES aad address 148 148 + * @aad_len: AES aad data length 149 149 + * 150 150 + * This function provides support to update AAD data. 151 151 + * 152 152 + * Return: Returns status, either success or error+reason 153 153 + */ 154 154 + int versal_pm_aes_update_aad(const u64 aad_addr, const u32 aad_len) 155 155 + { 156 156 + return zynqmp_pm_invoke_fn(XSECURE_API_AES_UPDATE_AAD, NULL, 3, 157 157 + lower_32_bits(aad_addr), 158 158 + upper_32_bits(aad_addr), 159 159 + aad_len); 160 160 + } 161 161 + EXPORT_SYMBOL_GPL(versal_pm_aes_update_aad); 162 162 + 163 163 + /** 164 164 + * versal_pm_aes_enc_update - Access AES hardware to encrypt the data using 165 165 + * AES-GCM core. 166 166 + * @in_params: Address of the AesParams structure 167 167 + * @in_addr: Address of input buffer 168 168 + * 169 169 + * Return: Returns status, either success or error code. 170 170 + */ 171 171 + int versal_pm_aes_enc_update(const u64 in_params, const u64 in_addr) 172 172 + { 173 173 + return zynqmp_pm_invoke_fn(XSECURE_API_AES_ENCRYPT_UPDATE, NULL, 4, 174 174 + lower_32_bits(in_params), 175 175 + upper_32_bits(in_params), 176 176 + lower_32_bits(in_addr), 177 177 + upper_32_bits(in_addr)); 178 178 + } 179 179 + EXPORT_SYMBOL_GPL(versal_pm_aes_enc_update); 180 180 + 181 181 + /** 182 182 + * versal_pm_aes_enc_final - Access AES hardware to store the GCM tag 183 183 + * @gcm_addr: Address of the gcm tag 184 184 + * 185 185 + * Return: Returns status, either success or error code. 186 186 + */ 187 187 + int versal_pm_aes_enc_final(const u64 gcm_addr) 188 188 + { 189 189 + return zynqmp_pm_invoke_fn(XSECURE_API_AES_ENCRYPT_FINAL, NULL, 2, 190 190 + lower_32_bits(gcm_addr), 191 191 + upper_32_bits(gcm_addr)); 192 192 + } 193 193 + EXPORT_SYMBOL_GPL(versal_pm_aes_enc_final); 194 194 + 195 195 + /** 196 196 + * versal_pm_aes_dec_update - Access AES hardware to decrypt the data using 197 197 + * AES-GCM core. 198 198 + * @in_params: Address of the AesParams structure 199 199 + * @in_addr: Address of input buffer 200 200 + * 201 201 + * Return: Returns status, either success or error code. 202 202 + */ 203 203 + int versal_pm_aes_dec_update(const u64 in_params, const u64 in_addr) 204 204 + { 205 205 + return zynqmp_pm_invoke_fn(XSECURE_API_AES_DECRYPT_UPDATE, NULL, 4, 206 206 + lower_32_bits(in_params), 207 207 + upper_32_bits(in_params), 208 208 + lower_32_bits(in_addr), 209 209 + upper_32_bits(in_addr)); 210 210 + } 211 211 + EXPORT_SYMBOL_GPL(versal_pm_aes_dec_update); 212 212 + 213 213 + /** 214 214 + * versal_pm_aes_dec_final - Access AES hardware to get the GCM tag 215 215 + * @gcm_addr: Address of the gcm tag 216 216 + * 217 217 + * Return: Returns status, either success or error code. 218 218 + */ 219 219 + int versal_pm_aes_dec_final(const u64 gcm_addr) 220 220 + { 221 221 + return zynqmp_pm_invoke_fn(XSECURE_API_AES_DECRYPT_FINAL, NULL, 2, 222 222 + lower_32_bits(gcm_addr), 223 223 + upper_32_bits(gcm_addr)); 224 224 + } 225 225 + EXPORT_SYMBOL_GPL(versal_pm_aes_dec_final); 226 226 + 227 227 + /** 228 228 + * versal_pm_aes_init - Init AES block 229 229 + * 230 230 + * This function initialise AES block. 231 231 + * 232 232 + * Return: Returns status, either success or error+reason 233 233 + */ 234 234 + int versal_pm_aes_init(void) 235 235 + { 236 236 + return zynqmp_pm_invoke_fn(XSECURE_API_AES_INIT, NULL, 0); 237 237 + } 238 238 + EXPORT_SYMBOL_GPL(versal_pm_aes_init);

-49

drivers/firmware/xilinx/zynqmp.c

reviewed

··· 1522 1522 EXPORT_SYMBOL_GPL(zynqmp_pm_load_pdi); 1523 1523 1524 1524 /** 1525 1525 - * zynqmp_pm_aes_engine - Access AES hardware to encrypt/decrypt the data using 1526 1526 - * AES-GCM core. 1527 1527 - * @address: Address of the AesParams structure. 1528 1528 - * @out: Returned output value 1529 1529 - * 1530 1530 - * Return: Returns status, either success or error code. 1531 1531 - */ 1532 1532 - int zynqmp_pm_aes_engine(const u64 address, u32 *out) 1533 1533 - { 1534 1534 - u32 ret_payload[PAYLOAD_ARG_CNT]; 1535 1535 - int ret; 1536 1536 - 1537 1537 - if (!out) 1538 1538 - return -EINVAL; 1539 1539 - 1540 1540 - ret = zynqmp_pm_invoke_fn(PM_SECURE_AES, ret_payload, 2, upper_32_bits(address), 1541 1541 - lower_32_bits(address)); 1542 1542 - *out = ret_payload[1]; 1543 1543 - 1544 1544 - return ret; 1545 1545 - } 1546 1546 - EXPORT_SYMBOL_GPL(zynqmp_pm_aes_engine); 1547 1547 - 1548 1548 - /** 1549 1525 * zynqmp_pm_efuse_access - Provides access to efuse memory. 1550 1526 * @address: Address of the efuse params structure 1551 1527 * @out: Returned output value ··· 1544 1568 return ret; 1545 1569 } 1546 1570 EXPORT_SYMBOL_GPL(zynqmp_pm_efuse_access); 1547 1547 - 1548 1548 - /** 1549 1549 - * zynqmp_pm_sha_hash - Access the SHA engine to calculate the hash 1550 1550 - * @address: Address of the data/ Address of output buffer where 1551 1551 - * hash should be stored. 1552 1552 - * @size: Size of the data. 1553 1553 - * @flags: 1554 1554 - * BIT(0) - for initializing csudma driver and SHA3(Here address 1555 1555 - * and size inputs can be NULL). 1556 1556 - * BIT(1) - to call Sha3_Update API which can be called multiple 1557 1557 - * times when data is not contiguous. 1558 1558 - * BIT(2) - to get final hash of the whole updated data. 1559 1559 - * Hash will be overwritten at provided address with 1560 1560 - * 48 bytes. 1561 1561 - * 1562 1562 - * Return: Returns status, either success or error code. 1563 1563 - */ 1564 1564 - int zynqmp_pm_sha_hash(const u64 address, const u32 size, const u32 flags) 1565 1565 - { 1566 1566 - u32 lower_addr = lower_32_bits(address); 1567 1567 - u32 upper_addr = upper_32_bits(address); 1568 1568 - 1569 1569 - return zynqmp_pm_invoke_fn(PM_SECURE_SHA, NULL, 4, upper_addr, lower_addr, size, flags); 1570 1570 - } 1571 1571 - EXPORT_SYMBOL_GPL(zynqmp_pm_sha_hash); 1572 1571 1573 1572 /** 1574 1573 * zynqmp_pm_register_notifier() - PM API for register a subsystem

+5

drivers/s390/crypto/pkey_cca.c

reviewed

··· 390 390 int i, len, rc; 391 391 u32 xflags; 392 392 393 393 + if (pflags & PKEY_XFLAG_NOCLEARKEY) { 394 394 + PKEY_DBF_ERR("%s clear key but xflag NOCLEARKEY\n", __func__); 395 395 + return -EINVAL; 396 396 + } 397 397 + 393 398 xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; 394 399 395 400 /* check keytype, subtype, clrkeylen, keybitsize */

+5

drivers/s390/crypto/pkey_ep11.c

reviewed

··· 358 358 int i, len, rc; 359 359 u32 xflags; 360 360 361 361 + if (pflags & PKEY_XFLAG_NOCLEARKEY) { 362 362 + PKEY_DBF_ERR("%s clear key but xflag NOCLEARKEY\n", __func__); 363 363 + return -EINVAL; 364 364 + } 365 365 + 361 366 xflags = pflags & PKEY_XFLAG_NOMEMALLOC ? ZCRYPT_XFLAG_NOMEMALLOC : 0; 362 367 363 368 /* check keytype, subtype, clrkeylen, keybitsize */

+9 -3

drivers/s390/crypto/pkey_pckmo.c

reviewed

··· 215 215 } 216 216 217 217 static int pckmo_key2protkey(const u8 *key, u32 keylen, 218 218 - u8 *protkey, u32 *protkeylen, u32 *protkeytype) 218 218 + u8 *protkey, u32 *protkeylen, u32 *protkeytype, 219 219 + u32 xflags) 219 220 { 220 221 struct keytoken_header *hdr = (struct keytoken_header *)key; 221 222 int rc = -EINVAL; ··· 267 266 struct clearkeytoken *t = (struct clearkeytoken *)key; 268 267 u32 keysize; 269 268 269 269 + if (xflags & PKEY_XFLAG_NOCLEARKEY) { 270 270 + PKEY_DBF_ERR("%s clear key token but xflag NOCLEARKEY\n", 271 271 + __func__); 272 272 + goto out; 273 273 + } 270 274 if (keylen < sizeof(*t) || 271 275 keylen < sizeof(*t) + t->len) 272 276 goto out; ··· 412 406 size_t _nr_apqns, 413 407 const u8 *key, u32 keylen, 414 408 u8 *protkey, u32 *protkeylen, u32 *keyinfo, 415 415 - u32 _xflags __always_unused) 409 409 + u32 xflags) 416 410 { 417 411 return pckmo_key2protkey(key, keylen, 418 418 - protkey, protkeylen, keyinfo); 412 412 + protkey, protkeylen, keyinfo, xflags); 419 413 } 420 414 421 415 static int pkey_pckmo_gen_key(const struct pkey_apqn *_apqns, size_t _nr_apqns,

+7

include/crypto/internal/skcipher.h

reviewed

··· 242 242 return crypto_tfm_ctx_dma(&tfm->base); 243 243 } 244 244 245 245 + static inline bool crypto_skcipher_tested(struct crypto_skcipher *tfm) 246 246 + { 247 247 + struct crypto_tfm *tfm_base = crypto_skcipher_tfm(tfm); 248 248 + 249 249 + return tfm_base->__crt_alg->cra_flags & CRYPTO_ALG_TESTED; 250 250 + } 251 251 + 245 252 static inline void *skcipher_request_ctx(struct skcipher_request *req) 246 253 { 247 254 return req->__ctx;

+119

include/linux/firmware/xlnx-zynqmp-crypto.h

reviewed

··· 1 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 2 + /* 3 3 + * Firmware layer for XilSECURE APIs. 4 4 + * 5 5 + * Copyright (C) 2014-2022 Xilinx, Inc. 6 6 + * Copyright (C) 2022-2025 Advanced Micro Devices, Inc. 7 7 + */ 8 8 + 9 9 + #ifndef __FIRMWARE_XLNX_ZYNQMP_CRYPTO_H__ 10 10 + #define __FIRMWARE_XLNX_ZYNQMP_CRYPTO_H__ 11 11 + 12 12 + /** 13 13 + * struct xlnx_feature - Feature data 14 14 + * @family: Family code of platform 15 15 + * @subfamily: Subfamily code of platform 16 16 + * @feature_id: Feature id of module 17 17 + * @data: Collection of all supported platform data 18 18 + */ 19 19 + struct xlnx_feature { 20 20 + u32 family; 21 21 + u32 feature_id; 22 22 + void *data; 23 23 + }; 24 24 + 25 25 + /* xilSecure API commands module id + api id */ 26 26 + #define XSECURE_API_AES_INIT 0x509 27 27 + #define XSECURE_API_AES_OP_INIT 0x50a 28 28 + #define XSECURE_API_AES_UPDATE_AAD 0x50b 29 29 + #define XSECURE_API_AES_ENCRYPT_UPDATE 0x50c 30 30 + #define XSECURE_API_AES_ENCRYPT_FINAL 0x50d 31 31 + #define XSECURE_API_AES_DECRYPT_UPDATE 0x50e 32 32 + #define XSECURE_API_AES_DECRYPT_FINAL 0x50f 33 33 + #define XSECURE_API_AES_KEY_ZERO 0x510 34 34 + #define XSECURE_API_AES_WRITE_KEY 0x511 35 35 + 36 36 + #if IS_REACHABLE(CONFIG_ZYNQMP_FIRMWARE) 37 37 + int zynqmp_pm_aes_engine(const u64 address, u32 *out); 38 38 + int zynqmp_pm_sha_hash(const u64 address, const u32 size, const u32 flags); 39 39 + void *xlnx_get_crypto_dev_data(struct xlnx_feature *feature_map); 40 40 + int versal_pm_aes_key_write(const u32 keylen, 41 41 + const u32 keysrc, const u64 keyaddr); 42 42 + int versal_pm_aes_key_zero(const u32 keysrc); 43 43 + int versal_pm_aes_op_init(const u64 hw_req); 44 44 + int versal_pm_aes_update_aad(const u64 aad_addr, const u32 aad_len); 45 45 + int versal_pm_aes_enc_update(const u64 in_params, const u64 in_addr); 46 46 + int versal_pm_aes_dec_update(const u64 in_params, const u64 in_addr); 47 47 + int versal_pm_aes_dec_final(const u64 gcm_addr); 48 48 + int versal_pm_aes_enc_final(const u64 gcm_addr); 49 49 + int versal_pm_aes_init(void); 50 50 + 51 51 + #else 52 52 + static inline int zynqmp_pm_aes_engine(const u64 address, u32 *out) 53 53 + { 54 54 + return -ENODEV; 55 55 + } 56 56 + 57 57 + static inline int zynqmp_pm_sha_hash(const u64 address, const u32 size, 58 58 + const u32 flags) 59 59 + { 60 60 + return -ENODEV; 61 61 + } 62 62 + 63 63 + static inline void *xlnx_get_crypto_dev_data(struct xlnx_feature *feature_map) 64 64 + { 65 65 + return ERR_PTR(-ENODEV); 66 66 + } 67 67 + 68 68 + static inline int versal_pm_aes_key_write(const u32 keylen, 69 69 + const u32 keysrc, const u64 keyaddr) 70 70 + { 71 71 + return -ENODEV; 72 72 + } 73 73 + 74 74 + static inline int versal_pm_aes_key_zero(const u32 keysrc) 75 75 + { 76 76 + return -ENODEV; 77 77 + } 78 78 + 79 79 + static inline int versal_pm_aes_op_init(const u64 hw_req) 80 80 + { 81 81 + return -ENODEV; 82 82 + } 83 83 + 84 84 + static inline int versal_pm_aes_update_aad(const u64 aad_addr, 85 85 + const u32 aad_len) 86 86 + { 87 87 + return -ENODEV; 88 88 + } 89 89 + 90 90 + static inline int versal_pm_aes_enc_update(const u64 in_params, 91 91 + const u64 in_addr) 92 92 + { 93 93 + return -ENODEV; 94 94 + } 95 95 + 96 96 + static inline int versal_pm_aes_dec_update(const u64 in_params, 97 97 + const u64 in_addr) 98 98 + { 99 99 + return -ENODEV; 100 100 + } 101 101 + 102 102 + static inline int versal_pm_aes_enc_final(const u64 gcm_addr) 103 103 + { 104 104 + return -ENODEV; 105 105 + } 106 106 + 107 107 + static inline int versal_pm_aes_dec_final(const u64 gcm_addr) 108 108 + { 109 109 + return -ENODEV; 110 110 + } 111 111 + 112 112 + static inline int versal_pm_aes_init(void) 113 113 + { 114 114 + return -ENODEV; 115 115 + } 116 116 + 117 117 + #endif 118 118 + 119 119 + #endif /* __FIRMWARE_XLNX_ZYNQMP_CRYPTO_H__ */

+1 -13

include/linux/firmware/xlnx-zynqmp.h

reviewed

··· 17 17 18 18 #include <linux/err.h> 19 19 #include <linux/firmware/xlnx-zynqmp-ufs.h> 20 20 + #include <linux/firmware/xlnx-zynqmp-crypto.h> 20 21 21 22 #define ZYNQMP_PM_VERSION_MAJOR 1 22 23 #define ZYNQMP_PM_VERSION_MINOR 0 ··· 590 589 int zynqmp_pm_set_requirement(const u32 node, const u32 capabilities, 591 590 const u32 qos, 592 591 const enum zynqmp_pm_request_ack ack); 593 593 - int zynqmp_pm_aes_engine(const u64 address, u32 *out); 594 592 int zynqmp_pm_efuse_access(const u64 address, u32 *out); 595 595 - int zynqmp_pm_sha_hash(const u64 address, const u32 size, const u32 flags); 596 593 int zynqmp_pm_fpga_load(const u64 address, const u32 size, const u32 flags); 597 594 int zynqmp_pm_fpga_get_status(u32 *value); 598 595 int zynqmp_pm_fpga_get_config_status(u32 *value); ··· 771 772 return -ENODEV; 772 773 } 773 774 774 774 - static inline int zynqmp_pm_aes_engine(const u64 address, u32 *out) 775 775 - { 776 776 - return -ENODEV; 777 777 - } 778 778 - 779 775 static inline int zynqmp_pm_efuse_access(const u64 address, u32 *out) 780 780 - { 781 781 - return -ENODEV; 782 782 - } 783 783 - 784 784 - static inline int zynqmp_pm_sha_hash(const u64 address, const u32 size, 785 785 - const u32 flags) 786 776 { 787 777 return -ENODEV; 788 778 }

+12 -3

include/linux/hisi_acc_qm.h

reviewed

··· 447 447 int (*fill_sqe)(void *sqe, void *q_parm, void *d_parm); 448 448 }; 449 449 450 450 + struct instance_backlog { 451 451 + struct list_head list; 452 452 + spinlock_t lock; 453 453 + }; 454 454 + 450 455 struct hisi_qp { 451 456 u32 qp_id; 452 457 u16 sq_depth; 453 458 u16 cq_depth; 454 459 u8 alg_type; 455 455 - u8 req_type; 456 460 457 461 struct qm_dma qdma; 458 462 void *sqe; ··· 466 462 467 463 struct hisi_qp_status qp_status; 468 464 struct hisi_qp_ops *hw_ops; 469 469 - void *qp_ctx; 470 465 void (*req_cb)(struct hisi_qp *qp, void *data); 471 466 void (*event_cb)(struct hisi_qp *qp); 472 467 ··· 474 471 bool is_in_kernel; 475 472 u16 pasid; 476 473 struct uacce_queue *uacce_q; 474 474 + 475 475 + u32 ref_count; 476 476 + spinlock_t qp_lock; 477 477 + struct instance_backlog backlog; 478 478 + const void **msg; 477 479 }; 478 480 479 481 static inline int vfs_num_set(const char *val, const struct kernel_param *kp) ··· 571 563 int hisi_qm_wait_mb_ready(struct hisi_qm *qm); 572 564 int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, 573 565 bool op); 566 566 + int hisi_qm_mb_read(struct hisi_qm *qm, u64 *base, u8 cmd, u16 queue); 574 567 575 568 struct hisi_acc_sgl_pool; 576 569 struct hisi_acc_hw_sgl *hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, ··· 584 575 void hisi_acc_free_sgl_pool(struct device *dev, 585 576 struct hisi_acc_sgl_pool *pool); 586 577 int hisi_qm_alloc_qps_node(struct hisi_qm_list *qm_list, int qp_num, 587 587 - u8 alg_type, int node, struct hisi_qp **qps); 578 578 + u8 *alg_type, int node, struct hisi_qp **qps); 588 579 void hisi_qm_free_qps(struct hisi_qp **qps, int qp_num); 589 580 void hisi_qm_dev_shutdown(struct pci_dev *pdev); 590 581 void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list);

+2

include/linux/hw_random.h

reviewed

··· 15 15 #include <linux/completion.h> 16 16 #include <linux/kref.h> 17 17 #include <linux/types.h> 18 18 + #include <linux/workqueue_types.h> 18 19 19 20 /** 20 21 * struct hwrng - Hardware Random Number Generator driver ··· 49 48 /* internal. */ 50 49 struct list_head list; 51 50 struct kref ref; 51 51 + struct work_struct cleanup_work; 52 52 struct completion cleanup_done; 53 53 struct completion dying; 54 54 };

+11 -11

kernel/padata.c

reviewed

··· 819 819 #define kobj2pinst(_kobj) \ 820 820 container_of(_kobj, struct padata_instance, kobj) 821 821 #define attr2pentry(_attr) \ 822 822 - container_of(_attr, struct padata_sysfs_entry, attr) 822 822 + container_of_const(_attr, struct padata_sysfs_entry, attr) 823 823 824 824 static void padata_sysfs_release(struct kobject *kobj) 825 825 { ··· 829 829 830 830 struct padata_sysfs_entry { 831 831 struct attribute attr; 832 832 - ssize_t (*show)(struct padata_instance *, struct attribute *, char *); 833 833 - ssize_t (*store)(struct padata_instance *, struct attribute *, 832 832 + ssize_t (*show)(struct padata_instance *, const struct attribute *, char *); 833 833 + ssize_t (*store)(struct padata_instance *, const struct attribute *, 834 834 const char *, size_t); 835 835 }; 836 836 837 837 static ssize_t show_cpumask(struct padata_instance *pinst, 838 838 - struct attribute *attr, char *buf) 838 838 + const struct attribute *attr, char *buf) 839 839 { 840 840 struct cpumask *cpumask; 841 841 ssize_t len; ··· 853 853 } 854 854 855 855 static ssize_t store_cpumask(struct padata_instance *pinst, 856 856 - struct attribute *attr, 856 856 + const struct attribute *attr, 857 857 const char *buf, size_t count) 858 858 { 859 859 cpumask_var_t new_cpumask; ··· 880 880 } 881 881 882 882 #define PADATA_ATTR_RW(_name, _show_name, _store_name) \ 883 883 - static struct padata_sysfs_entry _name##_attr = \ 883 883 + static const struct padata_sysfs_entry _name##_attr = \ 884 884 __ATTR(_name, 0644, _show_name, _store_name) 885 885 - #define PADATA_ATTR_RO(_name, _show_name) \ 886 886 - static struct padata_sysfs_entry _name##_attr = \ 885 885 + #define PADATA_ATTR_RO(_name, _show_name) \ 886 886 + static const struct padata_sysfs_entry _name##_attr = \ 887 887 __ATTR(_name, 0400, _show_name, NULL) 888 888 889 889 PADATA_ATTR_RW(serial_cpumask, show_cpumask, store_cpumask); ··· 894 894 * serial_cpumask [RW] - cpumask for serial workers 895 895 * parallel_cpumask [RW] - cpumask for parallel workers 896 896 */ 897 897 - static struct attribute *padata_default_attrs[] = { 897 897 + static const struct attribute *const padata_default_attrs[] = { 898 898 &serial_cpumask_attr.attr, 899 899 &parallel_cpumask_attr.attr, 900 900 NULL, ··· 904 904 static ssize_t padata_sysfs_show(struct kobject *kobj, 905 905 struct attribute *attr, char *buf) 906 906 { 907 907 + const struct padata_sysfs_entry *pentry; 907 908 struct padata_instance *pinst; 908 908 - struct padata_sysfs_entry *pentry; 909 909 ssize_t ret = -EIO; 910 910 911 911 pinst = kobj2pinst(kobj); ··· 919 919 static ssize_t padata_sysfs_store(struct kobject *kobj, struct attribute *attr, 920 920 const char *buf, size_t count) 921 921 { 922 922 + const struct padata_sysfs_entry *pentry; 922 923 struct padata_instance *pinst; 923 923 - struct padata_sysfs_entry *pentry; 924 924 ssize_t ret = -EIO; 925 925 926 926 pinst = kobj2pinst(kobj);