+2

Documentation/devicetree/bindings/crypto/qcom,inline-crypto-engine.yaml

reviewed

··· 13 13 compatible: 14 14 items: 15 15 - enum: 16 16 + - qcom,qcs8300-inline-crypto-engine 16 17 - qcom,sa8775p-inline-crypto-engine 17 18 - qcom,sc7180-inline-crypto-engine 18 19 - qcom,sc7280-inline-crypto-engine 19 20 - qcom,sm8450-inline-crypto-engine 20 21 - qcom,sm8550-inline-crypto-engine 21 22 - qcom,sm8650-inline-crypto-engine 23 23 + - qcom,sm8750-inline-crypto-engine 22 24 - const: qcom,inline-crypto-engine 23 25 24 26 reg:

+5

Documentation/devicetree/bindings/crypto/qcom,prng.yaml

reviewed

··· 17 17 - qcom,prng-ee # 8996 and later using EE 18 18 - items: 19 19 - enum: 20 20 + - qcom,ipq5332-trng 21 21 + - qcom,ipq5424-trng 22 22 + - qcom,ipq9574-trng 23 23 + - qcom,qcs8300-trng 20 24 - qcom,sa8255p-trng 21 25 - qcom,sa8775p-trng 22 26 - qcom,sc7280-trng 23 27 - qcom,sm8450-trng 24 28 - qcom,sm8550-trng 25 29 - qcom,sm8650-trng 30 30 + - qcom,sm8750-trng 26 31 - const: qcom,trng 27 32 28 33 reg:

+2

Documentation/devicetree/bindings/crypto/qcom-qce.yaml

reviewed

··· 45 45 46 46 - items: 47 47 - enum: 48 48 + - qcom,qcs8300-qce 48 49 - qcom,sa8775p-qce 49 50 - qcom,sc7280-qce 50 51 - qcom,sm6350-qce ··· 54 53 - qcom,sm8450-qce 55 54 - qcom,sm8550-qce 56 55 - qcom,sm8650-qce 56 56 + - qcom,sm8750-qce 57 57 - const: qcom,sm8150-qce 58 58 - const: qcom,qce 59 59

+8 -1

Documentation/driver-api/crypto/iaa/iaa-crypto.rst

reviewed

··· 272 272 echo async_irq > /sys/bus/dsa/drivers/crypto/sync_mode 273 273 274 274 Async mode without interrupts (caller must poll) can be enabled by 275 275 - writing 'async' to it:: 275 275 + writing 'async' to it (please see Caveat):: 276 276 277 277 echo async > /sys/bus/dsa/drivers/crypto/sync_mode 278 278 ··· 282 282 echo sync > /sys/bus/dsa/drivers/crypto/sync_mode 283 283 284 284 The default mode is 'sync'. 285 285 + 286 286 + Caveat: since the only mechanism that iaa_crypto currently implements 287 287 + for async polling without interrupts is via the 'sync' mode as 288 288 + described earlier, writing 'async' to 289 289 + '/sys/bus/dsa/drivers/crypto/sync_mode' will internally enable the 290 290 + 'sync' mode. This is to ensure correct iaa_crypto behavior until true 291 291 + async polling without interrupts is enabled in iaa_crypto. 285 292 286 293 .. _iaa_default_config: 287 294

+1 -1

MAINTAINERS

reviewed

··· 20164 20164 RHASHTABLE 20165 20165 M: Thomas Graf <tgraf@suug.ch> 20166 20166 M: Herbert Xu <herbert@gondor.apana.org.au> 20167 20167 - L: netdev@vger.kernel.org 20167 20167 + L: linux-crypto@vger.kernel.org 20168 20168 S: Maintained 20169 20169 F: include/linux/rhashtable-types.h 20170 20170 F: include/linux/rhashtable.h

-1

arch/arm/configs/pxa_defconfig

reviewed

··· 652 652 CONFIG_CRYPTO_LRW=m 653 653 CONFIG_CRYPTO_PCBC=m 654 654 CONFIG_CRYPTO_XTS=m 655 655 - CONFIG_CRYPTO_VMAC=m 656 655 CONFIG_CRYPTO_SHA512=m 657 656 CONFIG_CRYPTO_TGR192=m 658 657 CONFIG_CRYPTO_WP512=m

-1

arch/loongarch/configs/loongson3_defconfig

reviewed

··· 1029 1029 CONFIG_CRYPTO_TEA=m 1030 1030 CONFIG_CRYPTO_TWOFISH=m 1031 1031 CONFIG_CRYPTO_CHACHA20POLY1305=m 1032 1032 - CONFIG_CRYPTO_VMAC=m 1033 1032 CONFIG_CRYPTO_WP512=m 1034 1033 CONFIG_CRYPTO_DEFLATE=m 1035 1034 CONFIG_CRYPTO_LZO=m

-2

arch/m68k/configs/amiga_defconfig

reviewed

··· 579 579 CONFIG_CRYPTO_ARC4=m 580 580 CONFIG_CRYPTO_CTS=m 581 581 CONFIG_CRYPTO_HCTR2=m 582 582 - CONFIG_CRYPTO_KEYWRAP=m 583 582 CONFIG_CRYPTO_LRW=m 584 583 CONFIG_CRYPTO_PCBC=m 585 584 CONFIG_CRYPTO_XTS=m ··· 588 589 CONFIG_CRYPTO_MICHAEL_MIC=m 589 590 CONFIG_CRYPTO_RMD160=m 590 591 CONFIG_CRYPTO_SM3_GENERIC=m 591 591 - CONFIG_CRYPTO_VMAC=m 592 592 CONFIG_CRYPTO_WP512=m 593 593 CONFIG_CRYPTO_XCBC=m 594 594 CONFIG_CRYPTO_LZO=m

-2

arch/m68k/configs/apollo_defconfig

reviewed

··· 536 536 CONFIG_CRYPTO_ARC4=m 537 537 CONFIG_CRYPTO_CTS=m 538 538 CONFIG_CRYPTO_HCTR2=m 539 539 - CONFIG_CRYPTO_KEYWRAP=m 540 539 CONFIG_CRYPTO_LRW=m 541 540 CONFIG_CRYPTO_PCBC=m 542 541 CONFIG_CRYPTO_XTS=m ··· 545 546 CONFIG_CRYPTO_MICHAEL_MIC=m 546 547 CONFIG_CRYPTO_RMD160=m 547 548 CONFIG_CRYPTO_SM3_GENERIC=m 548 548 - CONFIG_CRYPTO_VMAC=m 549 549 CONFIG_CRYPTO_WP512=m 550 550 CONFIG_CRYPTO_XCBC=m 551 551 CONFIG_CRYPTO_LZO=m

-2

arch/m68k/configs/atari_defconfig

reviewed

··· 556 556 CONFIG_CRYPTO_ARC4=m 557 557 CONFIG_CRYPTO_CTS=m 558 558 CONFIG_CRYPTO_HCTR2=m 559 559 - CONFIG_CRYPTO_KEYWRAP=m 560 559 CONFIG_CRYPTO_LRW=m 561 560 CONFIG_CRYPTO_PCBC=m 562 561 CONFIG_CRYPTO_XTS=m ··· 565 566 CONFIG_CRYPTO_MICHAEL_MIC=m 566 567 CONFIG_CRYPTO_RMD160=m 567 568 CONFIG_CRYPTO_SM3_GENERIC=m 568 568 - CONFIG_CRYPTO_VMAC=m 569 569 CONFIG_CRYPTO_WP512=m 570 570 CONFIG_CRYPTO_XCBC=m 571 571 CONFIG_CRYPTO_LZO=m

-2

arch/m68k/configs/bvme6000_defconfig

reviewed

··· 528 528 CONFIG_CRYPTO_ARC4=m 529 529 CONFIG_CRYPTO_CTS=m 530 530 CONFIG_CRYPTO_HCTR2=m 531 531 - CONFIG_CRYPTO_KEYWRAP=m 532 531 CONFIG_CRYPTO_LRW=m 533 532 CONFIG_CRYPTO_PCBC=m 534 533 CONFIG_CRYPTO_XTS=m ··· 537 538 CONFIG_CRYPTO_MICHAEL_MIC=m 538 539 CONFIG_CRYPTO_RMD160=m 539 540 CONFIG_CRYPTO_SM3_GENERIC=m 540 540 - CONFIG_CRYPTO_VMAC=m 541 541 CONFIG_CRYPTO_WP512=m 542 542 CONFIG_CRYPTO_XCBC=m 543 543 CONFIG_CRYPTO_LZO=m

-2

arch/m68k/configs/hp300_defconfig

reviewed

··· 538 538 CONFIG_CRYPTO_ARC4=m 539 539 CONFIG_CRYPTO_CTS=m 540 540 CONFIG_CRYPTO_HCTR2=m 541 541 - CONFIG_CRYPTO_KEYWRAP=m 542 541 CONFIG_CRYPTO_LRW=m 543 542 CONFIG_CRYPTO_PCBC=m 544 543 CONFIG_CRYPTO_XTS=m ··· 547 548 CONFIG_CRYPTO_MICHAEL_MIC=m 548 549 CONFIG_CRYPTO_RMD160=m 549 550 CONFIG_CRYPTO_SM3_GENERIC=m 550 550 - CONFIG_CRYPTO_VMAC=m 551 551 CONFIG_CRYPTO_WP512=m 552 552 CONFIG_CRYPTO_XCBC=m 553 553 CONFIG_CRYPTO_LZO=m

-2

arch/m68k/configs/mac_defconfig

reviewed

··· 555 555 CONFIG_CRYPTO_ARC4=m 556 556 CONFIG_CRYPTO_CTS=m 557 557 CONFIG_CRYPTO_HCTR2=m 558 558 - CONFIG_CRYPTO_KEYWRAP=m 559 558 CONFIG_CRYPTO_LRW=m 560 559 CONFIG_CRYPTO_PCBC=m 561 560 CONFIG_CRYPTO_XTS=m ··· 564 565 CONFIG_CRYPTO_MICHAEL_MIC=m 565 566 CONFIG_CRYPTO_RMD160=m 566 567 CONFIG_CRYPTO_SM3_GENERIC=m 567 567 - CONFIG_CRYPTO_VMAC=m 568 568 CONFIG_CRYPTO_WP512=m 569 569 CONFIG_CRYPTO_XCBC=m 570 570 CONFIG_CRYPTO_LZO=m

-2

arch/m68k/configs/multi_defconfig

reviewed

··· 642 642 CONFIG_CRYPTO_ARC4=m 643 643 CONFIG_CRYPTO_CTS=m 644 644 CONFIG_CRYPTO_HCTR2=m 645 645 - CONFIG_CRYPTO_KEYWRAP=m 646 645 CONFIG_CRYPTO_LRW=m 647 646 CONFIG_CRYPTO_PCBC=m 648 647 CONFIG_CRYPTO_XTS=m ··· 651 652 CONFIG_CRYPTO_MICHAEL_MIC=m 652 653 CONFIG_CRYPTO_RMD160=m 653 654 CONFIG_CRYPTO_SM3_GENERIC=m 654 654 - CONFIG_CRYPTO_VMAC=m 655 655 CONFIG_CRYPTO_WP512=m 656 656 CONFIG_CRYPTO_XCBC=m 657 657 CONFIG_CRYPTO_LZO=m

-2

arch/m68k/configs/mvme147_defconfig

reviewed

··· 528 528 CONFIG_CRYPTO_ARC4=m 529 529 CONFIG_CRYPTO_CTS=m 530 530 CONFIG_CRYPTO_HCTR2=m 531 531 - CONFIG_CRYPTO_KEYWRAP=m 532 531 CONFIG_CRYPTO_LRW=m 533 532 CONFIG_CRYPTO_PCBC=m 534 533 CONFIG_CRYPTO_XTS=m ··· 537 538 CONFIG_CRYPTO_MICHAEL_MIC=m 538 539 CONFIG_CRYPTO_RMD160=m 539 540 CONFIG_CRYPTO_SM3_GENERIC=m 540 540 - CONFIG_CRYPTO_VMAC=m 541 541 CONFIG_CRYPTO_WP512=m 542 542 CONFIG_CRYPTO_XCBC=m 543 543 CONFIG_CRYPTO_LZO=m

-2

arch/m68k/configs/mvme16x_defconfig

reviewed

··· 529 529 CONFIG_CRYPTO_ARC4=m 530 530 CONFIG_CRYPTO_CTS=m 531 531 CONFIG_CRYPTO_HCTR2=m 532 532 - CONFIG_CRYPTO_KEYWRAP=m 533 532 CONFIG_CRYPTO_LRW=m 534 533 CONFIG_CRYPTO_PCBC=m 535 534 CONFIG_CRYPTO_XTS=m ··· 538 539 CONFIG_CRYPTO_MICHAEL_MIC=m 539 540 CONFIG_CRYPTO_RMD160=m 540 541 CONFIG_CRYPTO_SM3_GENERIC=m 541 541 - CONFIG_CRYPTO_VMAC=m 542 542 CONFIG_CRYPTO_WP512=m 543 543 CONFIG_CRYPTO_XCBC=m 544 544 CONFIG_CRYPTO_LZO=m

-2

arch/m68k/configs/q40_defconfig

reviewed

··· 545 545 CONFIG_CRYPTO_ARC4=m 546 546 CONFIG_CRYPTO_CTS=m 547 547 CONFIG_CRYPTO_HCTR2=m 548 548 - CONFIG_CRYPTO_KEYWRAP=m 549 548 CONFIG_CRYPTO_LRW=m 550 549 CONFIG_CRYPTO_PCBC=m 551 550 CONFIG_CRYPTO_XTS=m ··· 554 555 CONFIG_CRYPTO_MICHAEL_MIC=m 555 556 CONFIG_CRYPTO_RMD160=m 556 557 CONFIG_CRYPTO_SM3_GENERIC=m 557 557 - CONFIG_CRYPTO_VMAC=m 558 558 CONFIG_CRYPTO_WP512=m 559 559 CONFIG_CRYPTO_XCBC=m 560 560 CONFIG_CRYPTO_LZO=m

-2

arch/m68k/configs/sun3_defconfig

reviewed

··· 526 526 CONFIG_CRYPTO_ARC4=m 527 527 CONFIG_CRYPTO_CTS=m 528 528 CONFIG_CRYPTO_HCTR2=m 529 529 - CONFIG_CRYPTO_KEYWRAP=m 530 529 CONFIG_CRYPTO_LRW=m 531 530 CONFIG_CRYPTO_PCBC=m 532 531 CONFIG_CRYPTO_XTS=m ··· 535 536 CONFIG_CRYPTO_MICHAEL_MIC=m 536 537 CONFIG_CRYPTO_RMD160=m 537 538 CONFIG_CRYPTO_SM3_GENERIC=m 538 538 - CONFIG_CRYPTO_VMAC=m 539 539 CONFIG_CRYPTO_WP512=m 540 540 CONFIG_CRYPTO_XCBC=m 541 541 CONFIG_CRYPTO_LZO=m

-2

arch/m68k/configs/sun3x_defconfig

reviewed

··· 526 526 CONFIG_CRYPTO_ARC4=m 527 527 CONFIG_CRYPTO_CTS=m 528 528 CONFIG_CRYPTO_HCTR2=m 529 529 - CONFIG_CRYPTO_KEYWRAP=m 530 529 CONFIG_CRYPTO_LRW=m 531 530 CONFIG_CRYPTO_PCBC=m 532 531 CONFIG_CRYPTO_XTS=m ··· 535 536 CONFIG_CRYPTO_MICHAEL_MIC=m 536 537 CONFIG_CRYPTO_RMD160=m 537 538 CONFIG_CRYPTO_SM3_GENERIC=m 538 538 - CONFIG_CRYPTO_VMAC=m 539 539 CONFIG_CRYPTO_WP512=m 540 540 CONFIG_CRYPTO_XCBC=m 541 541 CONFIG_CRYPTO_LZO=m

-1

arch/mips/configs/bigsur_defconfig

reviewed

··· 222 222 CONFIG_CRYPTO_XTS=m 223 223 CONFIG_CRYPTO_HMAC=y 224 224 CONFIG_CRYPTO_XCBC=m 225 225 - CONFIG_CRYPTO_VMAC=m 226 225 CONFIG_CRYPTO_MD4=m 227 226 CONFIG_CRYPTO_MICHAEL_MIC=m 228 227 CONFIG_CRYPTO_RMD160=m

-2

arch/mips/configs/decstation_64_defconfig

reviewed

··· 177 177 CONFIG_CRYPTO_OFB=m 178 178 CONFIG_CRYPTO_PCBC=m 179 179 CONFIG_CRYPTO_XTS=m 180 180 - CONFIG_CRYPTO_KEYWRAP=m 181 180 CONFIG_CRYPTO_CMAC=m 182 181 CONFIG_CRYPTO_XCBC=m 183 183 - CONFIG_CRYPTO_VMAC=m 184 182 CONFIG_CRYPTO_CRC32=m 185 183 CONFIG_CRYPTO_CRCT10DIF=m 186 184 CONFIG_CRYPTO_MD4=m

-2

arch/mips/configs/decstation_defconfig

reviewed

··· 172 172 CONFIG_CRYPTO_OFB=m 173 173 CONFIG_CRYPTO_PCBC=m 174 174 CONFIG_CRYPTO_XTS=m 175 175 - CONFIG_CRYPTO_KEYWRAP=m 176 175 CONFIG_CRYPTO_CMAC=m 177 176 CONFIG_CRYPTO_XCBC=m 178 178 - CONFIG_CRYPTO_VMAC=m 179 177 CONFIG_CRYPTO_CRC32=m 180 178 CONFIG_CRYPTO_CRCT10DIF=m 181 179 CONFIG_CRYPTO_MD4=m

-2

arch/mips/configs/decstation_r4k_defconfig

reviewed

··· 172 172 CONFIG_CRYPTO_OFB=m 173 173 CONFIG_CRYPTO_PCBC=m 174 174 CONFIG_CRYPTO_XTS=m 175 175 - CONFIG_CRYPTO_KEYWRAP=m 176 175 CONFIG_CRYPTO_CMAC=m 177 176 CONFIG_CRYPTO_XCBC=m 178 178 - CONFIG_CRYPTO_VMAC=m 179 177 CONFIG_CRYPTO_CRC32=m 180 178 CONFIG_CRYPTO_CRCT10DIF=m 181 179 CONFIG_CRYPTO_MD4=m

-1

arch/mips/configs/ip27_defconfig

reviewed

··· 305 305 CONFIG_CRYPTO_XTS=m 306 306 CONFIG_CRYPTO_HMAC=y 307 307 CONFIG_CRYPTO_XCBC=m 308 308 - CONFIG_CRYPTO_VMAC=m 309 308 CONFIG_CRYPTO_MD4=m 310 309 CONFIG_CRYPTO_RMD160=m 311 310 CONFIG_CRYPTO_SHA512=m

-1

arch/mips/configs/ip30_defconfig

reviewed

··· 176 176 CONFIG_CRYPTO_HMAC=y 177 177 CONFIG_CRYPTO_MD4=m 178 178 CONFIG_CRYPTO_RMD160=m 179 179 - CONFIG_CRYPTO_VMAC=m 180 179 CONFIG_CRYPTO_WP512=m 181 180 CONFIG_CRYPTO_XCBC=m 182 181 CONFIG_CRYPTO_LZO=m

+2 -7

arch/powerpc/crypto/aes-gcm-p10-glue.c

reviewed

··· 214 214 struct gcm_ctx *gctx = PTR_ALIGN((void *)databuf, PPC_ALIGN); 215 215 u8 hashbuf[sizeof(struct Hash_ctx) + PPC_ALIGN]; 216 216 struct Hash_ctx *hash = PTR_ALIGN((void *)hashbuf, PPC_ALIGN); 217 217 - struct scatter_walk assoc_sg_walk; 218 217 struct skcipher_walk walk; 219 218 u8 *assocmem = NULL; 220 219 u8 *assoc; ··· 233 234 234 235 /* Linearize assoc, if not already linear */ 235 236 if (req->src->length >= assoclen && req->src->length) { 236 236 - scatterwalk_start(&assoc_sg_walk, req->src); 237 237 - assoc = scatterwalk_map(&assoc_sg_walk); 237 237 + assoc = sg_virt(req->src); /* ppc64 is !HIGHMEM */ 238 238 } else { 239 239 gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? 240 240 GFP_KERNEL : GFP_ATOMIC; ··· 251 253 gcmp10_init(gctx, iv, (unsigned char *) &ctx->enc_key, hash, assoc, assoclen); 252 254 vsx_end(); 253 255 254 254 - if (!assocmem) 255 255 - scatterwalk_unmap(assoc); 256 256 - else 257 257 - kfree(assocmem); 256 256 + kfree(assocmem); 258 257 259 258 if (enc) 260 259 ret = skcipher_walk_aead_encrypt(&walk, req, false);

-2

arch/s390/configs/debug_defconfig

reviewed

··· 770 770 CONFIG_CRYPTO_ADIANTUM=m 771 771 CONFIG_CRYPTO_ARC4=m 772 772 CONFIG_CRYPTO_HCTR2=m 773 773 - CONFIG_CRYPTO_KEYWRAP=m 774 773 CONFIG_CRYPTO_LRW=m 775 774 CONFIG_CRYPTO_PCBC=m 776 775 CONFIG_CRYPTO_AEGIS128=m ··· 781 782 CONFIG_CRYPTO_MICHAEL_MIC=m 782 783 CONFIG_CRYPTO_RMD160=m 783 784 CONFIG_CRYPTO_SM3_GENERIC=m 784 784 - CONFIG_CRYPTO_VMAC=m 785 785 CONFIG_CRYPTO_WP512=m 786 786 CONFIG_CRYPTO_XCBC=m 787 787 CONFIG_CRYPTO_CRC32=m

-2

arch/s390/configs/defconfig

reviewed

··· 756 756 CONFIG_CRYPTO_ADIANTUM=m 757 757 CONFIG_CRYPTO_ARC4=m 758 758 CONFIG_CRYPTO_HCTR2=m 759 759 - CONFIG_CRYPTO_KEYWRAP=m 760 759 CONFIG_CRYPTO_LRW=m 761 760 CONFIG_CRYPTO_PCBC=m 762 761 CONFIG_CRYPTO_AEGIS128=m ··· 767 768 CONFIG_CRYPTO_MICHAEL_MIC=m 768 769 CONFIG_CRYPTO_RMD160=m 769 770 CONFIG_CRYPTO_SM3_GENERIC=m 770 770 - CONFIG_CRYPTO_VMAC=m 771 771 CONFIG_CRYPTO_WP512=m 772 772 CONFIG_CRYPTO_XCBC=m 773 773 CONFIG_CRYPTO_CRC32=m

-1

arch/x86/crypto/aegis128-aesni-glue.c

reviewed

··· 240 240 .cra_blocksize = 1, 241 241 .cra_ctxsize = sizeof(struct aegis_ctx) + 242 242 __alignof__(struct aegis_ctx), 243 243 - .cra_alignmask = 0, 244 243 .cra_priority = 400, 245 244 246 245 .cra_name = "__aegis128",

+48 -71

arch/x86/crypto/aes-gcm-avx10-x86_64.S

reviewed

··· 88 88 89 89 // A shuffle mask that reflects the bytes of 16-byte blocks 90 90 .Lbswap_mask: 91 91 - .octa 0x000102030405060708090a0b0c0d0e0f 91 91 + .octa 0x000102030405060708090a0b0c0d0e0f 92 92 93 93 // This is the GHASH reducing polynomial without its constant term, i.e. 94 94 // x^128 + x^7 + x^2 + x, represented using the backwards mapping ··· 384 384 vpshufd $0xd3, H_CUR_XMM, %xmm0 385 385 vpsrad $31, %xmm0, %xmm0 386 386 vpaddq H_CUR_XMM, H_CUR_XMM, H_CUR_XMM 387 387 - vpand .Lgfpoly_and_internal_carrybit(%rip), %xmm0, %xmm0 388 388 - vpxor %xmm0, H_CUR_XMM, H_CUR_XMM 387 387 + // H_CUR_XMM ^= xmm0 & gfpoly_and_internal_carrybit 388 388 + vpternlogd $0x78, .Lgfpoly_and_internal_carrybit(%rip), %xmm0, H_CUR_XMM 389 389 390 390 // Load the gfpoly constant. 391 391 vbroadcasti32x4 .Lgfpoly(%rip), GFPOLY ··· 562 562 vpxord RNDKEY0, V3, V3 563 563 .endm 564 564 565 565 + // Do the last AES round for four vectors of counter blocks V0-V3, XOR source 566 566 + // data with the resulting keystream, and write the result to DST and 567 567 + // GHASHDATA[0-3]. (Implementation differs slightly, but has the same effect.) 568 568 + .macro _aesenclast_and_xor_4x 569 569 + // XOR the source data with the last round key, saving the result in 570 570 + // GHASHDATA[0-3]. This reduces latency by taking advantage of the 571 571 + // property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a). 572 572 + vpxord 0*VL(SRC), RNDKEYLAST, GHASHDATA0 573 573 + vpxord 1*VL(SRC), RNDKEYLAST, GHASHDATA1 574 574 + vpxord 2*VL(SRC), RNDKEYLAST, GHASHDATA2 575 575 + vpxord 3*VL(SRC), RNDKEYLAST, GHASHDATA3 576 576 + 577 577 + // Do the last AES round. This handles the XOR with the source data 578 578 + // too, as per the optimization described above. 579 579 + vaesenclast GHASHDATA0, V0, GHASHDATA0 580 580 + vaesenclast GHASHDATA1, V1, GHASHDATA1 581 581 + vaesenclast GHASHDATA2, V2, GHASHDATA2 582 582 + vaesenclast GHASHDATA3, V3, GHASHDATA3 583 583 + 584 584 + // Store the en/decrypted data to DST. 585 585 + vmovdqu8 GHASHDATA0, 0*VL(DST) 586 586 + vmovdqu8 GHASHDATA1, 1*VL(DST) 587 587 + vmovdqu8 GHASHDATA2, 2*VL(DST) 588 588 + vmovdqu8 GHASHDATA3, 3*VL(DST) 589 589 + .endm 590 590 + 565 591 // void aes_gcm_{enc,dec}_update_##suffix(const struct aes_gcm_key_avx10 *key, 566 592 // const u32 le_ctr[4], u8 ghash_acc[16], 567 593 // const u8 *src, u8 *dst, int datalen); ··· 666 640 // LE_CTR contains the next set of little-endian counter blocks. 667 641 .set LE_CTR, V12 668 642 669 669 - // RNDKEY0, RNDKEYLAST, and RNDKEY_M[9-5] contain cached AES round keys, 643 643 + // RNDKEY0, RNDKEYLAST, and RNDKEY_M[9-1] contain cached AES round keys, 670 644 // copied to all 128-bit lanes. RNDKEY0 is the zero-th round key, 671 645 // RNDKEYLAST the last, and RNDKEY_M\i the one \i-th from the last. 672 646 .set RNDKEY0, V13 ··· 676 650 .set RNDKEY_M7, V17 677 651 .set RNDKEY_M6, V18 678 652 .set RNDKEY_M5, V19 679 679 - 680 680 - // RNDKEYLAST[0-3] temporarily store the last AES round key XOR'd with 681 681 - // the corresponding block of source data. This is useful because 682 682 - // vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a), and key ^ b can 683 683 - // be computed in parallel with the AES rounds. 684 684 - .set RNDKEYLAST0, V20 685 685 - .set RNDKEYLAST1, V21 686 686 - .set RNDKEYLAST2, V22 687 687 - .set RNDKEYLAST3, V23 653 653 + .set RNDKEY_M4, V20 654 654 + .set RNDKEY_M3, V21 655 655 + .set RNDKEY_M2, V22 656 656 + .set RNDKEY_M1, V23 688 657 689 658 // GHASHTMP[0-2] are temporary variables used by _ghash_step_4x. These 690 659 // cannot coincide with anything used for AES encryption, since for ··· 734 713 // Pre-subtracting 4*VL from DATALEN saves an instruction from the main 735 714 // loop and also ensures that at least one write always occurs to 736 715 // DATALEN, zero-extending it and allowing DATALEN64 to be used later. 737 737 - sub $4*VL, DATALEN 716 716 + add $-4*VL, DATALEN // shorter than 'sub 4*VL' when VL=32 738 717 jl .Lcrypt_loop_4x_done\@ 739 718 740 719 // Load powers of the hash key. ··· 769 748 add $16, %rax 770 749 cmp %rax, RNDKEYLAST_PTR 771 750 jne 1b 772 772 - vpxord 0*VL(SRC), RNDKEYLAST, RNDKEYLAST0 773 773 - vpxord 1*VL(SRC), RNDKEYLAST, RNDKEYLAST1 774 774 - vpxord 2*VL(SRC), RNDKEYLAST, RNDKEYLAST2 775 775 - vpxord 3*VL(SRC), RNDKEYLAST, RNDKEYLAST3 776 776 - vaesenclast RNDKEYLAST0, V0, GHASHDATA0 777 777 - vaesenclast RNDKEYLAST1, V1, GHASHDATA1 778 778 - vaesenclast RNDKEYLAST2, V2, GHASHDATA2 779 779 - vaesenclast RNDKEYLAST3, V3, GHASHDATA3 780 780 - vmovdqu8 GHASHDATA0, 0*VL(DST) 781 781 - vmovdqu8 GHASHDATA1, 1*VL(DST) 782 782 - vmovdqu8 GHASHDATA2, 2*VL(DST) 783 783 - vmovdqu8 GHASHDATA3, 3*VL(DST) 784 784 - add $4*VL, SRC 785 785 - add $4*VL, DST 786 786 - sub $4*VL, DATALEN 751 751 + _aesenclast_and_xor_4x 752 752 + sub $-4*VL, SRC // shorter than 'add 4*VL' when VL=32 753 753 + sub $-4*VL, DST 754 754 + add $-4*VL, DATALEN 787 755 jl .Lghash_last_ciphertext_4x\@ 788 756 .endif 789 757 790 758 // Cache as many additional AES round keys as possible. 791 791 - .irp i, 9,8,7,6,5 759 759 + .irp i, 9,8,7,6,5,4,3,2,1 792 760 vbroadcasti32x4 -\i*16(RNDKEYLAST_PTR), RNDKEY_M\i 793 761 .endr 794 762 ··· 809 799 _vaesenc_4x RNDKEY 810 800 128: 811 801 812 812 - // XOR the source data with the last round key, saving the result in 813 813 - // RNDKEYLAST[0-3]. This reduces latency by taking advantage of the 814 814 - // property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a). 815 815 - .if \enc 816 816 - vpxord 0*VL(SRC), RNDKEYLAST, RNDKEYLAST0 817 817 - vpxord 1*VL(SRC), RNDKEYLAST, RNDKEYLAST1 818 818 - vpxord 2*VL(SRC), RNDKEYLAST, RNDKEYLAST2 819 819 - vpxord 3*VL(SRC), RNDKEYLAST, RNDKEYLAST3 820 820 - .else 821 821 - vpxord GHASHDATA0, RNDKEYLAST, RNDKEYLAST0 822 822 - vpxord GHASHDATA1, RNDKEYLAST, RNDKEYLAST1 823 823 - vpxord GHASHDATA2, RNDKEYLAST, RNDKEYLAST2 824 824 - vpxord GHASHDATA3, RNDKEYLAST, RNDKEYLAST3 825 825 - .endif 826 826 - 827 802 // Finish the AES encryption of the counter blocks in V0-V3, interleaved 828 803 // with the GHASH update of the ciphertext blocks in GHASHDATA[0-3]. 829 829 - .irp i, 9,8,7,6,5 804 804 + .irp i, 9,8,7,6,5,4,3,2,1 805 805 + _ghash_step_4x (9 - \i) 830 806 _vaesenc_4x RNDKEY_M\i 831 831 - _ghash_step_4x (9 - \i) 832 832 - .endr 833 833 - .irp i, 4,3,2,1 834 834 - vbroadcasti32x4 -\i*16(RNDKEYLAST_PTR), RNDKEY 835 835 - _vaesenc_4x RNDKEY 836 836 - _ghash_step_4x (9 - \i) 837 807 .endr 838 808 _ghash_step_4x 9 839 839 - 840 840 - // Do the last AES round. This handles the XOR with the source data 841 841 - // too, as per the optimization described above. 842 842 - vaesenclast RNDKEYLAST0, V0, GHASHDATA0 843 843 - vaesenclast RNDKEYLAST1, V1, GHASHDATA1 844 844 - vaesenclast RNDKEYLAST2, V2, GHASHDATA2 845 845 - vaesenclast RNDKEYLAST3, V3, GHASHDATA3 846 846 - 847 847 - // Store the en/decrypted data to DST. 848 848 - vmovdqu8 GHASHDATA0, 0*VL(DST) 849 849 - vmovdqu8 GHASHDATA1, 1*VL(DST) 850 850 - vmovdqu8 GHASHDATA2, 2*VL(DST) 851 851 - vmovdqu8 GHASHDATA3, 3*VL(DST) 852 852 - 853 853 - add $4*VL, SRC 854 854 - add $4*VL, DST 855 855 - sub $4*VL, DATALEN 809 809 + _aesenclast_and_xor_4x 810 810 + sub $-4*VL, SRC // shorter than 'add 4*VL' when VL=32 811 811 + sub $-4*VL, DST 812 812 + add $-4*VL, DATALEN 856 813 jge .Lcrypt_loop_4x\@ 857 814 858 815 .if \enc ··· 833 856 .Lcrypt_loop_4x_done\@: 834 857 835 858 // Undo the extra subtraction by 4*VL and check whether data remains. 836 836 - add $4*VL, DATALEN 859 859 + sub $-4*VL, DATALEN // shorter than 'add 4*VL' when VL=32 837 860 jz .Ldone\@ 838 861 839 862 // The data length isn't a multiple of 4*VL. Process the remaining data ··· 917 940 // GHASH. However, any such blocks are all-zeroes, and the values that 918 941 // they're multiplied with are also all-zeroes. Therefore they just add 919 942 // 0 * 0 = 0 to the final GHASH result, which makes no difference. 920 920 - vmovdqu8 (POWERS_PTR), H_POW1 943 943 + vmovdqu8 (POWERS_PTR), H_POW1 921 944 .if \enc 922 945 vmovdqu8 V0, V1{%k1}{z} 923 946 .endif

+168 -161

arch/x86/crypto/aes-xts-avx-x86_64.S

reviewed

··· 80 80 .byte 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80 81 81 .text 82 82 83 83 - // Function parameters 84 84 - .set KEY, %rdi // Initially points to crypto_aes_ctx, then is 85 85 - // advanced to point to 7th-from-last round key 86 86 - .set SRC, %rsi // Pointer to next source data 87 87 - .set DST, %rdx // Pointer to next destination data 88 88 - .set LEN, %ecx // Remaining length in bytes 89 89 - .set LEN8, %cl 90 90 - .set LEN64, %rcx 91 91 - .set TWEAK, %r8 // Pointer to next tweak 92 92 - 93 93 - // %rax holds the AES key length in bytes. 94 94 - .set KEYLEN, %eax 95 95 - .set KEYLEN64, %rax 96 96 - 97 97 - // %r9-r11 are available as temporaries. 98 98 - 99 83 .macro _define_Vi i 100 84 .if VL == 16 101 85 .set V\i, %xmm\i ··· 96 112 // Define register aliases V0-V15, or V0-V31 if all 32 SIMD registers 97 113 // are available, that map to the xmm, ymm, or zmm registers according 98 114 // to the selected Vector Length (VL). 99 99 - _define_Vi 0 100 100 - _define_Vi 1 101 101 - _define_Vi 2 102 102 - _define_Vi 3 103 103 - _define_Vi 4 104 104 - _define_Vi 5 105 105 - _define_Vi 6 106 106 - _define_Vi 7 107 107 - _define_Vi 8 108 108 - _define_Vi 9 109 109 - _define_Vi 10 110 110 - _define_Vi 11 111 111 - _define_Vi 12 112 112 - _define_Vi 13 113 113 - _define_Vi 14 114 114 - _define_Vi 15 115 115 + .irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15 116 116 + _define_Vi \i 117 117 + .endr 115 118 .if USE_AVX10 116 116 - _define_Vi 16 117 117 - _define_Vi 17 118 118 - _define_Vi 18 119 119 - _define_Vi 19 120 120 - _define_Vi 20 121 121 - _define_Vi 21 122 122 - _define_Vi 22 123 123 - _define_Vi 23 124 124 - _define_Vi 24 125 125 - _define_Vi 25 126 126 - _define_Vi 26 127 127 - _define_Vi 27 128 128 - _define_Vi 28 129 129 - _define_Vi 29 130 130 - _define_Vi 30 131 131 - _define_Vi 31 119 119 + .irp i, 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 120 120 + _define_Vi \i 121 121 + .endr 132 122 .endif 123 123 + 124 124 + // Function parameters 125 125 + .set KEY, %rdi // Initially points to crypto_aes_ctx, then is 126 126 + // advanced to point to 7th-from-last round key 127 127 + .set SRC, %rsi // Pointer to next source data 128 128 + .set DST, %rdx // Pointer to next destination data 129 129 + .set LEN, %ecx // Remaining length in bytes 130 130 + .set LEN8, %cl 131 131 + .set LEN64, %rcx 132 132 + .set TWEAK, %r8 // Pointer to next tweak 133 133 + 134 134 + // %rax holds the AES key length in bytes. 135 135 + .set KEYLEN, %eax 136 136 + .set KEYLEN64, %rax 137 137 + 138 138 + // %r9-r11 are available as temporaries. 133 139 134 140 // V0-V3 hold the data blocks during the main loop, or temporary values 135 141 // otherwise. V4-V5 hold temporary values. ··· 188 214 .endm 189 215 190 216 // Move a vector between memory and a register. 217 217 + // The register operand must be in the first 16 vector registers. 191 218 .macro _vmovdqu src, dst 192 219 .if VL < 64 193 220 vmovdqu \src, \dst ··· 209 234 .endm 210 235 211 236 // XOR two vectors together. 237 237 + // Any register operands must be in the first 16 vector registers. 212 238 .macro _vpxor src1, src2, dst 213 213 - .if USE_AVX10 214 214 - vpxord \src1, \src2, \dst 215 215 - .else 239 239 + .if VL < 64 216 240 vpxor \src1, \src2, \dst 241 241 + .else 242 242 + vpxord \src1, \src2, \dst 217 243 .endif 218 244 .endm 219 245 ··· 235 259 vpshufd $0x13, \src, \tmp 236 260 vpaddq \src, \src, \dst 237 261 vpsrad $31, \tmp, \tmp 262 262 + .if USE_AVX10 263 263 + vpternlogd $0x78, GF_POLY_XMM, \tmp, \dst 264 264 + .else 238 265 vpand GF_POLY_XMM, \tmp, \tmp 239 266 vpxor \tmp, \dst, \dst 267 267 + .endif 240 268 .endm 241 269 242 270 // Given the XTS tweak(s) in the vector \src, compute the next vector of ··· 349 369 350 370 // Do one step in computing the next set of tweaks using the VPCLMULQDQ method 351 371 // (the same method _next_tweakvec uses for VL > 16). This means multiplying 352 352 - // each tweak by x^(4*VL/16) independently. Since 4*VL/16 is a multiple of 8 353 353 - // when VL > 16 (which it is here), the needed shift amounts are byte-aligned, 354 354 - // which allows the use of vpsrldq and vpslldq to do 128-bit wide shifts. 372 372 + // each tweak by x^(4*VL/16) independently. 373 373 + // 374 374 + // Since 4*VL/16 is a multiple of 8 when VL > 16 (which it is here), the needed 375 375 + // shift amounts are byte-aligned, which allows the use of vpsrldq and vpslldq 376 376 + // to do 128-bit wide shifts. The 128-bit left shift (vpslldq) saves 377 377 + // instructions directly. The 128-bit right shift (vpsrldq) performs better 378 378 + // than a 64-bit right shift on Intel CPUs in the context where it is used here, 379 379 + // because it runs on a different execution port from the AES instructions. 355 380 .macro _tweak_step_pclmul i 356 381 .if \i == 0 357 382 vpsrldq $(128 - 4*VL/16) / 8, TWEAK0, NEXT_TWEAK0 ··· 391 406 // \i that include at least 0 through 19, then 1000 which signals the last step. 392 407 // 393 408 // This is used to interleave the computation of the next set of tweaks with the 394 394 - // AES en/decryptions, which increases performance in some cases. 409 409 + // AES en/decryptions, which increases performance in some cases. Clobbers V5. 395 410 .macro _tweak_step i 396 411 .if VL == 16 397 412 _tweak_step_mulx \i ··· 428 443 // the last round needs different instructions. 429 444 // 430 445 // An alternative approach would be to roll up all the round loops. We 431 431 - // don't do that because it isn't compatible with caching the round keys 432 432 - // in registers which we do when possible (see below), and also because 433 433 - // it seems unwise to rely *too* heavily on the CPU's branch predictor. 446 446 + // don't do that because (a) it isn't compatible with caching the round 447 447 + // keys in registers which we do when possible (see below), (b) we 448 448 + // interleave the AES rounds with the XTS tweak computation, and (c) it 449 449 + // seems unwise to rely *too* heavily on the CPU's branch predictor. 434 450 lea OFFS-16(KEY, KEYLEN64, 4), KEY 435 451 436 452 // If all 32 SIMD registers are available, cache all the round keys. ··· 458 472 .endif 459 473 .endm 460 474 461 461 - // Do a single round of AES encryption (if \enc==1) or decryption (if \enc==0) 462 462 - // on the block(s) in \data using the round key(s) in \key. The register length 463 463 - // determines the number of AES blocks en/decrypted. 464 464 - .macro _vaes enc, last, key, data 475 475 + // Do a single non-last round of AES encryption (if \enc==1) or decryption (if 476 476 + // \enc==0) on the block(s) in \data using the round key(s) in \key. The 477 477 + // register length determines the number of AES blocks en/decrypted. 478 478 + .macro _vaes enc, key, data 465 479 .if \enc 466 466 - .if \last 467 467 - vaesenclast \key, \data, \data 468 468 - .else 469 480 vaesenc \key, \data, \data 470 470 - .endif 471 471 - .else 472 472 - .if \last 473 473 - vaesdeclast \key, \data, \data 474 481 .else 475 482 vaesdec \key, \data, \data 476 483 .endif 484 484 + .endm 485 485 + 486 486 + // Same as _vaes, but does the last round. 487 487 + .macro _vaeslast enc, key, data 488 488 + .if \enc 489 489 + vaesenclast \key, \data, \data 490 490 + .else 491 491 + vaesdeclast \key, \data, \data 477 492 .endif 478 493 .endm 479 494 480 480 - // Do a single round of AES en/decryption on the block(s) in \data, using the 481 481 - // same key for all block(s). The round key is loaded from the appropriate 482 482 - // register or memory location for round \i. May clobber V4. 483 483 - .macro _vaes_1x enc, last, i, xmm_suffix, data 495 495 + // Do a single non-last round of AES en/decryption on the block(s) in \data, 496 496 + // using the same key for all block(s). The round key is loaded from the 497 497 + // appropriate register or memory location for round \i. May clobber \tmp. 498 498 + .macro _vaes_1x enc, i, xmm_suffix, data, tmp 484 499 .if USE_AVX10 485 485 - _vaes \enc, \last, KEY\i\xmm_suffix, \data 500 500 + _vaes \enc, KEY\i\xmm_suffix, \data 486 501 .else 487 502 .ifnb \xmm_suffix 488 488 - _vaes \enc, \last, (\i-7)*16(KEY), \data 503 503 + _vaes \enc, (\i-7)*16(KEY), \data 489 504 .else 490 490 - _vbroadcast128 (\i-7)*16(KEY), V4 491 491 - _vaes \enc, \last, V4, \data 505 505 + _vbroadcast128 (\i-7)*16(KEY), \tmp 506 506 + _vaes \enc, \tmp, \data 492 507 .endif 493 508 .endif 494 509 .endm 495 510 496 496 - // Do a single round of AES en/decryption on the blocks in registers V0-V3, 497 497 - // using the same key for all blocks. The round key is loaded from the 511 511 + // Do a single non-last round of AES en/decryption on the blocks in registers 512 512 + // V0-V3, using the same key for all blocks. The round key is loaded from the 498 513 // appropriate register or memory location for round \i. In addition, does two 499 499 - // steps of the computation of the next set of tweaks. May clobber V4. 500 500 - .macro _vaes_4x enc, last, i 514 514 + // steps of the computation of the next set of tweaks. May clobber V4 and V5. 515 515 + .macro _vaes_4x enc, i 501 516 .if USE_AVX10 502 517 _tweak_step (2*(\i-5)) 503 503 - _vaes \enc, \last, KEY\i, V0 504 504 - _vaes \enc, \last, KEY\i, V1 518 518 + _vaes \enc, KEY\i, V0 519 519 + _vaes \enc, KEY\i, V1 505 520 _tweak_step (2*(\i-5) + 1) 506 506 - _vaes \enc, \last, KEY\i, V2 507 507 - _vaes \enc, \last, KEY\i, V3 521 521 + _vaes \enc, KEY\i, V2 522 522 + _vaes \enc, KEY\i, V3 508 523 .else 509 524 _vbroadcast128 (\i-7)*16(KEY), V4 510 525 _tweak_step (2*(\i-5)) 511 511 - _vaes \enc, \last, V4, V0 512 512 - _vaes \enc, \last, V4, V1 526 526 + _vaes \enc, V4, V0 527 527 + _vaes \enc, V4, V1 513 528 _tweak_step (2*(\i-5) + 1) 514 514 - _vaes \enc, \last, V4, V2 515 515 - _vaes \enc, \last, V4, V3 529 529 + _vaes \enc, V4, V2 530 530 + _vaes \enc, V4, V3 516 531 .endif 517 532 .endm 518 533 519 534 // Do tweaked AES en/decryption (i.e., XOR with \tweak, then AES en/decrypt, 520 535 // then XOR with \tweak again) of the block(s) in \data. To process a single 521 536 // block, use xmm registers and set \xmm_suffix=_XMM. To process a vector of 522 522 - // length VL, use V* registers and leave \xmm_suffix empty. May clobber V4. 523 523 - .macro _aes_crypt enc, xmm_suffix, tweak, data 537 537 + // length VL, use V* registers and leave \xmm_suffix empty. Clobbers \tmp. 538 538 + .macro _aes_crypt enc, xmm_suffix, tweak, data, tmp 524 539 _xor3 KEY0\xmm_suffix, \tweak, \data 525 540 cmp $24, KEYLEN 526 541 jl .Laes128\@ 527 542 je .Laes192\@ 528 528 - _vaes_1x \enc, 0, 1, \xmm_suffix, \data 529 529 - _vaes_1x \enc, 0, 2, \xmm_suffix, \data 543 543 + _vaes_1x \enc, 1, \xmm_suffix, \data, tmp=\tmp 544 544 + _vaes_1x \enc, 2, \xmm_suffix, \data, tmp=\tmp 530 545 .Laes192\@: 531 531 - _vaes_1x \enc, 0, 3, \xmm_suffix, \data 532 532 - _vaes_1x \enc, 0, 4, \xmm_suffix, \data 546 546 + _vaes_1x \enc, 3, \xmm_suffix, \data, tmp=\tmp 547 547 + _vaes_1x \enc, 4, \xmm_suffix, \data, tmp=\tmp 533 548 .Laes128\@: 534 534 - _vaes_1x \enc, 0, 5, \xmm_suffix, \data 535 535 - _vaes_1x \enc, 0, 6, \xmm_suffix, \data 536 536 - _vaes_1x \enc, 0, 7, \xmm_suffix, \data 537 537 - _vaes_1x \enc, 0, 8, \xmm_suffix, \data 538 538 - _vaes_1x \enc, 0, 9, \xmm_suffix, \data 539 539 - _vaes_1x \enc, 0, 10, \xmm_suffix, \data 540 540 - _vaes_1x \enc, 0, 11, \xmm_suffix, \data 541 541 - _vaes_1x \enc, 0, 12, \xmm_suffix, \data 542 542 - _vaes_1x \enc, 0, 13, \xmm_suffix, \data 543 543 - _vaes_1x \enc, 1, 14, \xmm_suffix, \data 544 544 - _vpxor \tweak, \data, \data 549 549 + .irp i, 5,6,7,8,9,10,11,12,13 550 550 + _vaes_1x \enc, \i, \xmm_suffix, \data, tmp=\tmp 551 551 + .endr 552 552 + .if USE_AVX10 553 553 + vpxord KEY14\xmm_suffix, \tweak, \tmp 554 554 + .else 555 555 + .ifnb \xmm_suffix 556 556 + vpxor 7*16(KEY), \tweak, \tmp 557 557 + .else 558 558 + _vbroadcast128 7*16(KEY), \tmp 559 559 + vpxor \tweak, \tmp, \tmp 560 560 + .endif 561 561 + .endif 562 562 + _vaeslast \enc, \tmp, \data 545 563 .endm 546 564 547 565 .macro _aes_xts_crypt enc ··· 571 581 // Compute the first set of tweaks TWEAK[0-3]. 572 582 _compute_first_set_of_tweaks 573 583 574 574 - sub $4*VL, LEN 584 584 + add $-4*VL, LEN // shorter than 'sub 4*VL' when VL=32 575 585 jl .Lhandle_remainder\@ 576 586 577 587 .Lmain_loop\@: ··· 579 589 580 590 // XOR each source block with its tweak and the zero-th round key. 581 591 .if USE_AVX10 582 582 - vmovdqu8 0*VL(SRC), V0 583 583 - vmovdqu8 1*VL(SRC), V1 584 584 - vmovdqu8 2*VL(SRC), V2 585 585 - vmovdqu8 3*VL(SRC), V3 592 592 + _vmovdqu 0*VL(SRC), V0 593 593 + _vmovdqu 1*VL(SRC), V1 594 594 + _vmovdqu 2*VL(SRC), V2 595 595 + _vmovdqu 3*VL(SRC), V3 586 596 vpternlogd $0x96, TWEAK0, KEY0, V0 587 597 vpternlogd $0x96, TWEAK1, KEY0, V1 588 598 vpternlogd $0x96, TWEAK2, KEY0, V2 ··· 602 612 je .Laes192\@ 603 613 // Do all the AES rounds on the data blocks, interleaved with 604 614 // the computation of the next set of tweaks. 605 605 - _vaes_4x \enc, 0, 1 606 606 - _vaes_4x \enc, 0, 2 615 615 + _vaes_4x \enc, 1 616 616 + _vaes_4x \enc, 2 607 617 .Laes192\@: 608 608 - _vaes_4x \enc, 0, 3 609 609 - _vaes_4x \enc, 0, 4 618 618 + _vaes_4x \enc, 3 619 619 + _vaes_4x \enc, 4 610 620 .Laes128\@: 611 611 - _vaes_4x \enc, 0, 5 612 612 - _vaes_4x \enc, 0, 6 613 613 - _vaes_4x \enc, 0, 7 614 614 - _vaes_4x \enc, 0, 8 615 615 - _vaes_4x \enc, 0, 9 616 616 - _vaes_4x \enc, 0, 10 617 617 - _vaes_4x \enc, 0, 11 618 618 - _vaes_4x \enc, 0, 12 619 619 - _vaes_4x \enc, 0, 13 620 620 - _vaes_4x \enc, 1, 14 621 621 - 622 622 - // XOR in the tweaks again. 623 623 - _vpxor TWEAK0, V0, V0 624 624 - _vpxor TWEAK1, V1, V1 625 625 - _vpxor TWEAK2, V2, V2 626 626 - _vpxor TWEAK3, V3, V3 621 621 + .irp i, 5,6,7,8,9,10,11,12,13 622 622 + _vaes_4x \enc, \i 623 623 + .endr 624 624 + // Do the last AES round, then XOR the results with the tweaks again. 625 625 + // Reduce latency by doing the XOR before the vaesenclast, utilizing the 626 626 + // property vaesenclast(key, a) ^ b == vaesenclast(key ^ b, a) 627 627 + // (and likewise for vaesdeclast). 628 628 + .if USE_AVX10 629 629 + _tweak_step 18 630 630 + _tweak_step 19 631 631 + vpxord TWEAK0, KEY14, V4 632 632 + vpxord TWEAK1, KEY14, V5 633 633 + _vaeslast \enc, V4, V0 634 634 + _vaeslast \enc, V5, V1 635 635 + vpxord TWEAK2, KEY14, V4 636 636 + vpxord TWEAK3, KEY14, V5 637 637 + _vaeslast \enc, V4, V2 638 638 + _vaeslast \enc, V5, V3 639 639 + .else 640 640 + _vbroadcast128 7*16(KEY), V4 641 641 + _tweak_step 18 // uses V5 642 642 + _tweak_step 19 // uses V5 643 643 + vpxor TWEAK0, V4, V5 644 644 + _vaeslast \enc, V5, V0 645 645 + vpxor TWEAK1, V4, V5 646 646 + _vaeslast \enc, V5, V1 647 647 + vpxor TWEAK2, V4, V5 648 648 + vpxor TWEAK3, V4, V4 649 649 + _vaeslast \enc, V5, V2 650 650 + _vaeslast \enc, V4, V3 651 651 + .endif 627 652 628 653 // Store the destination blocks. 629 654 _vmovdqu V0, 0*VL(DST) ··· 649 644 // Finish computing the next set of tweaks. 650 645 _tweak_step 1000 651 646 652 652 - add $4*VL, SRC 653 653 - add $4*VL, DST 654 654 - sub $4*VL, LEN 647 647 + sub $-4*VL, SRC // shorter than 'add 4*VL' when VL=32 648 648 + sub $-4*VL, DST 649 649 + add $-4*VL, LEN 655 650 jge .Lmain_loop\@ 656 651 657 652 // Check for the uncommon case where the data length isn't a multiple of ··· 675 670 jl .Lvec_at_a_time_done\@ 676 671 .Lvec_at_a_time\@: 677 672 _vmovdqu (SRC), V0 678 678 - _aes_crypt \enc, , TWEAK0, V0 673 673 + _aes_crypt \enc, , TWEAK0, V0, tmp=V1 679 674 _vmovdqu V0, (DST) 680 675 _next_tweakvec TWEAK0, V0, V1, TWEAK0 681 676 add $VL, SRC ··· 692 687 jl .Lblock_at_a_time_done\@ 693 688 .Lblock_at_a_time\@: 694 689 vmovdqu (SRC), %xmm0 695 695 - _aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0 690 690 + _aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0, tmp=%xmm1 696 691 vmovdqu %xmm0, (DST) 697 692 _next_tweak TWEAK0_XMM, %xmm0, TWEAK0_XMM 698 693 add $16, SRC ··· 720 715 // Do it now by advancing the tweak and decrypting the last full block. 721 716 _next_tweak TWEAK0_XMM, %xmm0, TWEAK1_XMM 722 717 vmovdqu (SRC), %xmm0 723 723 - _aes_crypt \enc, _XMM, TWEAK1_XMM, %xmm0 718 718 + _aes_crypt \enc, _XMM, TWEAK1_XMM, %xmm0, tmp=%xmm1 724 719 .endif 725 720 726 721 .if USE_AVX10 ··· 763 758 vpblendvb %xmm3, %xmm0, %xmm1, %xmm0 764 759 .endif 765 760 // En/decrypt again and store the last full block. 766 766 - _aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0 761 761 + _aes_crypt \enc, _XMM, TWEAK0_XMM, %xmm0, tmp=%xmm1 767 762 vmovdqu %xmm0, (DST) 768 763 jmp .Ldone\@ 769 764 .endm 770 765 771 766 // void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key, 772 767 // u8 iv[AES_BLOCK_SIZE]); 768 768 + // 769 769 + // Encrypt |iv| using the AES key |tweak_key| to get the first tweak. Assumes 770 770 + // that the CPU supports AES-NI and AVX, but not necessarily VAES or AVX10. 773 771 SYM_TYPED_FUNC_START(aes_xts_encrypt_iv) 774 774 - vmovdqu (%rsi), %xmm0 775 775 - vpxor (%rdi), %xmm0, %xmm0 776 776 - movl 480(%rdi), %eax // AES key length 777 777 - lea -16(%rdi, %rax, 4), %rdi 778 778 - cmp $24, %eax 772 772 + .set TWEAK_KEY, %rdi 773 773 + .set IV, %rsi 774 774 + .set KEYLEN, %eax 775 775 + .set KEYLEN64, %rax 776 776 + 777 777 + vmovdqu (IV), %xmm0 778 778 + vpxor (TWEAK_KEY), %xmm0, %xmm0 779 779 + movl 480(TWEAK_KEY), KEYLEN 780 780 + lea -16(TWEAK_KEY, KEYLEN64, 4), TWEAK_KEY 781 781 + cmp $24, KEYLEN 779 782 jl .Lencrypt_iv_aes128 780 783 je .Lencrypt_iv_aes192 781 781 - vaesenc -6*16(%rdi), %xmm0, %xmm0 782 782 - vaesenc -5*16(%rdi), %xmm0, %xmm0 784 784 + vaesenc -6*16(TWEAK_KEY), %xmm0, %xmm0 785 785 + vaesenc -5*16(TWEAK_KEY), %xmm0, %xmm0 783 786 .Lencrypt_iv_aes192: 784 784 - vaesenc -4*16(%rdi), %xmm0, %xmm0 785 785 - vaesenc -3*16(%rdi), %xmm0, %xmm0 787 787 + vaesenc -4*16(TWEAK_KEY), %xmm0, %xmm0 788 788 + vaesenc -3*16(TWEAK_KEY), %xmm0, %xmm0 786 789 .Lencrypt_iv_aes128: 787 787 - vaesenc -2*16(%rdi), %xmm0, %xmm0 788 788 - vaesenc -1*16(%rdi), %xmm0, %xmm0 789 789 - vaesenc 0*16(%rdi), %xmm0, %xmm0 790 790 - vaesenc 1*16(%rdi), %xmm0, %xmm0 791 791 - vaesenc 2*16(%rdi), %xmm0, %xmm0 792 792 - vaesenc 3*16(%rdi), %xmm0, %xmm0 793 793 - vaesenc 4*16(%rdi), %xmm0, %xmm0 794 794 - vaesenc 5*16(%rdi), %xmm0, %xmm0 795 795 - vaesenc 6*16(%rdi), %xmm0, %xmm0 796 796 - vaesenclast 7*16(%rdi), %xmm0, %xmm0 797 797 - vmovdqu %xmm0, (%rsi) 790 790 + .irp i, -2,-1,0,1,2,3,4,5,6 791 791 + vaesenc \i*16(TWEAK_KEY), %xmm0, %xmm0 792 792 + .endr 793 793 + vaesenclast 7*16(TWEAK_KEY), %xmm0, %xmm0 794 794 + vmovdqu %xmm0, (IV) 798 795 RET 799 796 SYM_FUNC_END(aes_xts_encrypt_iv) 800 797 801 798 // Below are the actual AES-XTS encryption and decryption functions, 802 799 // instantiated from the above macro. They all have the following prototype: 803 800 // 804 804 - // void (*xts_asm_func)(const struct crypto_aes_ctx *key, 805 805 - // const u8 *src, u8 *dst, unsigned int len, 806 806 - // u8 tweak[AES_BLOCK_SIZE]); 801 801 + // void (*xts_crypt_func)(const struct crypto_aes_ctx *key, 802 802 + // const u8 *src, u8 *dst, int len, 803 803 + // u8 tweak[AES_BLOCK_SIZE]); 807 804 // 808 805 // |key| is the data key. |tweak| contains the next tweak; the encryption of 809 806 // the original IV with the tweak key was already done. This function supports

+5 -5

arch/x86/crypto/aesni-intel_glue.c

reviewed

··· 505 505 typedef void (*xts_encrypt_iv_func)(const struct crypto_aes_ctx *tweak_key, 506 506 u8 iv[AES_BLOCK_SIZE]); 507 507 typedef void (*xts_crypt_func)(const struct crypto_aes_ctx *key, 508 508 - const u8 *src, u8 *dst, unsigned int len, 508 508 + const u8 *src, u8 *dst, int len, 509 509 u8 tweak[AES_BLOCK_SIZE]); 510 510 511 511 /* This handles cases where the source and/or destination span pages. */ ··· 624 624 } 625 625 626 626 static void aesni_xts_encrypt(const struct crypto_aes_ctx *key, 627 627 - const u8 *src, u8 *dst, unsigned int len, 627 627 + const u8 *src, u8 *dst, int len, 628 628 u8 tweak[AES_BLOCK_SIZE]) 629 629 { 630 630 aesni_xts_enc(key, dst, src, len, tweak); 631 631 } 632 632 633 633 static void aesni_xts_decrypt(const struct crypto_aes_ctx *key, 634 634 - const u8 *src, u8 *dst, unsigned int len, 634 634 + const u8 *src, u8 *dst, int len, 635 635 u8 tweak[AES_BLOCK_SIZE]) 636 636 { 637 637 aesni_xts_dec(key, dst, src, len, tweak); ··· 790 790 \ 791 791 asmlinkage void \ 792 792 aes_xts_encrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \ 793 793 - u8 *dst, unsigned int len, u8 tweak[AES_BLOCK_SIZE]); \ 793 793 + u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \ 794 794 asmlinkage void \ 795 795 aes_xts_decrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \ 796 796 - u8 *dst, unsigned int len, u8 tweak[AES_BLOCK_SIZE]); \ 796 796 + u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \ 797 797 \ 798 798 static int xts_encrypt_##suffix(struct skcipher_request *req) \ 799 799 { \

-1

arch/x86/crypto/blowfish_glue.c

reviewed

··· 94 94 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 95 95 .cra_blocksize = BF_BLOCK_SIZE, 96 96 .cra_ctxsize = sizeof(struct bf_ctx), 97 97 - .cra_alignmask = 0, 98 97 .cra_module = THIS_MODULE, 99 98 .cra_u = { 100 99 .cipher = {

-1

arch/x86/crypto/camellia_glue.c

reviewed

··· 1313 1313 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 1314 1314 .cra_blocksize = CAMELLIA_BLOCK_SIZE, 1315 1315 .cra_ctxsize = sizeof(struct camellia_ctx), 1316 1316 - .cra_alignmask = 0, 1317 1316 .cra_module = THIS_MODULE, 1318 1317 .cra_u = { 1319 1318 .cipher = {

-1

arch/x86/crypto/des3_ede_glue.c

reviewed

··· 291 291 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 292 292 .cra_blocksize = DES3_EDE_BLOCK_SIZE, 293 293 .cra_ctxsize = sizeof(struct des3_ede_x86_ctx), 294 294 - .cra_alignmask = 0, 295 294 .cra_module = THIS_MODULE, 296 295 .cra_u = { 297 296 .cipher = {

-1

arch/x86/crypto/twofish_glue.c

reviewed

··· 68 68 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 69 69 .cra_blocksize = TF_BLOCK_SIZE, 70 70 .cra_ctxsize = sizeof(struct twofish_ctx), 71 71 - .cra_alignmask = 0, 72 71 .cra_module = THIS_MODULE, 73 72 .cra_u = { 74 73 .cipher = {

-18

crypto/Kconfig

reviewed

··· 684 684 685 685 See https://eprint.iacr.org/2021/1441 686 686 687 687 - config CRYPTO_KEYWRAP 688 688 - tristate "KW (AES Key Wrap)" 689 689 - select CRYPTO_SKCIPHER 690 690 - select CRYPTO_MANAGER 691 691 - help 692 692 - KW (AES Key Wrap) authenticated encryption mode (NIST SP800-38F 693 693 - and RFC3394) without padding. 694 694 - 695 687 config CRYPTO_LRW 696 688 tristate "LRW (Liskov Rivest Wagner)" 697 689 select CRYPTO_LIB_GF128MUL ··· 1020 1028 References: 1021 1029 https://tc26.ru/upload/iblock/fed/feddbb4d26b685903faa2ba11aea43f6.pdf 1022 1030 https://tools.ietf.org/html/rfc6986 1023 1023 - 1024 1024 - config CRYPTO_VMAC 1025 1025 - tristate "VMAC" 1026 1026 - select CRYPTO_HASH 1027 1027 - select CRYPTO_MANAGER 1028 1028 - help 1029 1029 - VMAC is a message authentication algorithm designed for 1030 1030 - very high speed on 64-bit architectures. 1031 1031 - 1032 1032 - See https://fastcrypto.org/vmac for further information. 1033 1031 1034 1032 config CRYPTO_WP512 1035 1033 tristate "Whirlpool"

-2

crypto/Makefile

reviewed

··· 69 69 obj-$(CONFIG_CRYPTO_USER) += crypto_user.o 70 70 obj-$(CONFIG_CRYPTO_CMAC) += cmac.o 71 71 obj-$(CONFIG_CRYPTO_HMAC) += hmac.o 72 72 - obj-$(CONFIG_CRYPTO_VMAC) += vmac.o 73 72 obj-$(CONFIG_CRYPTO_XCBC) += xcbc.o 74 73 obj-$(CONFIG_CRYPTO_NULL2) += crypto_null.o 75 74 obj-$(CONFIG_CRYPTO_MD4) += md4.o ··· 94 95 obj-$(CONFIG_CRYPTO_CTR) += ctr.o 95 96 obj-$(CONFIG_CRYPTO_XCTR) += xctr.o 96 97 obj-$(CONFIG_CRYPTO_HCTR2) += hctr2.o 97 97 - obj-$(CONFIG_CRYPTO_KEYWRAP) += keywrap.o 98 98 obj-$(CONFIG_CRYPTO_ADIANTUM) += adiantum.o 99 99 obj-$(CONFIG_CRYPTO_NHPOLY1305) += nhpoly1305.o 100 100 obj-$(CONFIG_CRYPTO_GCM) += gcm.o

-2

crypto/aegis128-core.c

reviewed

··· 516 516 517 517 .base.cra_blocksize = 1, 518 518 .base.cra_ctxsize = sizeof(struct aegis_ctx), 519 519 - .base.cra_alignmask = 0, 520 519 .base.cra_priority = 100, 521 520 .base.cra_name = "aegis128", 522 521 .base.cra_driver_name = "aegis128-generic", ··· 534 535 535 536 .base.cra_blocksize = 1, 536 537 .base.cra_ctxsize = sizeof(struct aegis_ctx), 537 537 - .base.cra_alignmask = 0, 538 538 .base.cra_priority = 200, 539 539 .base.cra_name = "aegis128", 540 540 .base.cra_driver_name = "aegis128-simd",

+87 -71

crypto/ahash.c

reviewed

··· 27 27 28 28 #define CRYPTO_ALG_TYPE_AHASH_MASK 0x0000000e 29 29 30 30 + struct crypto_hash_walk { 31 31 + char *data; 32 32 + 33 33 + unsigned int offset; 34 34 + unsigned int flags; 35 35 + 36 36 + struct page *pg; 37 37 + unsigned int entrylen; 38 38 + 39 39 + unsigned int total; 40 40 + struct scatterlist *sg; 41 41 + }; 42 42 + 43 43 + static int hash_walk_next(struct crypto_hash_walk *walk) 44 44 + { 45 45 + unsigned int offset = walk->offset; 46 46 + unsigned int nbytes = min(walk->entrylen, 47 47 + ((unsigned int)(PAGE_SIZE)) - offset); 48 48 + 49 49 + walk->data = kmap_local_page(walk->pg); 50 50 + walk->data += offset; 51 51 + walk->entrylen -= nbytes; 52 52 + return nbytes; 53 53 + } 54 54 + 55 55 + static int hash_walk_new_entry(struct crypto_hash_walk *walk) 56 56 + { 57 57 + struct scatterlist *sg; 58 58 + 59 59 + sg = walk->sg; 60 60 + walk->offset = sg->offset; 61 61 + walk->pg = sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT); 62 62 + walk->offset = offset_in_page(walk->offset); 63 63 + walk->entrylen = sg->length; 64 64 + 65 65 + if (walk->entrylen > walk->total) 66 66 + walk->entrylen = walk->total; 67 67 + walk->total -= walk->entrylen; 68 68 + 69 69 + return hash_walk_next(walk); 70 70 + } 71 71 + 72 72 + static int crypto_hash_walk_first(struct ahash_request *req, 73 73 + struct crypto_hash_walk *walk) 74 74 + { 75 75 + walk->total = req->nbytes; 76 76 + 77 77 + if (!walk->total) { 78 78 + walk->entrylen = 0; 79 79 + return 0; 80 80 + } 81 81 + 82 82 + walk->sg = req->src; 83 83 + walk->flags = req->base.flags; 84 84 + 85 85 + return hash_walk_new_entry(walk); 86 86 + } 87 87 + 88 88 + static int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err) 89 89 + { 90 90 + walk->data -= walk->offset; 91 91 + 92 92 + kunmap_local(walk->data); 93 93 + crypto_yield(walk->flags); 94 94 + 95 95 + if (err) 96 96 + return err; 97 97 + 98 98 + if (walk->entrylen) { 99 99 + walk->offset = 0; 100 100 + walk->pg++; 101 101 + return hash_walk_next(walk); 102 102 + } 103 103 + 104 104 + if (!walk->total) 105 105 + return 0; 106 106 + 107 107 + walk->sg = sg_next(walk->sg); 108 108 + 109 109 + return hash_walk_new_entry(walk); 110 110 + } 111 111 + 112 112 + static inline int crypto_hash_walk_last(struct crypto_hash_walk *walk) 113 113 + { 114 114 + return !(walk->entrylen | walk->total); 115 115 + } 116 116 + 30 117 /* 31 118 * For an ahash tfm that is using an shash algorithm (instead of an ahash 32 119 * algorithm), this returns the underlying shash tfm. ··· 223 136 224 137 return 0; 225 138 } 226 226 - 227 227 - static int hash_walk_next(struct crypto_hash_walk *walk) 228 228 - { 229 229 - unsigned int offset = walk->offset; 230 230 - unsigned int nbytes = min(walk->entrylen, 231 231 - ((unsigned int)(PAGE_SIZE)) - offset); 232 232 - 233 233 - walk->data = kmap_local_page(walk->pg); 234 234 - walk->data += offset; 235 235 - walk->entrylen -= nbytes; 236 236 - return nbytes; 237 237 - } 238 238 - 239 239 - static int hash_walk_new_entry(struct crypto_hash_walk *walk) 240 240 - { 241 241 - struct scatterlist *sg; 242 242 - 243 243 - sg = walk->sg; 244 244 - walk->offset = sg->offset; 245 245 - walk->pg = sg_page(walk->sg) + (walk->offset >> PAGE_SHIFT); 246 246 - walk->offset = offset_in_page(walk->offset); 247 247 - walk->entrylen = sg->length; 248 248 - 249 249 - if (walk->entrylen > walk->total) 250 250 - walk->entrylen = walk->total; 251 251 - walk->total -= walk->entrylen; 252 252 - 253 253 - return hash_walk_next(walk); 254 254 - } 255 255 - 256 256 - int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err) 257 257 - { 258 258 - walk->data -= walk->offset; 259 259 - 260 260 - kunmap_local(walk->data); 261 261 - crypto_yield(walk->flags); 262 262 - 263 263 - if (err) 264 264 - return err; 265 265 - 266 266 - if (walk->entrylen) { 267 267 - walk->offset = 0; 268 268 - walk->pg++; 269 269 - return hash_walk_next(walk); 270 270 - } 271 271 - 272 272 - if (!walk->total) 273 273 - return 0; 274 274 - 275 275 - walk->sg = sg_next(walk->sg); 276 276 - 277 277 - return hash_walk_new_entry(walk); 278 278 - } 279 279 - EXPORT_SYMBOL_GPL(crypto_hash_walk_done); 280 280 - 281 281 - int crypto_hash_walk_first(struct ahash_request *req, 282 282 - struct crypto_hash_walk *walk) 283 283 - { 284 284 - walk->total = req->nbytes; 285 285 - 286 286 - if (!walk->total) { 287 287 - walk->entrylen = 0; 288 288 - return 0; 289 289 - } 290 290 - 291 291 - walk->sg = req->src; 292 292 - walk->flags = req->base.flags; 293 293 - 294 294 - return hash_walk_new_entry(walk); 295 295 - } 296 296 - EXPORT_SYMBOL_GPL(crypto_hash_walk_first); 297 139 298 140 static int ahash_nosetkey(struct crypto_ahash *tfm, const u8 *key, 299 141 unsigned int keylen)

+19 -12

crypto/algapi.c

reviewed

··· 407 407 int crypto_register_alg(struct crypto_alg *alg) 408 408 { 409 409 struct crypto_larval *larval; 410 410 + bool test_started = false; 410 411 LIST_HEAD(algs_to_put); 411 412 int err; 412 413 ··· 419 418 down_write(&crypto_alg_sem); 420 419 larval = __crypto_register_alg(alg, &algs_to_put); 421 420 if (!IS_ERR_OR_NULL(larval)) { 422 422 - bool test_started = crypto_boot_test_finished(); 423 423 - 421 421 + test_started = crypto_boot_test_finished(); 424 422 larval->test_started = test_started; 425 425 - if (test_started) 426 426 - crypto_schedule_test(larval); 427 423 } 428 424 up_write(&crypto_alg_sem); 429 425 430 426 if (IS_ERR(larval)) 431 427 return PTR_ERR(larval); 432 432 - crypto_remove_final(&algs_to_put); 428 428 + 429 429 + if (test_started) 430 430 + crypto_schedule_test(larval); 431 431 + else 432 432 + crypto_remove_final(&algs_to_put); 433 433 + 433 434 return 0; 434 435 } 435 436 EXPORT_SYMBOL_GPL(crypto_register_alg); ··· 645 642 larval = __crypto_register_alg(&inst->alg, &algs_to_put); 646 643 if (IS_ERR(larval)) 647 644 goto unlock; 648 648 - else if (larval) { 645 645 + else if (larval) 649 646 larval->test_started = true; 650 650 - crypto_schedule_test(larval); 651 651 - } 652 647 653 648 hlist_add_head(&inst->list, &tmpl->instances); 654 649 inst->tmpl = tmpl; ··· 656 655 657 656 if (IS_ERR(larval)) 658 657 return PTR_ERR(larval); 659 659 - crypto_remove_final(&algs_to_put); 658 658 + 659 659 + if (larval) 660 660 + crypto_schedule_test(larval); 661 661 + else 662 662 + crypto_remove_final(&algs_to_put); 663 663 + 660 664 return 0; 661 665 } 662 666 EXPORT_SYMBOL_GPL(crypto_register_instance); ··· 1022 1016 if (IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)) 1023 1017 return; 1024 1018 1019 1019 + set_crypto_boot_test_finished(); 1020 1020 + 1025 1021 for (;;) { 1026 1022 struct crypto_larval *larval = NULL; 1027 1023 struct crypto_alg *q; ··· 1046 1038 1047 1039 l->test_started = true; 1048 1040 larval = l; 1049 1049 - crypto_schedule_test(larval); 1050 1041 break; 1051 1042 } 1052 1043 ··· 1053 1046 1054 1047 if (!larval) 1055 1048 break; 1056 1056 - } 1057 1049 1058 1058 - set_crypto_boot_test_finished(); 1050 1050 + crypto_schedule_test(larval); 1051 1051 + } 1059 1052 } 1060 1053 1061 1054 static int __init crypto_algapi_init(void)

+5 -9

crypto/anubis.c

reviewed

··· 33 33 #include <linux/init.h> 34 34 #include <linux/module.h> 35 35 #include <linux/mm.h> 36 36 - #include <asm/byteorder.h> 36 36 + #include <linux/unaligned.h> 37 37 #include <linux/types.h> 38 38 39 39 #define ANUBIS_MIN_KEY_SIZE 16 ··· 463 463 unsigned int key_len) 464 464 { 465 465 struct anubis_ctx *ctx = crypto_tfm_ctx(tfm); 466 466 - const __be32 *key = (const __be32 *)in_key; 467 466 int N, R, i, r; 468 467 u32 kappa[ANUBIS_MAX_N]; 469 468 u32 inter[ANUBIS_MAX_N]; ··· 481 482 482 483 /* * map cipher key to initial key state (mu): */ 483 484 for (i = 0; i < N; i++) 484 484 - kappa[i] = be32_to_cpu(key[i]); 485 485 + kappa[i] = get_unaligned_be32(&in_key[4 * i]); 485 486 486 487 /* 487 488 * generate R + 1 round keys: ··· 569 570 } 570 571 571 572 static void anubis_crypt(u32 roundKey[ANUBIS_MAX_ROUNDS + 1][4], 572 572 - u8 *ciphertext, const u8 *plaintext, const int R) 573 573 + u8 *dst, const u8 *src, const int R) 573 574 { 574 574 - const __be32 *src = (const __be32 *)plaintext; 575 575 - __be32 *dst = (__be32 *)ciphertext; 576 575 int i, r; 577 576 u32 state[4]; 578 577 u32 inter[4]; ··· 580 583 * and add initial round key (sigma[K^0]): 581 584 */ 582 585 for (i = 0; i < 4; i++) 583 583 - state[i] = be32_to_cpu(src[i]) ^ roundKey[0][i]; 586 586 + state[i] = get_unaligned_be32(&src[4 * i]) ^ roundKey[0][i]; 584 587 585 588 /* 586 589 * R - 1 full rounds: ··· 651 654 */ 652 655 653 656 for (i = 0; i < 4; i++) 654 654 - dst[i] = cpu_to_be32(inter[i]); 657 657 + put_unaligned_be32(inter[i], &dst[4 * i]); 655 658 } 656 659 657 660 static void anubis_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) ··· 672 675 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 673 676 .cra_blocksize = ANUBIS_BLOCK_SIZE, 674 677 .cra_ctxsize = sizeof (struct anubis_ctx), 675 675 - .cra_alignmask = 3, 676 678 .cra_module = THIS_MODULE, 677 679 .cra_u = { .cipher = { 678 680 .cia_min_keysize = ANUBIS_MIN_KEY_SIZE,

+17 -20

crypto/aria_generic.c

reviewed

··· 15 15 */ 16 16 17 17 #include <crypto/aria.h> 18 18 + #include <linux/unaligned.h> 18 19 19 20 static const u32 key_rc[20] = { 20 21 0x517cc1b7, 0x27220a94, 0xfe13abe8, 0xfa9a6ee0, ··· 28 27 static void aria_set_encrypt_key(struct aria_ctx *ctx, const u8 *in_key, 29 28 unsigned int key_len) 30 29 { 31 31 - const __be32 *key = (const __be32 *)in_key; 32 30 u32 w0[4], w1[4], w2[4], w3[4]; 33 31 u32 reg0, reg1, reg2, reg3; 34 32 const u32 *ck; ··· 35 35 36 36 ck = &key_rc[(key_len - 16) / 2]; 37 37 38 38 - w0[0] = be32_to_cpu(key[0]); 39 39 - w0[1] = be32_to_cpu(key[1]); 40 40 - w0[2] = be32_to_cpu(key[2]); 41 41 - w0[3] = be32_to_cpu(key[3]); 38 38 + w0[0] = get_unaligned_be32(&in_key[0]); 39 39 + w0[1] = get_unaligned_be32(&in_key[4]); 40 40 + w0[2] = get_unaligned_be32(&in_key[8]); 41 41 + w0[3] = get_unaligned_be32(&in_key[12]); 42 42 43 43 reg0 = w0[0] ^ ck[0]; 44 44 reg1 = w0[1] ^ ck[1]; ··· 48 48 aria_subst_diff_odd(&reg0, &reg1, &reg2, &reg3); 49 49 50 50 if (key_len > 16) { 51 51 - w1[0] = be32_to_cpu(key[4]); 52 52 - w1[1] = be32_to_cpu(key[5]); 51 51 + w1[0] = get_unaligned_be32(&in_key[16]); 52 52 + w1[1] = get_unaligned_be32(&in_key[20]); 53 53 if (key_len > 24) { 54 54 - w1[2] = be32_to_cpu(key[6]); 55 55 - w1[3] = be32_to_cpu(key[7]); 54 54 + w1[2] = get_unaligned_be32(&in_key[24]); 55 55 + w1[3] = get_unaligned_be32(&in_key[28]); 56 56 } else { 57 57 w1[2] = 0; 58 58 w1[3] = 0; ··· 195 195 static void __aria_crypt(struct aria_ctx *ctx, u8 *out, const u8 *in, 196 196 u32 key[][ARIA_RD_KEY_WORDS]) 197 197 { 198 198 - const __be32 *src = (const __be32 *)in; 199 199 - __be32 *dst = (__be32 *)out; 200 198 u32 reg0, reg1, reg2, reg3; 201 199 int rounds, rkidx = 0; 202 200 203 201 rounds = ctx->rounds; 204 202 205 205 - reg0 = be32_to_cpu(src[0]); 206 206 - reg1 = be32_to_cpu(src[1]); 207 207 - reg2 = be32_to_cpu(src[2]); 208 208 - reg3 = be32_to_cpu(src[3]); 203 203 + reg0 = get_unaligned_be32(&in[0]); 204 204 + reg1 = get_unaligned_be32(&in[4]); 205 205 + reg2 = get_unaligned_be32(&in[8]); 206 206 + reg3 = get_unaligned_be32(&in[12]); 209 207 210 208 aria_add_round_key(key[rkidx], &reg0, &reg1, &reg2, &reg3); 211 209 rkidx++; ··· 239 241 (u8)(s1[get_u8(reg3, 2)]), 240 242 (u8)(s2[get_u8(reg3, 3)])); 241 243 242 242 - dst[0] = cpu_to_be32(reg0); 243 243 - dst[1] = cpu_to_be32(reg1); 244 244 - dst[2] = cpu_to_be32(reg2); 245 245 - dst[3] = cpu_to_be32(reg3); 244 244 + put_unaligned_be32(reg0, &out[0]); 245 245 + put_unaligned_be32(reg1, &out[4]); 246 246 + put_unaligned_be32(reg2, &out[8]); 247 247 + put_unaligned_be32(reg3, &out[12]); 246 248 } 247 249 248 250 void aria_encrypt(void *_ctx, u8 *out, const u8 *in) ··· 282 284 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 283 285 .cra_blocksize = ARIA_BLOCK_SIZE, 284 286 .cra_ctxsize = sizeof(struct aria_ctx), 285 285 - .cra_alignmask = 3, 286 287 .cra_module = THIS_MODULE, 287 288 .cra_u = { 288 289 .cipher = {

-10

crypto/asymmetric_keys/asymmetric_type.c

reviewed

··· 18 18 #include "asymmetric_keys.h" 19 19 20 20 21 21 - const char *const key_being_used_for[NR__KEY_BEING_USED_FOR] = { 22 22 - [VERIFYING_MODULE_SIGNATURE] = "mod sig", 23 23 - [VERIFYING_FIRMWARE_SIGNATURE] = "firmware sig", 24 24 - [VERIFYING_KEXEC_PE_SIGNATURE] = "kexec PE sig", 25 25 - [VERIFYING_KEY_SIGNATURE] = "key sig", 26 26 - [VERIFYING_KEY_SELF_SIGNATURE] = "key self sig", 27 27 - [VERIFYING_UNSPECIFIED_SIGNATURE] = "unspec sig", 28 28 - }; 29 29 - EXPORT_SYMBOL_GPL(key_being_used_for); 30 30 - 31 21 static LIST_HEAD(asymmetric_key_parsers); 32 22 static DECLARE_RWSEM(asymmetric_key_parsers_sem); 33 23

+2 -2

crypto/fips.c

reviewed

··· 12 12 #include <linux/kernel.h> 13 13 #include <linux/sysctl.h> 14 14 #include <linux/notifier.h> 15 15 + #include <linux/string_choices.h> 15 16 #include <generated/utsrelease.h> 16 17 17 18 int fips_enabled; ··· 25 24 static int fips_enable(char *str) 26 25 { 27 26 fips_enabled = !!simple_strtol(str, NULL, 0); 28 28 - printk(KERN_INFO "fips mode: %s\n", 29 29 - fips_enabled ? "enabled" : "disabled"); 27 27 + pr_info("fips mode: %s\n", str_enabled_disabled(fips_enabled)); 30 28 return 1; 31 29 } 32 30

-320

crypto/keywrap.c

reviewed

··· 1 1 - /* 2 2 - * Key Wrapping: RFC3394 / NIST SP800-38F 3 3 - * 4 4 - * Copyright (C) 2015, Stephan Mueller <smueller@chronox.de> 5 5 - * 6 6 - * Redistribution and use in source and binary forms, with or without 7 7 - * modification, are permitted provided that the following conditions 8 8 - * are met: 9 9 - * 1. Redistributions of source code must retain the above copyright 10 10 - * notice, and the entire permission notice in its entirety, 11 11 - * including the disclaimer of warranties. 12 12 - * 2. Redistributions in binary form must reproduce the above copyright 13 13 - * notice, this list of conditions and the following disclaimer in the 14 14 - * documentation and/or other materials provided with the distribution. 15 15 - * 3. The name of the author may not be used to endorse or promote 16 16 - * products derived from this software without specific prior 17 17 - * written permission. 18 18 - * 19 19 - * ALTERNATIVELY, this product may be distributed under the terms of 20 20 - * the GNU General Public License, in which case the provisions of the GPL2 21 21 - * are required INSTEAD OF the above restrictions. (This clause is 22 22 - * necessary due to a potential bad interaction between the GPL and 23 23 - * the restrictions contained in a BSD-style copyright.) 24 24 - * 25 25 - * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED 26 26 - * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 27 27 - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF 28 28 - * WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE 29 29 - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 30 - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 31 31 - * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 32 32 - * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 33 33 - * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 34 34 - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 35 35 - * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH 36 36 - * DAMAGE. 37 37 - */ 38 38 - 39 39 - /* 40 40 - * Note for using key wrapping: 41 41 - * 42 42 - * * The result of the encryption operation is the ciphertext starting 43 43 - * with the 2nd semiblock. The first semiblock is provided as the IV. 44 44 - * The IV used to start the encryption operation is the default IV. 45 45 - * 46 46 - * * The input for the decryption is the first semiblock handed in as an 47 47 - * IV. The ciphertext is the data starting with the 2nd semiblock. The 48 48 - * return code of the decryption operation will be EBADMSG in case an 49 49 - * integrity error occurs. 50 50 - * 51 51 - * To obtain the full result of an encryption as expected by SP800-38F, the 52 52 - * caller must allocate a buffer of plaintext + 8 bytes: 53 53 - * 54 54 - * unsigned int datalen = ptlen + crypto_skcipher_ivsize(tfm); 55 55 - * u8 data[datalen]; 56 56 - * u8 *iv = data; 57 57 - * u8 *pt = data + crypto_skcipher_ivsize(tfm); 58 58 - * <ensure that pt contains the plaintext of size ptlen> 59 59 - * sg_init_one(&sg, pt, ptlen); 60 60 - * skcipher_request_set_crypt(req, &sg, &sg, ptlen, iv); 61 61 - * 62 62 - * ==> After encryption, data now contains full KW result as per SP800-38F. 63 63 - * 64 64 - * In case of decryption, ciphertext now already has the expected length 65 65 - * and must be segmented appropriately: 66 66 - * 67 67 - * unsigned int datalen = CTLEN; 68 68 - * u8 data[datalen]; 69 69 - * <ensure that data contains full ciphertext> 70 70 - * u8 *iv = data; 71 71 - * u8 *ct = data + crypto_skcipher_ivsize(tfm); 72 72 - * unsigned int ctlen = datalen - crypto_skcipher_ivsize(tfm); 73 73 - * sg_init_one(&sg, ct, ctlen); 74 74 - * skcipher_request_set_crypt(req, &sg, &sg, ctlen, iv); 75 75 - * 76 76 - * ==> After decryption (which hopefully does not return EBADMSG), the ct 77 77 - * pointer now points to the plaintext of size ctlen. 78 78 - * 79 79 - * Note 2: KWP is not implemented as this would defy in-place operation. 80 80 - * If somebody wants to wrap non-aligned data, he should simply pad 81 81 - * the input with zeros to fill it up to the 8 byte boundary. 82 82 - */ 83 83 - 84 84 - #include <linux/module.h> 85 85 - #include <linux/crypto.h> 86 86 - #include <linux/scatterlist.h> 87 87 - #include <crypto/scatterwalk.h> 88 88 - #include <crypto/internal/cipher.h> 89 89 - #include <crypto/internal/skcipher.h> 90 90 - 91 91 - struct crypto_kw_block { 92 92 - #define SEMIBSIZE 8 93 93 - __be64 A; 94 94 - __be64 R; 95 95 - }; 96 96 - 97 97 - /* 98 98 - * Fast forward the SGL to the "end" length minus SEMIBSIZE. 99 99 - * The start in the SGL defined by the fast-forward is returned with 100 100 - * the walk variable 101 101 - */ 102 102 - static void crypto_kw_scatterlist_ff(struct scatter_walk *walk, 103 103 - struct scatterlist *sg, 104 104 - unsigned int end) 105 105 - { 106 106 - unsigned int skip = 0; 107 107 - 108 108 - /* The caller should only operate on full SEMIBLOCKs. */ 109 109 - BUG_ON(end < SEMIBSIZE); 110 110 - 111 111 - skip = end - SEMIBSIZE; 112 112 - while (sg) { 113 113 - if (sg->length > skip) { 114 114 - scatterwalk_start(walk, sg); 115 115 - scatterwalk_advance(walk, skip); 116 116 - break; 117 117 - } 118 118 - 119 119 - skip -= sg->length; 120 120 - sg = sg_next(sg); 121 121 - } 122 122 - } 123 123 - 124 124 - static int crypto_kw_decrypt(struct skcipher_request *req) 125 125 - { 126 126 - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 127 127 - struct crypto_cipher *cipher = skcipher_cipher_simple(tfm); 128 128 - struct crypto_kw_block block; 129 129 - struct scatterlist *src, *dst; 130 130 - u64 t = 6 * ((req->cryptlen) >> 3); 131 131 - unsigned int i; 132 132 - int ret = 0; 133 133 - 134 134 - /* 135 135 - * Require at least 2 semiblocks (note, the 3rd semiblock that is 136 136 - * required by SP800-38F is the IV. 137 137 - */ 138 138 - if (req->cryptlen < (2 * SEMIBSIZE) || req->cryptlen % SEMIBSIZE) 139 139 - return -EINVAL; 140 140 - 141 141 - /* Place the IV into block A */ 142 142 - memcpy(&block.A, req->iv, SEMIBSIZE); 143 143 - 144 144 - /* 145 145 - * src scatterlist is read-only. dst scatterlist is r/w. During the 146 146 - * first loop, src points to req->src and dst to req->dst. For any 147 147 - * subsequent round, the code operates on req->dst only. 148 148 - */ 149 149 - src = req->src; 150 150 - dst = req->dst; 151 151 - 152 152 - for (i = 0; i < 6; i++) { 153 153 - struct scatter_walk src_walk, dst_walk; 154 154 - unsigned int nbytes = req->cryptlen; 155 155 - 156 156 - while (nbytes) { 157 157 - /* move pointer by nbytes in the SGL */ 158 158 - crypto_kw_scatterlist_ff(&src_walk, src, nbytes); 159 159 - /* get the source block */ 160 160 - scatterwalk_copychunks(&block.R, &src_walk, SEMIBSIZE, 161 161 - false); 162 162 - 163 163 - /* perform KW operation: modify IV with counter */ 164 164 - block.A ^= cpu_to_be64(t); 165 165 - t--; 166 166 - /* perform KW operation: decrypt block */ 167 167 - crypto_cipher_decrypt_one(cipher, (u8 *)&block, 168 168 - (u8 *)&block); 169 169 - 170 170 - /* move pointer by nbytes in the SGL */ 171 171 - crypto_kw_scatterlist_ff(&dst_walk, dst, nbytes); 172 172 - /* Copy block->R into place */ 173 173 - scatterwalk_copychunks(&block.R, &dst_walk, SEMIBSIZE, 174 174 - true); 175 175 - 176 176 - nbytes -= SEMIBSIZE; 177 177 - } 178 178 - 179 179 - /* we now start to operate on the dst SGL only */ 180 180 - src = req->dst; 181 181 - dst = req->dst; 182 182 - } 183 183 - 184 184 - /* Perform authentication check */ 185 185 - if (block.A != cpu_to_be64(0xa6a6a6a6a6a6a6a6ULL)) 186 186 - ret = -EBADMSG; 187 187 - 188 188 - memzero_explicit(&block, sizeof(struct crypto_kw_block)); 189 189 - 190 190 - return ret; 191 191 - } 192 192 - 193 193 - static int crypto_kw_encrypt(struct skcipher_request *req) 194 194 - { 195 195 - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 196 196 - struct crypto_cipher *cipher = skcipher_cipher_simple(tfm); 197 197 - struct crypto_kw_block block; 198 198 - struct scatterlist *src, *dst; 199 199 - u64 t = 1; 200 200 - unsigned int i; 201 201 - 202 202 - /* 203 203 - * Require at least 2 semiblocks (note, the 3rd semiblock that is 204 204 - * required by SP800-38F is the IV that occupies the first semiblock. 205 205 - * This means that the dst memory must be one semiblock larger than src. 206 206 - * Also ensure that the given data is aligned to semiblock. 207 207 - */ 208 208 - if (req->cryptlen < (2 * SEMIBSIZE) || req->cryptlen % SEMIBSIZE) 209 209 - return -EINVAL; 210 210 - 211 211 - /* 212 212 - * Place the predefined IV into block A -- for encrypt, the caller 213 213 - * does not need to provide an IV, but he needs to fetch the final IV. 214 214 - */ 215 215 - block.A = cpu_to_be64(0xa6a6a6a6a6a6a6a6ULL); 216 216 - 217 217 - /* 218 218 - * src scatterlist is read-only. dst scatterlist is r/w. During the 219 219 - * first loop, src points to req->src and dst to req->dst. For any 220 220 - * subsequent round, the code operates on req->dst only. 221 221 - */ 222 222 - src = req->src; 223 223 - dst = req->dst; 224 224 - 225 225 - for (i = 0; i < 6; i++) { 226 226 - struct scatter_walk src_walk, dst_walk; 227 227 - unsigned int nbytes = req->cryptlen; 228 228 - 229 229 - scatterwalk_start(&src_walk, src); 230 230 - scatterwalk_start(&dst_walk, dst); 231 231 - 232 232 - while (nbytes) { 233 233 - /* get the source block */ 234 234 - scatterwalk_copychunks(&block.R, &src_walk, SEMIBSIZE, 235 235 - false); 236 236 - 237 237 - /* perform KW operation: encrypt block */ 238 238 - crypto_cipher_encrypt_one(cipher, (u8 *)&block, 239 239 - (u8 *)&block); 240 240 - /* perform KW operation: modify IV with counter */ 241 241 - block.A ^= cpu_to_be64(t); 242 242 - t++; 243 243 - 244 244 - /* Copy block->R into place */ 245 245 - scatterwalk_copychunks(&block.R, &dst_walk, SEMIBSIZE, 246 246 - true); 247 247 - 248 248 - nbytes -= SEMIBSIZE; 249 249 - } 250 250 - 251 251 - /* we now start to operate on the dst SGL only */ 252 252 - src = req->dst; 253 253 - dst = req->dst; 254 254 - } 255 255 - 256 256 - /* establish the IV for the caller to pick up */ 257 257 - memcpy(req->iv, &block.A, SEMIBSIZE); 258 258 - 259 259 - memzero_explicit(&block, sizeof(struct crypto_kw_block)); 260 260 - 261 261 - return 0; 262 262 - } 263 263 - 264 264 - static int crypto_kw_create(struct crypto_template *tmpl, struct rtattr **tb) 265 265 - { 266 266 - struct skcipher_instance *inst; 267 267 - struct crypto_alg *alg; 268 268 - int err; 269 269 - 270 270 - inst = skcipher_alloc_instance_simple(tmpl, tb); 271 271 - if (IS_ERR(inst)) 272 272 - return PTR_ERR(inst); 273 273 - 274 274 - alg = skcipher_ialg_simple(inst); 275 275 - 276 276 - err = -EINVAL; 277 277 - /* Section 5.1 requirement for KW */ 278 278 - if (alg->cra_blocksize != sizeof(struct crypto_kw_block)) 279 279 - goto out_free_inst; 280 280 - 281 281 - inst->alg.base.cra_blocksize = SEMIBSIZE; 282 282 - inst->alg.base.cra_alignmask = 0; 283 283 - inst->alg.ivsize = SEMIBSIZE; 284 284 - 285 285 - inst->alg.encrypt = crypto_kw_encrypt; 286 286 - inst->alg.decrypt = crypto_kw_decrypt; 287 287 - 288 288 - err = skcipher_register_instance(tmpl, inst); 289 289 - if (err) { 290 290 - out_free_inst: 291 291 - inst->free(inst); 292 292 - } 293 293 - 294 294 - return err; 295 295 - } 296 296 - 297 297 - static struct crypto_template crypto_kw_tmpl = { 298 298 - .name = "kw", 299 299 - .create = crypto_kw_create, 300 300 - .module = THIS_MODULE, 301 301 - }; 302 302 - 303 303 - static int __init crypto_kw_init(void) 304 304 - { 305 305 - return crypto_register_template(&crypto_kw_tmpl); 306 306 - } 307 307 - 308 308 - static void __exit crypto_kw_exit(void) 309 309 - { 310 310 - crypto_unregister_template(&crypto_kw_tmpl); 311 311 - } 312 312 - 313 313 - subsys_initcall(crypto_kw_init); 314 314 - module_exit(crypto_kw_exit); 315 315 - 316 316 - MODULE_LICENSE("Dual BSD/GPL"); 317 317 - MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>"); 318 318 - MODULE_DESCRIPTION("Key Wrapping (RFC3394 / NIST SP800-38F)"); 319 319 - MODULE_ALIAS_CRYPTO("kw"); 320 320 - MODULE_IMPORT_NS("CRYPTO_INTERNAL");

+6 -11

crypto/khazad.c

reviewed

··· 23 23 #include <linux/init.h> 24 24 #include <linux/module.h> 25 25 #include <linux/mm.h> 26 26 - #include <asm/byteorder.h> 26 26 + #include <linux/unaligned.h> 27 27 #include <linux/types.h> 28 28 29 29 #define KHAZAD_KEY_SIZE 16 ··· 757 757 unsigned int key_len) 758 758 { 759 759 struct khazad_ctx *ctx = crypto_tfm_ctx(tfm); 760 760 - const __be32 *key = (const __be32 *)in_key; 761 760 int r; 762 761 const u64 *S = T7; 763 762 u64 K2, K1; 764 763 765 765 - /* key is supposed to be 32-bit aligned */ 766 766 - K2 = ((u64)be32_to_cpu(key[0]) << 32) | be32_to_cpu(key[1]); 767 767 - K1 = ((u64)be32_to_cpu(key[2]) << 32) | be32_to_cpu(key[3]); 764 764 + K2 = get_unaligned_be64(&in_key[0]); 765 765 + K1 = get_unaligned_be64(&in_key[8]); 768 766 769 767 /* setup the encrypt key */ 770 768 for (r = 0; r <= KHAZAD_ROUNDS; r++) { ··· 798 800 } 799 801 800 802 static void khazad_crypt(const u64 roundKey[KHAZAD_ROUNDS + 1], 801 801 - u8 *ciphertext, const u8 *plaintext) 803 803 + u8 *dst, const u8 *src) 802 804 { 803 803 - const __be64 *src = (const __be64 *)plaintext; 804 804 - __be64 *dst = (__be64 *)ciphertext; 805 805 int r; 806 806 u64 state; 807 807 808 808 - state = be64_to_cpu(*src) ^ roundKey[0]; 808 808 + state = get_unaligned_be64(src) ^ roundKey[0]; 809 809 810 810 for (r = 1; r < KHAZAD_ROUNDS; r++) { 811 811 state = T0[(int)(state >> 56) ] ^ ··· 827 831 (T7[(int)(state ) & 0xff] & 0x00000000000000ffULL) ^ 828 832 roundKey[KHAZAD_ROUNDS]; 829 833 830 830 - *dst = cpu_to_be64(state); 834 834 + put_unaligned_be64(state, dst); 831 835 } 832 836 833 837 static void khazad_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) ··· 848 852 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 849 853 .cra_blocksize = KHAZAD_BLOCK_SIZE, 850 854 .cra_ctxsize = sizeof (struct khazad_ctx), 851 851 - .cra_alignmask = 7, 852 855 .cra_module = THIS_MODULE, 853 856 .cra_u = { .cipher = { 854 857 .cia_min_keysize = KHAZAD_KEY_SIZE,

+3 -6

crypto/proc.c

reviewed

··· 47 47 (alg->cra_flags & CRYPTO_ALG_TESTED) ? 48 48 "passed" : "unknown"); 49 49 seq_printf(m, "internal : %s\n", 50 50 - (alg->cra_flags & CRYPTO_ALG_INTERNAL) ? 51 51 - "yes" : "no"); 52 52 - if (fips_enabled) { 50 50 + str_yes_no(alg->cra_flags & CRYPTO_ALG_INTERNAL)); 51 51 + if (fips_enabled) 53 52 seq_printf(m, "fips : %s\n", 54 54 - (alg->cra_flags & CRYPTO_ALG_FIPS_INTERNAL) ? 55 55 - "no" : "yes"); 56 56 - } 53 53 + str_no_yes(alg->cra_flags & CRYPTO_ALG_FIPS_INTERNAL)); 57 54 58 55 if (alg->cra_flags & CRYPTO_ALG_LARVAL) { 59 56 seq_printf(m, "type : larval\n");

+21 -27

crypto/seed.c

reviewed

··· 13 13 #include <linux/init.h> 14 14 #include <linux/types.h> 15 15 #include <linux/errno.h> 16 16 - #include <asm/byteorder.h> 16 16 + #include <linux/unaligned.h> 17 17 18 18 #define SEED_NUM_KCONSTANTS 16 19 19 #define SEED_KEY_SIZE 16 ··· 329 329 { 330 330 struct seed_ctx *ctx = crypto_tfm_ctx(tfm); 331 331 u32 *keyout = ctx->keysched; 332 332 - const __be32 *key = (const __be32 *)in_key; 333 332 u32 i, t0, t1, x1, x2, x3, x4; 334 333 335 335 - x1 = be32_to_cpu(key[0]); 336 336 - x2 = be32_to_cpu(key[1]); 337 337 - x3 = be32_to_cpu(key[2]); 338 338 - x4 = be32_to_cpu(key[3]); 334 334 + x1 = get_unaligned_be32(&in_key[0]); 335 335 + x2 = get_unaligned_be32(&in_key[4]); 336 336 + x3 = get_unaligned_be32(&in_key[8]); 337 337 + x4 = get_unaligned_be32(&in_key[12]); 339 338 340 339 for (i = 0; i < SEED_NUM_KCONSTANTS; i++) { 341 340 t0 = x1 + x3 - KC[i]; ··· 363 364 static void seed_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 364 365 { 365 366 const struct seed_ctx *ctx = crypto_tfm_ctx(tfm); 366 366 - const __be32 *src = (const __be32 *)in; 367 367 - __be32 *dst = (__be32 *)out; 368 367 u32 x1, x2, x3, x4, t0, t1; 369 368 const u32 *ks = ctx->keysched; 370 369 371 371 - x1 = be32_to_cpu(src[0]); 372 372 - x2 = be32_to_cpu(src[1]); 373 373 - x3 = be32_to_cpu(src[2]); 374 374 - x4 = be32_to_cpu(src[3]); 370 370 + x1 = get_unaligned_be32(&in[0]); 371 371 + x2 = get_unaligned_be32(&in[4]); 372 372 + x3 = get_unaligned_be32(&in[8]); 373 373 + x4 = get_unaligned_be32(&in[12]); 375 374 376 375 OP(x1, x2, x3, x4, 0); 377 376 OP(x3, x4, x1, x2, 2); ··· 388 391 OP(x1, x2, x3, x4, 28); 389 392 OP(x3, x4, x1, x2, 30); 390 393 391 391 - dst[0] = cpu_to_be32(x3); 392 392 - dst[1] = cpu_to_be32(x4); 393 393 - dst[2] = cpu_to_be32(x1); 394 394 - dst[3] = cpu_to_be32(x2); 394 394 + put_unaligned_be32(x3, &out[0]); 395 395 + put_unaligned_be32(x4, &out[4]); 396 396 + put_unaligned_be32(x1, &out[8]); 397 397 + put_unaligned_be32(x2, &out[12]); 395 398 } 396 399 397 400 /* decrypt a block of text */ ··· 399 402 static void seed_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in) 400 403 { 401 404 const struct seed_ctx *ctx = crypto_tfm_ctx(tfm); 402 402 - const __be32 *src = (const __be32 *)in; 403 403 - __be32 *dst = (__be32 *)out; 404 405 u32 x1, x2, x3, x4, t0, t1; 405 406 const u32 *ks = ctx->keysched; 406 407 407 407 - x1 = be32_to_cpu(src[0]); 408 408 - x2 = be32_to_cpu(src[1]); 409 409 - x3 = be32_to_cpu(src[2]); 410 410 - x4 = be32_to_cpu(src[3]); 408 408 + x1 = get_unaligned_be32(&in[0]); 409 409 + x2 = get_unaligned_be32(&in[4]); 410 410 + x3 = get_unaligned_be32(&in[8]); 411 411 + x4 = get_unaligned_be32(&in[12]); 411 412 412 413 OP(x1, x2, x3, x4, 30); 413 414 OP(x3, x4, x1, x2, 28); ··· 424 429 OP(x1, x2, x3, x4, 2); 425 430 OP(x3, x4, x1, x2, 0); 426 431 427 427 - dst[0] = cpu_to_be32(x3); 428 428 - dst[1] = cpu_to_be32(x4); 429 429 - dst[2] = cpu_to_be32(x1); 430 430 - dst[3] = cpu_to_be32(x2); 432 432 + put_unaligned_be32(x3, &out[0]); 433 433 + put_unaligned_be32(x4, &out[4]); 434 434 + put_unaligned_be32(x1, &out[8]); 435 435 + put_unaligned_be32(x2, &out[12]); 431 436 } 432 437 433 438 ··· 438 443 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 439 444 .cra_blocksize = SEED_BLOCK_SIZE, 440 445 .cra_ctxsize = sizeof(struct seed_ctx), 441 441 - .cra_alignmask = 3, 442 446 .cra_module = THIS_MODULE, 443 447 .cra_u = { 444 448 .cipher = {

+1 -3

crypto/sig.c

reviewed

··· 15 15 16 16 #include "internal.h" 17 17 18 18 - #define CRYPTO_ALG_TYPE_SIG_MASK 0x0000000e 19 19 - 20 18 static void crypto_sig_exit_tfm(struct crypto_tfm *tfm) 21 19 { 22 20 struct crypto_sig *sig = __crypto_sig_tfm(tfm); ··· 71 73 .report = crypto_sig_report, 72 74 #endif 73 75 .maskclear = ~CRYPTO_ALG_TYPE_MASK, 74 74 - .maskset = CRYPTO_ALG_TYPE_SIG_MASK, 76 76 + .maskset = CRYPTO_ALG_TYPE_MASK, 75 77 .type = CRYPTO_ALG_TYPE_SIG, 76 78 .tfmsize = offsetof(struct crypto_sig, base), 77 79 };

+102 -265

crypto/skcipher.c

reviewed

··· 17 17 #include <linux/cryptouser.h> 18 18 #include <linux/err.h> 19 19 #include <linux/kernel.h> 20 20 - #include <linux/list.h> 21 20 #include <linux/mm.h> 22 21 #include <linux/module.h> 23 22 #include <linux/seq_file.h> ··· 28 29 #define CRYPTO_ALG_TYPE_SKCIPHER_MASK 0x0000000e 29 30 30 31 enum { 31 31 - SKCIPHER_WALK_PHYS = 1 << 0, 32 32 - SKCIPHER_WALK_SLOW = 1 << 1, 33 33 - SKCIPHER_WALK_COPY = 1 << 2, 34 34 - SKCIPHER_WALK_DIFF = 1 << 3, 35 35 - SKCIPHER_WALK_SLEEP = 1 << 4, 36 36 - }; 37 37 - 38 38 - struct skcipher_walk_buffer { 39 39 - struct list_head entry; 40 40 - struct scatter_walk dst; 41 41 - unsigned int len; 42 42 - u8 *data; 43 43 - u8 buffer[]; 32 32 + SKCIPHER_WALK_SLOW = 1 << 0, 33 33 + SKCIPHER_WALK_COPY = 1 << 1, 34 34 + SKCIPHER_WALK_DIFF = 1 << 2, 35 35 + SKCIPHER_WALK_SLEEP = 1 << 3, 44 36 }; 45 37 46 38 static const struct crypto_type crypto_skcipher_type; ··· 63 73 return walk->flags & SKCIPHER_WALK_SLEEP ? GFP_KERNEL : GFP_ATOMIC; 64 74 } 65 75 66 66 - /* Get a spot of the specified length that does not straddle a page. 67 67 - * The caller needs to ensure that there is enough space for this operation. 68 68 - */ 69 69 - static inline u8 *skcipher_get_spot(u8 *start, unsigned int len) 70 70 - { 71 71 - u8 *end_page = (u8 *)(((unsigned long)(start + len - 1)) & PAGE_MASK); 72 72 - 73 73 - return max(start, end_page); 74 74 - } 75 75 - 76 76 static inline struct skcipher_alg *__crypto_skcipher_alg( 77 77 struct crypto_alg *alg) 78 78 { ··· 71 91 72 92 static int skcipher_done_slow(struct skcipher_walk *walk, unsigned int bsize) 73 93 { 74 74 - u8 *addr; 94 94 + u8 *addr = PTR_ALIGN(walk->buffer, walk->alignmask + 1); 75 95 76 76 - addr = (u8 *)ALIGN((unsigned long)walk->buffer, walk->alignmask + 1); 77 77 - addr = skcipher_get_spot(addr, bsize); 78 78 - scatterwalk_copychunks(addr, &walk->out, bsize, 79 79 - (walk->flags & SKCIPHER_WALK_PHYS) ? 2 : 1); 96 96 + scatterwalk_copychunks(addr, &walk->out, bsize, 1); 80 97 return 0; 81 98 } 82 99 83 83 - int skcipher_walk_done(struct skcipher_walk *walk, int err) 100 100 + /** 101 101 + * skcipher_walk_done() - finish one step of a skcipher_walk 102 102 + * @walk: the skcipher_walk 103 103 + * @res: number of bytes *not* processed (>= 0) from walk->nbytes, 104 104 + * or a -errno value to terminate the walk due to an error 105 105 + * 106 106 + * This function cleans up after one step of walking through the source and 107 107 + * destination scatterlists, and advances to the next step if applicable. 108 108 + * walk->nbytes is set to the number of bytes available in the next step, 109 109 + * walk->total is set to the new total number of bytes remaining, and 110 110 + * walk->{src,dst}.virt.addr is set to the next pair of data pointers. If there 111 111 + * is no more data, or if an error occurred (i.e. -errno return), then 112 112 + * walk->nbytes and walk->total are set to 0 and all resources owned by the 113 113 + * skcipher_walk are freed. 114 114 + * 115 115 + * Return: 0 or a -errno value. If @res was a -errno value then it will be 116 116 + * returned, but other errors may occur too. 117 117 + */ 118 118 + int skcipher_walk_done(struct skcipher_walk *walk, int res) 84 119 { 85 85 - unsigned int n = walk->nbytes; 86 86 - unsigned int nbytes = 0; 120 120 + unsigned int n = walk->nbytes; /* num bytes processed this step */ 121 121 + unsigned int total = 0; /* new total remaining */ 87 122 88 123 if (!n) 89 124 goto finish; 90 125 91 91 - if (likely(err >= 0)) { 92 92 - n -= err; 93 93 - nbytes = walk->total - n; 126 126 + if (likely(res >= 0)) { 127 127 + n -= res; /* subtract num bytes *not* processed */ 128 128 + total = walk->total - n; 94 129 } 95 130 96 96 - if (likely(!(walk->flags & (SKCIPHER_WALK_PHYS | 97 97 - SKCIPHER_WALK_SLOW | 131 131 + if (likely(!(walk->flags & (SKCIPHER_WALK_SLOW | 98 132 SKCIPHER_WALK_COPY | 99 133 SKCIPHER_WALK_DIFF)))) { 100 134 unmap_src: ··· 120 126 skcipher_map_dst(walk); 121 127 memcpy(walk->dst.virt.addr, walk->page, n); 122 128 skcipher_unmap_dst(walk); 123 123 - } else if (unlikely(walk->flags & SKCIPHER_WALK_SLOW)) { 124 124 - if (err > 0) { 129 129 + } else { /* SKCIPHER_WALK_SLOW */ 130 130 + if (res > 0) { 125 131 /* 126 132 * Didn't process all bytes. Either the algorithm is 127 133 * broken, or this was the last step and it turned out 128 134 * the message wasn't evenly divisible into blocks but 129 135 * the algorithm requires it. 130 136 */ 131 131 - err = -EINVAL; 132 132 - nbytes = 0; 137 137 + res = -EINVAL; 138 138 + total = 0; 133 139 } else 134 140 n = skcipher_done_slow(walk, n); 135 141 } 136 142 137 137 - if (err > 0) 138 138 - err = 0; 143 143 + if (res > 0) 144 144 + res = 0; 139 145 140 140 - walk->total = nbytes; 146 146 + walk->total = total; 141 147 walk->nbytes = 0; 142 148 143 149 scatterwalk_advance(&walk->in, n); 144 150 scatterwalk_advance(&walk->out, n); 145 145 - scatterwalk_done(&walk->in, 0, nbytes); 146 146 - scatterwalk_done(&walk->out, 1, nbytes); 151 151 + scatterwalk_done(&walk->in, 0, total); 152 152 + scatterwalk_done(&walk->out, 1, total); 147 153 148 148 - if (nbytes) { 149 149 - crypto_yield(walk->flags & SKCIPHER_WALK_SLEEP ? 150 150 - CRYPTO_TFM_REQ_MAY_SLEEP : 0); 154 154 + if (total) { 155 155 + if (walk->flags & SKCIPHER_WALK_SLEEP) 156 156 + cond_resched(); 157 157 + walk->flags &= ~(SKCIPHER_WALK_SLOW | SKCIPHER_WALK_COPY | 158 158 + SKCIPHER_WALK_DIFF); 151 159 return skcipher_walk_next(walk); 152 160 } 153 161 154 162 finish: 155 163 /* Short-circuit for the common/fast path. */ 156 164 if (!((unsigned long)walk->buffer | (unsigned long)walk->page)) 157 157 - goto out; 158 158 - 159 159 - if (walk->flags & SKCIPHER_WALK_PHYS) 160 165 goto out; 161 166 162 167 if (walk->iv != walk->oiv) ··· 166 173 free_page((unsigned long)walk->page); 167 174 168 175 out: 169 169 - return err; 176 176 + return res; 170 177 } 171 178 EXPORT_SYMBOL_GPL(skcipher_walk_done); 172 179 173 173 - void skcipher_walk_complete(struct skcipher_walk *walk, int err) 174 174 - { 175 175 - struct skcipher_walk_buffer *p, *tmp; 176 176 - 177 177 - list_for_each_entry_safe(p, tmp, &walk->buffers, entry) { 178 178 - u8 *data; 179 179 - 180 180 - if (err) 181 181 - goto done; 182 182 - 183 183 - data = p->data; 184 184 - if (!data) { 185 185 - data = PTR_ALIGN(&p->buffer[0], walk->alignmask + 1); 186 186 - data = skcipher_get_spot(data, walk->stride); 187 187 - } 188 188 - 189 189 - scatterwalk_copychunks(data, &p->dst, p->len, 1); 190 190 - 191 191 - if (offset_in_page(p->data) + p->len + walk->stride > 192 192 - PAGE_SIZE) 193 193 - free_page((unsigned long)p->data); 194 194 - 195 195 - done: 196 196 - list_del(&p->entry); 197 197 - kfree(p); 198 198 - } 199 199 - 200 200 - if (!err && walk->iv != walk->oiv) 201 201 - memcpy(walk->oiv, walk->iv, walk->ivsize); 202 202 - if (walk->buffer != walk->page) 203 203 - kfree(walk->buffer); 204 204 - if (walk->page) 205 205 - free_page((unsigned long)walk->page); 206 206 - } 207 207 - EXPORT_SYMBOL_GPL(skcipher_walk_complete); 208 208 - 209 209 - static void skcipher_queue_write(struct skcipher_walk *walk, 210 210 - struct skcipher_walk_buffer *p) 211 211 - { 212 212 - p->dst = walk->out; 213 213 - list_add_tail(&p->entry, &walk->buffers); 214 214 - } 215 215 - 216 180 static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize) 217 181 { 218 218 - bool phys = walk->flags & SKCIPHER_WALK_PHYS; 219 182 unsigned alignmask = walk->alignmask; 220 220 - struct skcipher_walk_buffer *p; 221 221 - unsigned a; 222 183 unsigned n; 223 184 u8 *buffer; 224 224 - void *v; 225 185 226 226 - if (!phys) { 227 227 - if (!walk->buffer) 228 228 - walk->buffer = walk->page; 229 229 - buffer = walk->buffer; 230 230 - if (buffer) 231 231 - goto ok; 186 186 + if (!walk->buffer) 187 187 + walk->buffer = walk->page; 188 188 + buffer = walk->buffer; 189 189 + if (!buffer) { 190 190 + /* Min size for a buffer of bsize bytes aligned to alignmask */ 191 191 + n = bsize + (alignmask & ~(crypto_tfm_ctx_alignment() - 1)); 192 192 + 193 193 + buffer = kzalloc(n, skcipher_walk_gfp(walk)); 194 194 + if (!buffer) 195 195 + return skcipher_walk_done(walk, -ENOMEM); 196 196 + walk->buffer = buffer; 232 197 } 233 233 - 234 234 - /* Start with the minimum alignment of kmalloc. */ 235 235 - a = crypto_tfm_ctx_alignment() - 1; 236 236 - n = bsize; 237 237 - 238 238 - if (phys) { 239 239 - /* Calculate the minimum alignment of p->buffer. */ 240 240 - a &= (sizeof(*p) ^ (sizeof(*p) - 1)) >> 1; 241 241 - n += sizeof(*p); 242 242 - } 243 243 - 244 244 - /* Minimum size to align p->buffer by alignmask. */ 245 245 - n += alignmask & ~a; 246 246 - 247 247 - /* Minimum size to ensure p->buffer does not straddle a page. */ 248 248 - n += (bsize - 1) & ~(alignmask | a); 249 249 - 250 250 - v = kzalloc(n, skcipher_walk_gfp(walk)); 251 251 - if (!v) 252 252 - return skcipher_walk_done(walk, -ENOMEM); 253 253 - 254 254 - if (phys) { 255 255 - p = v; 256 256 - p->len = bsize; 257 257 - skcipher_queue_write(walk, p); 258 258 - buffer = p->buffer; 259 259 - } else { 260 260 - walk->buffer = v; 261 261 - buffer = v; 262 262 - } 263 263 - 264 264 - ok: 265 198 walk->dst.virt.addr = PTR_ALIGN(buffer, alignmask + 1); 266 266 - walk->dst.virt.addr = skcipher_get_spot(walk->dst.virt.addr, bsize); 267 199 walk->src.virt.addr = walk->dst.virt.addr; 268 200 269 201 scatterwalk_copychunks(walk->src.virt.addr, &walk->in, bsize, 0); ··· 201 283 202 284 static int skcipher_next_copy(struct skcipher_walk *walk) 203 285 { 204 204 - struct skcipher_walk_buffer *p; 205 286 u8 *tmp = walk->page; 206 287 207 288 skcipher_map_src(walk); ··· 209 292 210 293 walk->src.virt.addr = tmp; 211 294 walk->dst.virt.addr = tmp; 212 212 - 213 213 - if (!(walk->flags & SKCIPHER_WALK_PHYS)) 214 214 - return 0; 215 215 - 216 216 - p = kmalloc(sizeof(*p), skcipher_walk_gfp(walk)); 217 217 - if (!p) 218 218 - return -ENOMEM; 219 219 - 220 220 - p->data = walk->page; 221 221 - p->len = walk->nbytes; 222 222 - skcipher_queue_write(walk, p); 223 223 - 224 224 - if (offset_in_page(walk->page) + walk->nbytes + walk->stride > 225 225 - PAGE_SIZE) 226 226 - walk->page = NULL; 227 227 - else 228 228 - walk->page += walk->nbytes; 229 229 - 230 295 return 0; 231 296 } 232 297 ··· 216 317 { 217 318 unsigned long diff; 218 319 219 219 - walk->src.phys.page = scatterwalk_page(&walk->in); 220 220 - walk->src.phys.offset = offset_in_page(walk->in.offset); 221 221 - walk->dst.phys.page = scatterwalk_page(&walk->out); 222 222 - walk->dst.phys.offset = offset_in_page(walk->out.offset); 223 223 - 224 224 - if (walk->flags & SKCIPHER_WALK_PHYS) 225 225 - return 0; 226 226 - 227 227 - diff = walk->src.phys.offset - walk->dst.phys.offset; 228 228 - diff |= walk->src.virt.page - walk->dst.virt.page; 320 320 + diff = offset_in_page(walk->in.offset) - 321 321 + offset_in_page(walk->out.offset); 322 322 + diff |= (u8 *)scatterwalk_page(&walk->in) - 323 323 + (u8 *)scatterwalk_page(&walk->out); 229 324 230 325 skcipher_map_src(walk); 231 326 walk->dst.virt.addr = walk->src.virt.addr; ··· 236 343 { 237 344 unsigned int bsize; 238 345 unsigned int n; 239 239 - int err; 240 240 - 241 241 - walk->flags &= ~(SKCIPHER_WALK_SLOW | SKCIPHER_WALK_COPY | 242 242 - SKCIPHER_WALK_DIFF); 243 346 244 347 n = walk->total; 245 348 bsize = min(walk->stride, max(n, walk->blocksize)); ··· 247 358 return skcipher_walk_done(walk, -EINVAL); 248 359 249 360 slow_path: 250 250 - err = skcipher_next_slow(walk, bsize); 251 251 - goto set_phys_lowmem; 361 361 + return skcipher_next_slow(walk, bsize); 252 362 } 363 363 + walk->nbytes = n; 253 364 254 365 if (unlikely((walk->in.offset | walk->out.offset) & walk->alignmask)) { 255 366 if (!walk->page) { ··· 259 370 if (!walk->page) 260 371 goto slow_path; 261 372 } 262 262 - 263 263 - walk->nbytes = min_t(unsigned, n, 264 264 - PAGE_SIZE - offset_in_page(walk->page)); 265 373 walk->flags |= SKCIPHER_WALK_COPY; 266 266 - err = skcipher_next_copy(walk); 267 267 - goto set_phys_lowmem; 374 374 + return skcipher_next_copy(walk); 268 375 } 269 269 - 270 270 - walk->nbytes = n; 271 376 272 377 return skcipher_next_fast(walk); 273 273 - 274 274 - set_phys_lowmem: 275 275 - if (!err && (walk->flags & SKCIPHER_WALK_PHYS)) { 276 276 - walk->src.phys.page = virt_to_page(walk->src.virt.addr); 277 277 - walk->dst.phys.page = virt_to_page(walk->dst.virt.addr); 278 278 - walk->src.phys.offset &= PAGE_SIZE - 1; 279 279 - walk->dst.phys.offset &= PAGE_SIZE - 1; 280 280 - } 281 281 - return err; 282 378 } 283 379 284 380 static int skcipher_copy_iv(struct skcipher_walk *walk) 285 381 { 286 286 - unsigned a = crypto_tfm_ctx_alignment() - 1; 287 382 unsigned alignmask = walk->alignmask; 288 383 unsigned ivsize = walk->ivsize; 289 289 - unsigned bs = walk->stride; 290 290 - unsigned aligned_bs; 384 384 + unsigned aligned_stride = ALIGN(walk->stride, alignmask + 1); 291 385 unsigned size; 292 386 u8 *iv; 293 387 294 294 - aligned_bs = ALIGN(bs, alignmask + 1); 295 295 - 296 296 - /* Minimum size to align buffer by alignmask. */ 297 297 - size = alignmask & ~a; 298 298 - 299 299 - if (walk->flags & SKCIPHER_WALK_PHYS) 300 300 - size += ivsize; 301 301 - else { 302 302 - size += aligned_bs + ivsize; 303 303 - 304 304 - /* Minimum size to ensure buffer does not straddle a page. */ 305 305 - size += (bs - 1) & ~(alignmask | a); 306 306 - } 388 388 + /* Min size for a buffer of stride + ivsize, aligned to alignmask */ 389 389 + size = aligned_stride + ivsize + 390 390 + (alignmask & ~(crypto_tfm_ctx_alignment() - 1)); 307 391 308 392 walk->buffer = kmalloc(size, skcipher_walk_gfp(walk)); 309 393 if (!walk->buffer) 310 394 return -ENOMEM; 311 395 312 312 - iv = PTR_ALIGN(walk->buffer, alignmask + 1); 313 313 - iv = skcipher_get_spot(iv, bs) + aligned_bs; 396 396 + iv = PTR_ALIGN(walk->buffer, alignmask + 1) + aligned_stride; 314 397 315 398 walk->iv = memcpy(iv, walk->iv, walk->ivsize); 316 399 return 0; ··· 305 444 return skcipher_walk_next(walk); 306 445 } 307 446 308 308 - static int skcipher_walk_skcipher(struct skcipher_walk *walk, 309 309 - struct skcipher_request *req) 447 447 + int skcipher_walk_virt(struct skcipher_walk *walk, 448 448 + struct skcipher_request *req, bool atomic) 310 449 { 311 311 - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 312 312 - struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 450 450 + const struct skcipher_alg *alg = 451 451 + crypto_skcipher_alg(crypto_skcipher_reqtfm(req)); 452 452 + 453 453 + might_sleep_if(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); 313 454 314 455 walk->total = req->cryptlen; 315 456 walk->nbytes = 0; 316 457 walk->iv = req->iv; 317 458 walk->oiv = req->iv; 459 459 + if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic) 460 460 + walk->flags = SKCIPHER_WALK_SLEEP; 461 461 + else 462 462 + walk->flags = 0; 318 463 319 464 if (unlikely(!walk->total)) 320 465 return 0; ··· 328 461 scatterwalk_start(&walk->in, req->src); 329 462 scatterwalk_start(&walk->out, req->dst); 330 463 331 331 - walk->flags &= ~SKCIPHER_WALK_SLEEP; 332 332 - walk->flags |= req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 333 333 - SKCIPHER_WALK_SLEEP : 0; 334 334 - 335 335 - walk->blocksize = crypto_skcipher_blocksize(tfm); 336 336 - walk->ivsize = crypto_skcipher_ivsize(tfm); 337 337 - walk->alignmask = crypto_skcipher_alignmask(tfm); 464 464 + /* 465 465 + * Accessing 'alg' directly generates better code than using the 466 466 + * crypto_skcipher_blocksize() and similar helper functions here, as it 467 467 + * prevents the algorithm pointer from being repeatedly reloaded. 468 468 + */ 469 469 + walk->blocksize = alg->base.cra_blocksize; 470 470 + walk->ivsize = alg->co.ivsize; 471 471 + walk->alignmask = alg->base.cra_alignmask; 338 472 339 473 if (alg->co.base.cra_type != &crypto_skcipher_type) 340 474 walk->stride = alg->co.chunksize; ··· 344 476 345 477 return skcipher_walk_first(walk); 346 478 } 347 347 - 348 348 - int skcipher_walk_virt(struct skcipher_walk *walk, 349 349 - struct skcipher_request *req, bool atomic) 350 350 - { 351 351 - int err; 352 352 - 353 353 - might_sleep_if(req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP); 354 354 - 355 355 - walk->flags &= ~SKCIPHER_WALK_PHYS; 356 356 - 357 357 - err = skcipher_walk_skcipher(walk, req); 358 358 - 359 359 - walk->flags &= atomic ? ~SKCIPHER_WALK_SLEEP : ~0; 360 360 - 361 361 - return err; 362 362 - } 363 479 EXPORT_SYMBOL_GPL(skcipher_walk_virt); 364 364 - 365 365 - int skcipher_walk_async(struct skcipher_walk *walk, 366 366 - struct skcipher_request *req) 367 367 - { 368 368 - walk->flags |= SKCIPHER_WALK_PHYS; 369 369 - 370 370 - INIT_LIST_HEAD(&walk->buffers); 371 371 - 372 372 - return skcipher_walk_skcipher(walk, req); 373 373 - } 374 374 - EXPORT_SYMBOL_GPL(skcipher_walk_async); 375 480 376 481 static int skcipher_walk_aead_common(struct skcipher_walk *walk, 377 482 struct aead_request *req, bool atomic) 378 483 { 379 379 - struct crypto_aead *tfm = crypto_aead_reqtfm(req); 380 380 - int err; 484 484 + const struct aead_alg *alg = crypto_aead_alg(crypto_aead_reqtfm(req)); 381 485 382 486 walk->nbytes = 0; 383 487 walk->iv = req->iv; 384 488 walk->oiv = req->iv; 489 489 + if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic) 490 490 + walk->flags = SKCIPHER_WALK_SLEEP; 491 491 + else 492 492 + walk->flags = 0; 385 493 386 494 if (unlikely(!walk->total)) 387 495 return 0; 388 388 - 389 389 - walk->flags &= ~SKCIPHER_WALK_PHYS; 390 496 391 497 scatterwalk_start(&walk->in, req->src); 392 498 scatterwalk_start(&walk->out, req->dst); ··· 371 529 scatterwalk_done(&walk->in, 0, walk->total); 372 530 scatterwalk_done(&walk->out, 0, walk->total); 373 531 374 374 - if (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) 375 375 - walk->flags |= SKCIPHER_WALK_SLEEP; 376 376 - else 377 377 - walk->flags &= ~SKCIPHER_WALK_SLEEP; 532 532 + /* 533 533 + * Accessing 'alg' directly generates better code than using the 534 534 + * crypto_aead_blocksize() and similar helper functions here, as it 535 535 + * prevents the algorithm pointer from being repeatedly reloaded. 536 536 + */ 537 537 + walk->blocksize = alg->base.cra_blocksize; 538 538 + walk->stride = alg->chunksize; 539 539 + walk->ivsize = alg->ivsize; 540 540 + walk->alignmask = alg->base.cra_alignmask; 378 541 379 379 - walk->blocksize = crypto_aead_blocksize(tfm); 380 380 - walk->stride = crypto_aead_chunksize(tfm); 381 381 - walk->ivsize = crypto_aead_ivsize(tfm); 382 382 - walk->alignmask = crypto_aead_alignmask(tfm); 383 383 - 384 384 - err = skcipher_walk_first(walk); 385 385 - 386 386 - if (atomic) 387 387 - walk->flags &= ~SKCIPHER_WALK_SLEEP; 388 388 - 389 389 - return err; 542 542 + return skcipher_walk_first(walk); 390 543 } 391 544 392 545 int skcipher_walk_aead_encrypt(struct skcipher_walk *walk,

-4

crypto/tcrypt.c

reviewed

··· 1738 1738 ret = min(ret, tcrypt_test("hmac(rmd160)")); 1739 1739 break; 1740 1740 1741 1741 - case 109: 1742 1742 - ret = min(ret, tcrypt_test("vmac64(aes)")); 1743 1743 - break; 1744 1744 - 1745 1741 case 111: 1746 1742 ret = min(ret, tcrypt_test("hmac(sha3-224)")); 1747 1743 break;

+33 -50

crypto/tea.c

reviewed

··· 18 18 #include <linux/init.h> 19 19 #include <linux/module.h> 20 20 #include <linux/mm.h> 21 21 - #include <asm/byteorder.h> 21 21 + #include <linux/unaligned.h> 22 22 #include <linux/types.h> 23 23 24 24 #define TEA_KEY_SIZE 16 ··· 43 43 unsigned int key_len) 44 44 { 45 45 struct tea_ctx *ctx = crypto_tfm_ctx(tfm); 46 46 - const __le32 *key = (const __le32 *)in_key; 47 46 48 48 - ctx->KEY[0] = le32_to_cpu(key[0]); 49 49 - ctx->KEY[1] = le32_to_cpu(key[1]); 50 50 - ctx->KEY[2] = le32_to_cpu(key[2]); 51 51 - ctx->KEY[3] = le32_to_cpu(key[3]); 47 47 + ctx->KEY[0] = get_unaligned_le32(&in_key[0]); 48 48 + ctx->KEY[1] = get_unaligned_le32(&in_key[4]); 49 49 + ctx->KEY[2] = get_unaligned_le32(&in_key[8]); 50 50 + ctx->KEY[3] = get_unaligned_le32(&in_key[12]); 52 51 53 52 return 0; 54 53 ··· 58 59 u32 y, z, n, sum = 0; 59 60 u32 k0, k1, k2, k3; 60 61 struct tea_ctx *ctx = crypto_tfm_ctx(tfm); 61 61 - const __le32 *in = (const __le32 *)src; 62 62 - __le32 *out = (__le32 *)dst; 63 62 64 64 - y = le32_to_cpu(in[0]); 65 65 - z = le32_to_cpu(in[1]); 63 63 + y = get_unaligned_le32(&src[0]); 64 64 + z = get_unaligned_le32(&src[4]); 66 65 67 66 k0 = ctx->KEY[0]; 68 67 k1 = ctx->KEY[1]; ··· 75 78 z += ((y << 4) + k2) ^ (y + sum) ^ ((y >> 5) + k3); 76 79 } 77 80 78 78 - out[0] = cpu_to_le32(y); 79 79 - out[1] = cpu_to_le32(z); 81 81 + put_unaligned_le32(y, &dst[0]); 82 82 + put_unaligned_le32(z, &dst[4]); 80 83 } 81 84 82 85 static void tea_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) ··· 84 87 u32 y, z, n, sum; 85 88 u32 k0, k1, k2, k3; 86 89 struct tea_ctx *ctx = crypto_tfm_ctx(tfm); 87 87 - const __le32 *in = (const __le32 *)src; 88 88 - __le32 *out = (__le32 *)dst; 89 90 90 90 - y = le32_to_cpu(in[0]); 91 91 - z = le32_to_cpu(in[1]); 91 91 + y = get_unaligned_le32(&src[0]); 92 92 + z = get_unaligned_le32(&src[4]); 92 93 93 94 k0 = ctx->KEY[0]; 94 95 k1 = ctx->KEY[1]; ··· 103 108 sum -= TEA_DELTA; 104 109 } 105 110 106 106 - out[0] = cpu_to_le32(y); 107 107 - out[1] = cpu_to_le32(z); 111 111 + put_unaligned_le32(y, &dst[0]); 112 112 + put_unaligned_le32(z, &dst[4]); 108 113 } 109 114 110 115 static int xtea_setkey(struct crypto_tfm *tfm, const u8 *in_key, 111 116 unsigned int key_len) 112 117 { 113 118 struct xtea_ctx *ctx = crypto_tfm_ctx(tfm); 114 114 - const __le32 *key = (const __le32 *)in_key; 115 119 116 116 - ctx->KEY[0] = le32_to_cpu(key[0]); 117 117 - ctx->KEY[1] = le32_to_cpu(key[1]); 118 118 - ctx->KEY[2] = le32_to_cpu(key[2]); 119 119 - ctx->KEY[3] = le32_to_cpu(key[3]); 120 120 + ctx->KEY[0] = get_unaligned_le32(&in_key[0]); 121 121 + ctx->KEY[1] = get_unaligned_le32(&in_key[4]); 122 122 + ctx->KEY[2] = get_unaligned_le32(&in_key[8]); 123 123 + ctx->KEY[3] = get_unaligned_le32(&in_key[12]); 120 124 121 125 return 0; 122 126 ··· 126 132 u32 y, z, sum = 0; 127 133 u32 limit = XTEA_DELTA * XTEA_ROUNDS; 128 134 struct xtea_ctx *ctx = crypto_tfm_ctx(tfm); 129 129 - const __le32 *in = (const __le32 *)src; 130 130 - __le32 *out = (__le32 *)dst; 131 135 132 132 - y = le32_to_cpu(in[0]); 133 133 - z = le32_to_cpu(in[1]); 136 136 + y = get_unaligned_le32(&src[0]); 137 137 + z = get_unaligned_le32(&src[4]); 134 138 135 139 while (sum != limit) { 136 140 y += ((z << 4 ^ z >> 5) + z) ^ (sum + ctx->KEY[sum&3]); ··· 136 144 z += ((y << 4 ^ y >> 5) + y) ^ (sum + ctx->KEY[sum>>11 &3]); 137 145 } 138 146 139 139 - out[0] = cpu_to_le32(y); 140 140 - out[1] = cpu_to_le32(z); 147 147 + put_unaligned_le32(y, &dst[0]); 148 148 + put_unaligned_le32(z, &dst[4]); 141 149 } 142 150 143 151 static void xtea_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 144 152 { 145 153 u32 y, z, sum; 146 154 struct tea_ctx *ctx = crypto_tfm_ctx(tfm); 147 147 - const __le32 *in = (const __le32 *)src; 148 148 - __le32 *out = (__le32 *)dst; 149 155 150 150 - y = le32_to_cpu(in[0]); 151 151 - z = le32_to_cpu(in[1]); 156 156 + y = get_unaligned_le32(&src[0]); 157 157 + z = get_unaligned_le32(&src[4]); 152 158 153 159 sum = XTEA_DELTA * XTEA_ROUNDS; 154 160 ··· 156 166 y -= ((z << 4 ^ z >> 5) + z) ^ (sum + ctx->KEY[sum & 3]); 157 167 } 158 168 159 159 - out[0] = cpu_to_le32(y); 160 160 - out[1] = cpu_to_le32(z); 169 169 + put_unaligned_le32(y, &dst[0]); 170 170 + put_unaligned_le32(z, &dst[4]); 161 171 } 162 172 163 173 ··· 166 176 u32 y, z, sum = 0; 167 177 u32 limit = XTEA_DELTA * XTEA_ROUNDS; 168 178 struct xtea_ctx *ctx = crypto_tfm_ctx(tfm); 169 169 - const __le32 *in = (const __le32 *)src; 170 170 - __le32 *out = (__le32 *)dst; 171 179 172 172 - y = le32_to_cpu(in[0]); 173 173 - z = le32_to_cpu(in[1]); 180 180 + y = get_unaligned_le32(&src[0]); 181 181 + z = get_unaligned_le32(&src[4]); 174 182 175 183 while (sum != limit) { 176 184 y += (z << 4 ^ z >> 5) + (z ^ sum) + ctx->KEY[sum&3]; ··· 176 188 z += (y << 4 ^ y >> 5) + (y ^ sum) + ctx->KEY[sum>>11 &3]; 177 189 } 178 190 179 179 - out[0] = cpu_to_le32(y); 180 180 - out[1] = cpu_to_le32(z); 191 191 + put_unaligned_le32(y, &dst[0]); 192 192 + put_unaligned_le32(z, &dst[4]); 181 193 } 182 194 183 195 static void xeta_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) 184 196 { 185 197 u32 y, z, sum; 186 198 struct tea_ctx *ctx = crypto_tfm_ctx(tfm); 187 187 - const __le32 *in = (const __le32 *)src; 188 188 - __le32 *out = (__le32 *)dst; 189 199 190 190 - y = le32_to_cpu(in[0]); 191 191 - z = le32_to_cpu(in[1]); 200 200 + y = get_unaligned_le32(&src[0]); 201 201 + z = get_unaligned_le32(&src[4]); 192 202 193 203 sum = XTEA_DELTA * XTEA_ROUNDS; 194 204 ··· 196 210 y -= (z << 4 ^ z >> 5) + (z ^ sum) + ctx->KEY[sum & 3]; 197 211 } 198 212 199 199 - out[0] = cpu_to_le32(y); 200 200 - out[1] = cpu_to_le32(z); 213 213 + put_unaligned_le32(y, &dst[0]); 214 214 + put_unaligned_le32(z, &dst[4]); 201 215 } 202 216 203 217 static struct crypto_alg tea_algs[3] = { { ··· 206 220 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 207 221 .cra_blocksize = TEA_BLOCK_SIZE, 208 222 .cra_ctxsize = sizeof (struct tea_ctx), 209 209 - .cra_alignmask = 3, 210 223 .cra_module = THIS_MODULE, 211 224 .cra_u = { .cipher = { 212 225 .cia_min_keysize = TEA_KEY_SIZE, ··· 219 234 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 220 235 .cra_blocksize = XTEA_BLOCK_SIZE, 221 236 .cra_ctxsize = sizeof (struct xtea_ctx), 222 222 - .cra_alignmask = 3, 223 237 .cra_module = THIS_MODULE, 224 238 .cra_u = { .cipher = { 225 239 .cia_min_keysize = XTEA_KEY_SIZE, ··· 232 248 .cra_flags = CRYPTO_ALG_TYPE_CIPHER, 233 249 .cra_blocksize = XTEA_BLOCK_SIZE, 234 250 .cra_ctxsize = sizeof (struct xtea_ctx), 235 235 - .cra_alignmask = 3, 236 251 .cra_module = THIS_MODULE, 237 252 .cra_u = { .cipher = { 238 253 .cia_min_keysize = XTEA_KEY_SIZE,

+1 -25

crypto/testmgr.c

reviewed

··· 2885 2885 if (ivsize) { 2886 2886 if (WARN_ON(ivsize > MAX_IVLEN)) 2887 2887 return -EINVAL; 2888 2888 - if (vec->generates_iv && !enc) 2889 2889 - memcpy(iv, vec->iv_out, ivsize); 2890 2890 - else if (vec->iv) 2888 2888 + if (vec->iv) 2891 2889 memcpy(iv, vec->iv, ivsize); 2892 2890 else 2893 2891 memset(iv, 0, ivsize); 2894 2892 } else { 2895 2895 - if (vec->generates_iv) { 2896 2896 - pr_err("alg: skcipher: %s has ivsize=0 but test vector %s generates IV!\n", 2897 2897 - driver, vec_name); 2898 2898 - return -EINVAL; 2899 2899 - } 2900 2893 iv = NULL; 2901 2894 } 2902 2895 ··· 3124 3131 int err; 3125 3132 3126 3133 if (noextratests) 3127 3127 - return 0; 3128 3128 - 3129 3129 - /* Keywrap isn't supported here yet as it handles its IV differently. */ 3130 3130 - if (strncmp(algname, "kw(", 3) == 0) 3131 3134 return 0; 3132 3135 3133 3136 init_rnd_state(&rng); ··· 5398 5409 .fips_allowed = 1, 5399 5410 .test = alg_test_null, 5400 5411 }, { 5401 5401 - .alg = "kw(aes)", 5402 5402 - .test = alg_test_skcipher, 5403 5403 - .fips_allowed = 1, 5404 5404 - .suite = { 5405 5405 - .cipher = __VECS(aes_kw_tv_template) 5406 5406 - } 5407 5407 - }, { 5408 5412 .alg = "lrw(aes)", 5409 5413 .generic_driver = "lrw(ecb(aes-generic))", 5410 5414 .test = alg_test_skcipher, ··· 5730 5748 .test = alg_test_hash, 5731 5749 .suite = { 5732 5750 .hash = __VECS(streebog512_tv_template) 5733 5733 - } 5734 5734 - }, { 5735 5735 - .alg = "vmac64(aes)", 5736 5736 - .test = alg_test_hash, 5737 5737 - .suite = { 5738 5738 - .hash = __VECS(vmac64_aes_tv_template) 5739 5751 } 5740 5752 }, { 5741 5753 .alg = "wp256",

-192

crypto/testmgr.h

reviewed

··· 59 59 * @wk: Does the test need CRYPTO_TFM_REQ_FORBID_WEAK_KEYS? 60 60 * ( e.g. test needs to fail due to a weak key ) 61 61 * @fips_skip: Skip the test vector in FIPS mode 62 62 - * @generates_iv: Encryption should ignore the given IV, and output @iv_out. 63 63 - * Decryption takes @iv_out. Needed for AES Keywrap ("kw(aes)"). 64 62 * @setkey_error: Expected error from setkey() 65 63 * @crypt_error: Expected error from encrypt() and decrypt() 66 64 */ ··· 72 74 unsigned short klen; 73 75 unsigned int len; 74 76 bool fips_skip; 75 75 - bool generates_iv; 76 77 int setkey_error; 77 78 int crypt_error; 78 79 }; ··· 8556 8559 .psize = 34, 8557 8560 .ksize = 16, 8558 8561 } 8559 8559 - }; 8560 8560 - 8561 8561 - static const char vmac64_string1[144] = { 8562 8562 - '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 8563 8563 - '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 8564 8564 - '\x01', '\x01', '\x01', '\x01', '\x02', '\x03', '\x02', '\x02', 8565 8565 - '\x02', '\x04', '\x01', '\x07', '\x04', '\x01', '\x04', '\x03', 8566 8566 - }; 8567 8567 - 8568 8568 - static const char vmac64_string2[144] = { 8569 8569 - '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 8570 8570 - '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 8571 8571 - 'a', 'b', 'c', 8572 8572 - }; 8573 8573 - 8574 8574 - static const char vmac64_string3[144] = { 8575 8575 - '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 8576 8576 - '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 8577 8577 - 'a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 8578 8578 - 'c', 'a', 'b', 'c', 'a', 'b', 'c', 'a', 8579 8579 - 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c', 8580 8580 - 'a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 8581 8581 - 'c', 'a', 'b', 'c', 'a', 'b', 'c', 'a', 8582 8582 - 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c', 8583 8583 - }; 8584 8584 - 8585 8585 - static const char vmac64_string4[33] = { 8586 8586 - '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 8587 8587 - '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 8588 8588 - 'b', 'c', 'e', 'f', 'i', 'j', 'l', 'm', 8589 8589 - 'o', 'p', 'r', 's', 't', 'u', 'w', 'x', 8590 8590 - 'z', 8591 8591 - }; 8592 8592 - 8593 8593 - static const char vmac64_string5[143] = { 8594 8594 - '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 8595 8595 - '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 8596 8596 - 'r', 'm', 'b', 't', 'c', 'o', 'l', 'k', 8597 8597 - ']', '%', '9', '2', '7', '!', 'A', 8598 8598 - }; 8599 8599 - 8600 8600 - static const char vmac64_string6[145] = { 8601 8601 - '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 8602 8602 - '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', 8603 8603 - 'p', 't', '*', '7', 'l', 'i', '!', '#', 8604 8604 - 'w', '0', 'z', '/', '4', 'A', 'n', 8605 8605 - }; 8606 8606 - 8607 8607 - static const struct hash_testvec vmac64_aes_tv_template[] = { 8608 8608 - { /* draft-krovetz-vmac-01 test vector 1 */ 8609 8609 - .key = "abcdefghijklmnop", 8610 8610 - .ksize = 16, 8611 8611 - .plaintext = "\0\0\0\0\0\0\0\0bcdefghi", 8612 8612 - .psize = 16, 8613 8613 - .digest = "\x25\x76\xbe\x1c\x56\xd8\xb8\x1b", 8614 8614 - }, { /* draft-krovetz-vmac-01 test vector 2 */ 8615 8615 - .key = "abcdefghijklmnop", 8616 8616 - .ksize = 16, 8617 8617 - .plaintext = "\0\0\0\0\0\0\0\0bcdefghiabc", 8618 8618 - .psize = 19, 8619 8619 - .digest = "\x2d\x37\x6c\xf5\xb1\x81\x3c\xe5", 8620 8620 - }, { /* draft-krovetz-vmac-01 test vector 3 */ 8621 8621 - .key = "abcdefghijklmnop", 8622 8622 - .ksize = 16, 8623 8623 - .plaintext = "\0\0\0\0\0\0\0\0bcdefghi" 8624 8624 - "abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc", 8625 8625 - .psize = 64, 8626 8626 - .digest = "\xe8\x42\x1f\x61\xd5\x73\xd2\x98", 8627 8627 - }, { /* draft-krovetz-vmac-01 test vector 4 */ 8628 8628 - .key = "abcdefghijklmnop", 8629 8629 - .ksize = 16, 8630 8630 - .plaintext = "\0\0\0\0\0\0\0\0bcdefghi" 8631 8631 - "abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc" 8632 8632 - "abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc" 8633 8633 - "abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc" 8634 8634 - "abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc" 8635 8635 - "abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc" 8636 8636 - "abcabcabcabcabcabcabcabcabcabcabcabcabcabcabc", 8637 8637 - .psize = 316, 8638 8638 - .digest = "\x44\x92\xdf\x6c\x5c\xac\x1b\xbe", 8639 8639 - }, { 8640 8640 - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" 8641 8641 - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 8642 8642 - .ksize = 16, 8643 8643 - .plaintext = "\x00\x00\x00\x00\x00\x00\x00\x00" 8644 8644 - "\x00\x00\x00\x00\x00\x00\x00\x00", 8645 8645 - .psize = 16, 8646 8646 - .digest = "\x54\x7b\xa4\x77\x35\x80\x58\x07", 8647 8647 - }, { 8648 8648 - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" 8649 8649 - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 8650 8650 - .ksize = 16, 8651 8651 - .plaintext = vmac64_string1, 8652 8652 - .psize = sizeof(vmac64_string1), 8653 8653 - .digest = "\xa1\x8c\x68\xae\xd3\x3c\xf5\xce", 8654 8654 - }, { 8655 8655 - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" 8656 8656 - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 8657 8657 - .ksize = 16, 8658 8658 - .plaintext = vmac64_string2, 8659 8659 - .psize = sizeof(vmac64_string2), 8660 8660 - .digest = "\x2d\x14\xbd\x81\x73\xb0\x27\xc9", 8661 8661 - }, { 8662 8662 - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" 8663 8663 - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 8664 8664 - .ksize = 16, 8665 8665 - .plaintext = vmac64_string3, 8666 8666 - .psize = sizeof(vmac64_string3), 8667 8667 - .digest = "\x19\x0b\x47\x98\x8c\x95\x1a\x8d", 8668 8668 - }, { 8669 8669 - .key = "abcdefghijklmnop", 8670 8670 - .ksize = 16, 8671 8671 - .plaintext = "\x00\x00\x00\x00\x00\x00\x00\x00" 8672 8672 - "\x00\x00\x00\x00\x00\x00\x00\x00", 8673 8673 - .psize = 16, 8674 8674 - .digest = "\x84\x8f\x55\x9e\x26\xa1\x89\x3b", 8675 8675 - }, { 8676 8676 - .key = "abcdefghijklmnop", 8677 8677 - .ksize = 16, 8678 8678 - .plaintext = vmac64_string1, 8679 8679 - .psize = sizeof(vmac64_string1), 8680 8680 - .digest = "\xc2\x74\x8d\xf6\xb0\xab\x5e\xab", 8681 8681 - }, { 8682 8682 - .key = "abcdefghijklmnop", 8683 8683 - .ksize = 16, 8684 8684 - .plaintext = vmac64_string2, 8685 8685 - .psize = sizeof(vmac64_string2), 8686 8686 - .digest = "\xdf\x09\x7b\x3d\x42\x68\x15\x11", 8687 8687 - }, { 8688 8688 - .key = "abcdefghijklmnop", 8689 8689 - .ksize = 16, 8690 8690 - .plaintext = vmac64_string3, 8691 8691 - .psize = sizeof(vmac64_string3), 8692 8692 - .digest = "\xd4\xfa\x8f\xed\xe1\x8f\x32\x8b", 8693 8693 - }, { 8694 8694 - .key = "a09b5cd!f#07K\x00\x00\x00", 8695 8695 - .ksize = 16, 8696 8696 - .plaintext = vmac64_string4, 8697 8697 - .psize = sizeof(vmac64_string4), 8698 8698 - .digest = "\x5f\xa1\x4e\x42\xea\x0f\xa5\xab", 8699 8699 - }, { 8700 8700 - .key = "a09b5cd!f#07K\x00\x00\x00", 8701 8701 - .ksize = 16, 8702 8702 - .plaintext = vmac64_string5, 8703 8703 - .psize = sizeof(vmac64_string5), 8704 8704 - .digest = "\x60\x67\xe8\x1d\xbc\x98\x31\x25", 8705 8705 - }, { 8706 8706 - .key = "a09b5cd!f#07K\x00\x00\x00", 8707 8707 - .ksize = 16, 8708 8708 - .plaintext = vmac64_string6, 8709 8709 - .psize = sizeof(vmac64_string6), 8710 8710 - .digest = "\x41\xeb\x65\x95\x47\x9b\xae\xc4", 8711 8711 - }, 8712 8562 }; 8713 8563 8714 8564 /* ··· 24189 24345 "\xf5\x57\x0f\x2f\x49\x0e\x11\x3b" 24190 24346 "\x78\x93\xec\xfc\xf4\xff\xe1\x2d", 24191 24347 .clen = 24, 24192 24192 - }, 24193 24193 - }; 24194 24194 - 24195 24195 - /* 24196 24196 - * All key wrapping test vectors taken from 24197 24197 - * http://csrc.nist.gov/groups/STM/cavp/documents/mac/kwtestvectors.zip 24198 24198 - * 24199 24199 - * Note: as documented in keywrap.c, the ivout for encryption is the first 24200 24200 - * semiblock of the ciphertext from the test vector. For decryption, iv is 24201 24201 - * the first semiblock of the ciphertext. 24202 24202 - */ 24203 24203 - static const struct cipher_testvec aes_kw_tv_template[] = { 24204 24204 - { 24205 24205 - .key = "\x75\x75\xda\x3a\x93\x60\x7c\xc2" 24206 24206 - "\xbf\xd8\xce\xc7\xaa\xdf\xd9\xa6", 24207 24207 - .klen = 16, 24208 24208 - .ptext = "\x42\x13\x6d\x3c\x38\x4a\x3e\xea" 24209 24209 - "\xc9\x5a\x06\x6f\xd2\x8f\xed\x3f", 24210 24210 - .ctext = "\xf6\x85\x94\x81\x6f\x64\xca\xa3" 24211 24211 - "\xf5\x6f\xab\xea\x25\x48\xf5\xfb", 24212 24212 - .len = 16, 24213 24213 - .iv_out = "\x03\x1f\x6b\xd7\xe6\x1e\x64\x3d", 24214 24214 - .generates_iv = true, 24215 24215 - }, { 24216 24216 - .key = "\x80\xaa\x99\x73\x27\xa4\x80\x6b" 24217 24217 - "\x6a\x7a\x41\xa5\x2b\x86\xc3\x71" 24218 24218 - "\x03\x86\xf9\x32\x78\x6e\xf7\x96" 24219 24219 - "\x76\xfa\xfb\x90\xb8\x26\x3c\x5f", 24220 24220 - .klen = 32, 24221 24221 - .ptext = "\x0a\x25\x6b\xa7\x5c\xfa\x03\xaa" 24222 24222 - "\xa0\x2b\xa9\x42\x03\xf1\x5b\xaa", 24223 24223 - .ctext = "\xd3\x3d\x3d\x97\x7b\xf0\xa9\x15" 24224 24224 - "\x59\xf9\x9c\x8a\xcd\x29\x3d\x43", 24225 24225 - .len = 16, 24226 24226 - .iv_out = "\x42\x3c\x96\x0d\x8a\x2a\xc4\xc1", 24227 24227 - .generates_iv = true, 24228 24348 }, 24229 24349 }; 24230 24350

-696

crypto/vmac.c

reviewed

··· 1 1 - /* 2 2 - * VMAC: Message Authentication Code using Universal Hashing 3 3 - * 4 4 - * Reference: https://tools.ietf.org/html/draft-krovetz-vmac-01 5 5 - * 6 6 - * Copyright (c) 2009, Intel Corporation. 7 7 - * Copyright (c) 2018, Google Inc. 8 8 - * 9 9 - * This program is free software; you can redistribute it and/or modify it 10 10 - * under the terms and conditions of the GNU General Public License, 11 11 - * version 2, as published by the Free Software Foundation. 12 12 - * 13 13 - * This program is distributed in the hope it will be useful, but WITHOUT 14 14 - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 15 15 - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 16 16 - * more details. 17 17 - * 18 18 - * You should have received a copy of the GNU General Public License along with 19 19 - * this program; if not, write to the Free Software Foundation, Inc., 59 Temple 20 20 - * Place - Suite 330, Boston, MA 02111-1307 USA. 21 21 - */ 22 22 - 23 23 - /* 24 24 - * Derived from: 25 25 - * VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai. 26 26 - * This implementation is herby placed in the public domain. 27 27 - * The authors offers no warranty. Use at your own risk. 28 28 - * Last modified: 17 APR 08, 1700 PDT 29 29 - */ 30 30 - 31 31 - #include <linux/unaligned.h> 32 32 - #include <linux/init.h> 33 33 - #include <linux/types.h> 34 34 - #include <linux/crypto.h> 35 35 - #include <linux/module.h> 36 36 - #include <linux/scatterlist.h> 37 37 - #include <asm/byteorder.h> 38 38 - #include <crypto/scatterwalk.h> 39 39 - #include <crypto/internal/cipher.h> 40 40 - #include <crypto/internal/hash.h> 41 41 - 42 42 - /* 43 43 - * User definable settings. 44 44 - */ 45 45 - #define VMAC_TAG_LEN 64 46 46 - #define VMAC_KEY_SIZE 128/* Must be 128, 192 or 256 */ 47 47 - #define VMAC_KEY_LEN (VMAC_KEY_SIZE/8) 48 48 - #define VMAC_NHBYTES 128/* Must 2^i for any 3 < i < 13 Standard = 128*/ 49 49 - #define VMAC_NONCEBYTES 16 50 50 - 51 51 - /* per-transform (per-key) context */ 52 52 - struct vmac_tfm_ctx { 53 53 - struct crypto_cipher *cipher; 54 54 - u64 nhkey[(VMAC_NHBYTES/8)+2*(VMAC_TAG_LEN/64-1)]; 55 55 - u64 polykey[2*VMAC_TAG_LEN/64]; 56 56 - u64 l3key[2*VMAC_TAG_LEN/64]; 57 57 - }; 58 58 - 59 59 - /* per-request context */ 60 60 - struct vmac_desc_ctx { 61 61 - union { 62 62 - u8 partial[VMAC_NHBYTES]; /* partial block */ 63 63 - __le64 partial_words[VMAC_NHBYTES / 8]; 64 64 - }; 65 65 - unsigned int partial_size; /* size of the partial block */ 66 66 - bool first_block_processed; 67 67 - u64 polytmp[2*VMAC_TAG_LEN/64]; /* running total of L2-hash */ 68 68 - union { 69 69 - u8 bytes[VMAC_NONCEBYTES]; 70 70 - __be64 pads[VMAC_NONCEBYTES / 8]; 71 71 - } nonce; 72 72 - unsigned int nonce_size; /* nonce bytes filled so far */ 73 73 - }; 74 74 - 75 75 - /* 76 76 - * Constants and masks 77 77 - */ 78 78 - #define UINT64_C(x) x##ULL 79 79 - static const u64 p64 = UINT64_C(0xfffffffffffffeff); /* 2^64 - 257 prime */ 80 80 - static const u64 m62 = UINT64_C(0x3fffffffffffffff); /* 62-bit mask */ 81 81 - static const u64 m63 = UINT64_C(0x7fffffffffffffff); /* 63-bit mask */ 82 82 - static const u64 m64 = UINT64_C(0xffffffffffffffff); /* 64-bit mask */ 83 83 - static const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */ 84 84 - 85 85 - #define pe64_to_cpup le64_to_cpup /* Prefer little endian */ 86 86 - 87 87 - #ifdef __LITTLE_ENDIAN 88 88 - #define INDEX_HIGH 1 89 89 - #define INDEX_LOW 0 90 90 - #else 91 91 - #define INDEX_HIGH 0 92 92 - #define INDEX_LOW 1 93 93 - #endif 94 94 - 95 95 - /* 96 96 - * The following routines are used in this implementation. They are 97 97 - * written via macros to simulate zero-overhead call-by-reference. 98 98 - * 99 99 - * MUL64: 64x64->128-bit multiplication 100 100 - * PMUL64: assumes top bits cleared on inputs 101 101 - * ADD128: 128x128->128-bit addition 102 102 - */ 103 103 - 104 104 - #define ADD128(rh, rl, ih, il) \ 105 105 - do { \ 106 106 - u64 _il = (il); \ 107 107 - (rl) += (_il); \ 108 108 - if ((rl) < (_il)) \ 109 109 - (rh)++; \ 110 110 - (rh) += (ih); \ 111 111 - } while (0) 112 112 - 113 113 - #define MUL32(i1, i2) ((u64)(u32)(i1)*(u32)(i2)) 114 114 - 115 115 - #define PMUL64(rh, rl, i1, i2) /* Assumes m doesn't overflow */ \ 116 116 - do { \ 117 117 - u64 _i1 = (i1), _i2 = (i2); \ 118 118 - u64 m = MUL32(_i1, _i2>>32) + MUL32(_i1>>32, _i2); \ 119 119 - rh = MUL32(_i1>>32, _i2>>32); \ 120 120 - rl = MUL32(_i1, _i2); \ 121 121 - ADD128(rh, rl, (m >> 32), (m << 32)); \ 122 122 - } while (0) 123 123 - 124 124 - #define MUL64(rh, rl, i1, i2) \ 125 125 - do { \ 126 126 - u64 _i1 = (i1), _i2 = (i2); \ 127 127 - u64 m1 = MUL32(_i1, _i2>>32); \ 128 128 - u64 m2 = MUL32(_i1>>32, _i2); \ 129 129 - rh = MUL32(_i1>>32, _i2>>32); \ 130 130 - rl = MUL32(_i1, _i2); \ 131 131 - ADD128(rh, rl, (m1 >> 32), (m1 << 32)); \ 132 132 - ADD128(rh, rl, (m2 >> 32), (m2 << 32)); \ 133 133 - } while (0) 134 134 - 135 135 - /* 136 136 - * For highest performance the L1 NH and L2 polynomial hashes should be 137 137 - * carefully implemented to take advantage of one's target architecture. 138 138 - * Here these two hash functions are defined multiple time; once for 139 139 - * 64-bit architectures, once for 32-bit SSE2 architectures, and once 140 140 - * for the rest (32-bit) architectures. 141 141 - * For each, nh_16 *must* be defined (works on multiples of 16 bytes). 142 142 - * Optionally, nh_vmac_nhbytes can be defined (for multiples of 143 143 - * VMAC_NHBYTES), and nh_16_2 and nh_vmac_nhbytes_2 (versions that do two 144 144 - * NH computations at once). 145 145 - */ 146 146 - 147 147 - #ifdef CONFIG_64BIT 148 148 - 149 149 - #define nh_16(mp, kp, nw, rh, rl) \ 150 150 - do { \ 151 151 - int i; u64 th, tl; \ 152 152 - rh = rl = 0; \ 153 153 - for (i = 0; i < nw; i += 2) { \ 154 154 - MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \ 155 155 - pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \ 156 156 - ADD128(rh, rl, th, tl); \ 157 157 - } \ 158 158 - } while (0) 159 159 - 160 160 - #define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1) \ 161 161 - do { \ 162 162 - int i; u64 th, tl; \ 163 163 - rh1 = rl1 = rh = rl = 0; \ 164 164 - for (i = 0; i < nw; i += 2) { \ 165 165 - MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \ 166 166 - pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \ 167 167 - ADD128(rh, rl, th, tl); \ 168 168 - MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \ 169 169 - pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \ 170 170 - ADD128(rh1, rl1, th, tl); \ 171 171 - } \ 172 172 - } while (0) 173 173 - 174 174 - #if (VMAC_NHBYTES >= 64) /* These versions do 64-bytes of message at a time */ 175 175 - #define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \ 176 176 - do { \ 177 177 - int i; u64 th, tl; \ 178 178 - rh = rl = 0; \ 179 179 - for (i = 0; i < nw; i += 8) { \ 180 180 - MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \ 181 181 - pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \ 182 182 - ADD128(rh, rl, th, tl); \ 183 183 - MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \ 184 184 - pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \ 185 185 - ADD128(rh, rl, th, tl); \ 186 186 - MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \ 187 187 - pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \ 188 188 - ADD128(rh, rl, th, tl); \ 189 189 - MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \ 190 190 - pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \ 191 191 - ADD128(rh, rl, th, tl); \ 192 192 - } \ 193 193 - } while (0) 194 194 - 195 195 - #define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1) \ 196 196 - do { \ 197 197 - int i; u64 th, tl; \ 198 198 - rh1 = rl1 = rh = rl = 0; \ 199 199 - for (i = 0; i < nw; i += 8) { \ 200 200 - MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \ 201 201 - pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \ 202 202 - ADD128(rh, rl, th, tl); \ 203 203 - MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \ 204 204 - pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \ 205 205 - ADD128(rh1, rl1, th, tl); \ 206 206 - MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \ 207 207 - pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \ 208 208 - ADD128(rh, rl, th, tl); \ 209 209 - MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+4], \ 210 210 - pe64_to_cpup((mp)+i+3)+(kp)[i+5]); \ 211 211 - ADD128(rh1, rl1, th, tl); \ 212 212 - MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \ 213 213 - pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \ 214 214 - ADD128(rh, rl, th, tl); \ 215 215 - MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+6], \ 216 216 - pe64_to_cpup((mp)+i+5)+(kp)[i+7]); \ 217 217 - ADD128(rh1, rl1, th, tl); \ 218 218 - MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \ 219 219 - pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \ 220 220 - ADD128(rh, rl, th, tl); \ 221 221 - MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+8], \ 222 222 - pe64_to_cpup((mp)+i+7)+(kp)[i+9]); \ 223 223 - ADD128(rh1, rl1, th, tl); \ 224 224 - } \ 225 225 - } while (0) 226 226 - #endif 227 227 - 228 228 - #define poly_step(ah, al, kh, kl, mh, ml) \ 229 229 - do { \ 230 230 - u64 t1h, t1l, t2h, t2l, t3h, t3l, z = 0; \ 231 231 - /* compute ab*cd, put bd into result registers */ \ 232 232 - PMUL64(t3h, t3l, al, kh); \ 233 233 - PMUL64(t2h, t2l, ah, kl); \ 234 234 - PMUL64(t1h, t1l, ah, 2*kh); \ 235 235 - PMUL64(ah, al, al, kl); \ 236 236 - /* add 2 * ac to result */ \ 237 237 - ADD128(ah, al, t1h, t1l); \ 238 238 - /* add together ad + bc */ \ 239 239 - ADD128(t2h, t2l, t3h, t3l); \ 240 240 - /* now (ah,al), (t2l,2*t2h) need summing */ \ 241 241 - /* first add the high registers, carrying into t2h */ \ 242 242 - ADD128(t2h, ah, z, t2l); \ 243 243 - /* double t2h and add top bit of ah */ \ 244 244 - t2h = 2 * t2h + (ah >> 63); \ 245 245 - ah &= m63; \ 246 246 - /* now add the low registers */ \ 247 247 - ADD128(ah, al, mh, ml); \ 248 248 - ADD128(ah, al, z, t2h); \ 249 249 - } while (0) 250 250 - 251 251 - #else /* ! CONFIG_64BIT */ 252 252 - 253 253 - #ifndef nh_16 254 254 - #define nh_16(mp, kp, nw, rh, rl) \ 255 255 - do { \ 256 256 - u64 t1, t2, m1, m2, t; \ 257 257 - int i; \ 258 258 - rh = rl = t = 0; \ 259 259 - for (i = 0; i < nw; i += 2) { \ 260 260 - t1 = pe64_to_cpup(mp+i) + kp[i]; \ 261 261 - t2 = pe64_to_cpup(mp+i+1) + kp[i+1]; \ 262 262 - m2 = MUL32(t1 >> 32, t2); \ 263 263 - m1 = MUL32(t1, t2 >> 32); \ 264 264 - ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32), \ 265 265 - MUL32(t1, t2)); \ 266 266 - rh += (u64)(u32)(m1 >> 32) \ 267 267 - + (u32)(m2 >> 32); \ 268 268 - t += (u64)(u32)m1 + (u32)m2; \ 269 269 - } \ 270 270 - ADD128(rh, rl, (t >> 32), (t << 32)); \ 271 271 - } while (0) 272 272 - #endif 273 273 - 274 274 - static void poly_step_func(u64 *ahi, u64 *alo, 275 275 - const u64 *kh, const u64 *kl, 276 276 - const u64 *mh, const u64 *ml) 277 277 - { 278 278 - #define a0 (*(((u32 *)alo)+INDEX_LOW)) 279 279 - #define a1 (*(((u32 *)alo)+INDEX_HIGH)) 280 280 - #define a2 (*(((u32 *)ahi)+INDEX_LOW)) 281 281 - #define a3 (*(((u32 *)ahi)+INDEX_HIGH)) 282 282 - #define k0 (*(((u32 *)kl)+INDEX_LOW)) 283 283 - #define k1 (*(((u32 *)kl)+INDEX_HIGH)) 284 284 - #define k2 (*(((u32 *)kh)+INDEX_LOW)) 285 285 - #define k3 (*(((u32 *)kh)+INDEX_HIGH)) 286 286 - 287 287 - u64 p, q, t; 288 288 - u32 t2; 289 289 - 290 290 - p = MUL32(a3, k3); 291 291 - p += p; 292 292 - p += *(u64 *)mh; 293 293 - p += MUL32(a0, k2); 294 294 - p += MUL32(a1, k1); 295 295 - p += MUL32(a2, k0); 296 296 - t = (u32)(p); 297 297 - p >>= 32; 298 298 - p += MUL32(a0, k3); 299 299 - p += MUL32(a1, k2); 300 300 - p += MUL32(a2, k1); 301 301 - p += MUL32(a3, k0); 302 302 - t |= ((u64)((u32)p & 0x7fffffff)) << 32; 303 303 - p >>= 31; 304 304 - p += (u64)(((u32 *)ml)[INDEX_LOW]); 305 305 - p += MUL32(a0, k0); 306 306 - q = MUL32(a1, k3); 307 307 - q += MUL32(a2, k2); 308 308 - q += MUL32(a3, k1); 309 309 - q += q; 310 310 - p += q; 311 311 - t2 = (u32)(p); 312 312 - p >>= 32; 313 313 - p += (u64)(((u32 *)ml)[INDEX_HIGH]); 314 314 - p += MUL32(a0, k1); 315 315 - p += MUL32(a1, k0); 316 316 - q = MUL32(a2, k3); 317 317 - q += MUL32(a3, k2); 318 318 - q += q; 319 319 - p += q; 320 320 - *(u64 *)(alo) = (p << 32) | t2; 321 321 - p >>= 32; 322 322 - *(u64 *)(ahi) = p + t; 323 323 - 324 324 - #undef a0 325 325 - #undef a1 326 326 - #undef a2 327 327 - #undef a3 328 328 - #undef k0 329 329 - #undef k1 330 330 - #undef k2 331 331 - #undef k3 332 332 - } 333 333 - 334 334 - #define poly_step(ah, al, kh, kl, mh, ml) \ 335 335 - poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml)) 336 336 - 337 337 - #endif /* end of specialized NH and poly definitions */ 338 338 - 339 339 - /* At least nh_16 is defined. Defined others as needed here */ 340 340 - #ifndef nh_16_2 341 341 - #define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2) \ 342 342 - do { \ 343 343 - nh_16(mp, kp, nw, rh, rl); \ 344 344 - nh_16(mp, ((kp)+2), nw, rh2, rl2); \ 345 345 - } while (0) 346 346 - #endif 347 347 - #ifndef nh_vmac_nhbytes 348 348 - #define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \ 349 349 - nh_16(mp, kp, nw, rh, rl) 350 350 - #endif 351 351 - #ifndef nh_vmac_nhbytes_2 352 352 - #define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2) \ 353 353 - do { \ 354 354 - nh_vmac_nhbytes(mp, kp, nw, rh, rl); \ 355 355 - nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2); \ 356 356 - } while (0) 357 357 - #endif 358 358 - 359 359 - static u64 l3hash(u64 p1, u64 p2, u64 k1, u64 k2, u64 len) 360 360 - { 361 361 - u64 rh, rl, t, z = 0; 362 362 - 363 363 - /* fully reduce (p1,p2)+(len,0) mod p127 */ 364 364 - t = p1 >> 63; 365 365 - p1 &= m63; 366 366 - ADD128(p1, p2, len, t); 367 367 - /* At this point, (p1,p2) is at most 2^127+(len<<64) */ 368 368 - t = (p1 > m63) + ((p1 == m63) && (p2 == m64)); 369 369 - ADD128(p1, p2, z, t); 370 370 - p1 &= m63; 371 371 - 372 372 - /* compute (p1,p2)/(2^64-2^32) and (p1,p2)%(2^64-2^32) */ 373 373 - t = p1 + (p2 >> 32); 374 374 - t += (t >> 32); 375 375 - t += (u32)t > 0xfffffffeu; 376 376 - p1 += (t >> 32); 377 377 - p2 += (p1 << 32); 378 378 - 379 379 - /* compute (p1+k1)%p64 and (p2+k2)%p64 */ 380 380 - p1 += k1; 381 381 - p1 += (0 - (p1 < k1)) & 257; 382 382 - p2 += k2; 383 383 - p2 += (0 - (p2 < k2)) & 257; 384 384 - 385 385 - /* compute (p1+k1)*(p2+k2)%p64 */ 386 386 - MUL64(rh, rl, p1, p2); 387 387 - t = rh >> 56; 388 388 - ADD128(t, rl, z, rh); 389 389 - rh <<= 8; 390 390 - ADD128(t, rl, z, rh); 391 391 - t += t << 8; 392 392 - rl += t; 393 393 - rl += (0 - (rl < t)) & 257; 394 394 - rl += (0 - (rl > p64-1)) & 257; 395 395 - return rl; 396 396 - } 397 397 - 398 398 - /* L1 and L2-hash one or more VMAC_NHBYTES-byte blocks */ 399 399 - static void vhash_blocks(const struct vmac_tfm_ctx *tctx, 400 400 - struct vmac_desc_ctx *dctx, 401 401 - const __le64 *mptr, unsigned int blocks) 402 402 - { 403 403 - const u64 *kptr = tctx->nhkey; 404 404 - const u64 pkh = tctx->polykey[0]; 405 405 - const u64 pkl = tctx->polykey[1]; 406 406 - u64 ch = dctx->polytmp[0]; 407 407 - u64 cl = dctx->polytmp[1]; 408 408 - u64 rh, rl; 409 409 - 410 410 - if (!dctx->first_block_processed) { 411 411 - dctx->first_block_processed = true; 412 412 - nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl); 413 413 - rh &= m62; 414 414 - ADD128(ch, cl, rh, rl); 415 415 - mptr += (VMAC_NHBYTES/sizeof(u64)); 416 416 - blocks--; 417 417 - } 418 418 - 419 419 - while (blocks--) { 420 420 - nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl); 421 421 - rh &= m62; 422 422 - poly_step(ch, cl, pkh, pkl, rh, rl); 423 423 - mptr += (VMAC_NHBYTES/sizeof(u64)); 424 424 - } 425 425 - 426 426 - dctx->polytmp[0] = ch; 427 427 - dctx->polytmp[1] = cl; 428 428 - } 429 429 - 430 430 - static int vmac_setkey(struct crypto_shash *tfm, 431 431 - const u8 *key, unsigned int keylen) 432 432 - { 433 433 - struct vmac_tfm_ctx *tctx = crypto_shash_ctx(tfm); 434 434 - __be64 out[2]; 435 435 - u8 in[16] = { 0 }; 436 436 - unsigned int i; 437 437 - int err; 438 438 - 439 439 - if (keylen != VMAC_KEY_LEN) 440 440 - return -EINVAL; 441 441 - 442 442 - err = crypto_cipher_setkey(tctx->cipher, key, keylen); 443 443 - if (err) 444 444 - return err; 445 445 - 446 446 - /* Fill nh key */ 447 447 - in[0] = 0x80; 448 448 - for (i = 0; i < ARRAY_SIZE(tctx->nhkey); i += 2) { 449 449 - crypto_cipher_encrypt_one(tctx->cipher, (u8 *)out, in); 450 450 - tctx->nhkey[i] = be64_to_cpu(out[0]); 451 451 - tctx->nhkey[i+1] = be64_to_cpu(out[1]); 452 452 - in[15]++; 453 453 - } 454 454 - 455 455 - /* Fill poly key */ 456 456 - in[0] = 0xC0; 457 457 - in[15] = 0; 458 458 - for (i = 0; i < ARRAY_SIZE(tctx->polykey); i += 2) { 459 459 - crypto_cipher_encrypt_one(tctx->cipher, (u8 *)out, in); 460 460 - tctx->polykey[i] = be64_to_cpu(out[0]) & mpoly; 461 461 - tctx->polykey[i+1] = be64_to_cpu(out[1]) & mpoly; 462 462 - in[15]++; 463 463 - } 464 464 - 465 465 - /* Fill ip key */ 466 466 - in[0] = 0xE0; 467 467 - in[15] = 0; 468 468 - for (i = 0; i < ARRAY_SIZE(tctx->l3key); i += 2) { 469 469 - do { 470 470 - crypto_cipher_encrypt_one(tctx->cipher, (u8 *)out, in); 471 471 - tctx->l3key[i] = be64_to_cpu(out[0]); 472 472 - tctx->l3key[i+1] = be64_to_cpu(out[1]); 473 473 - in[15]++; 474 474 - } while (tctx->l3key[i] >= p64 || tctx->l3key[i+1] >= p64); 475 475 - } 476 476 - 477 477 - return 0; 478 478 - } 479 479 - 480 480 - static int vmac_init(struct shash_desc *desc) 481 481 - { 482 482 - const struct vmac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 483 483 - struct vmac_desc_ctx *dctx = shash_desc_ctx(desc); 484 484 - 485 485 - dctx->partial_size = 0; 486 486 - dctx->first_block_processed = false; 487 487 - memcpy(dctx->polytmp, tctx->polykey, sizeof(dctx->polytmp)); 488 488 - dctx->nonce_size = 0; 489 489 - return 0; 490 490 - } 491 491 - 492 492 - static int vmac_update(struct shash_desc *desc, const u8 *p, unsigned int len) 493 493 - { 494 494 - const struct vmac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 495 495 - struct vmac_desc_ctx *dctx = shash_desc_ctx(desc); 496 496 - unsigned int n; 497 497 - 498 498 - /* Nonce is passed as first VMAC_NONCEBYTES bytes of data */ 499 499 - if (dctx->nonce_size < VMAC_NONCEBYTES) { 500 500 - n = min(len, VMAC_NONCEBYTES - dctx->nonce_size); 501 501 - memcpy(&dctx->nonce.bytes[dctx->nonce_size], p, n); 502 502 - dctx->nonce_size += n; 503 503 - p += n; 504 504 - len -= n; 505 505 - } 506 506 - 507 507 - if (dctx->partial_size) { 508 508 - n = min(len, VMAC_NHBYTES - dctx->partial_size); 509 509 - memcpy(&dctx->partial[dctx->partial_size], p, n); 510 510 - dctx->partial_size += n; 511 511 - p += n; 512 512 - len -= n; 513 513 - if (dctx->partial_size == VMAC_NHBYTES) { 514 514 - vhash_blocks(tctx, dctx, dctx->partial_words, 1); 515 515 - dctx->partial_size = 0; 516 516 - } 517 517 - } 518 518 - 519 519 - if (len >= VMAC_NHBYTES) { 520 520 - n = round_down(len, VMAC_NHBYTES); 521 521 - /* TODO: 'p' may be misaligned here */ 522 522 - vhash_blocks(tctx, dctx, (const __le64 *)p, n / VMAC_NHBYTES); 523 523 - p += n; 524 524 - len -= n; 525 525 - } 526 526 - 527 527 - if (len) { 528 528 - memcpy(dctx->partial, p, len); 529 529 - dctx->partial_size = len; 530 530 - } 531 531 - 532 532 - return 0; 533 533 - } 534 534 - 535 535 - static u64 vhash_final(const struct vmac_tfm_ctx *tctx, 536 536 - struct vmac_desc_ctx *dctx) 537 537 - { 538 538 - unsigned int partial = dctx->partial_size; 539 539 - u64 ch = dctx->polytmp[0]; 540 540 - u64 cl = dctx->polytmp[1]; 541 541 - 542 542 - /* L1 and L2-hash the final block if needed */ 543 543 - if (partial) { 544 544 - /* Zero-pad to next 128-bit boundary */ 545 545 - unsigned int n = round_up(partial, 16); 546 546 - u64 rh, rl; 547 547 - 548 548 - memset(&dctx->partial[partial], 0, n - partial); 549 549 - nh_16(dctx->partial_words, tctx->nhkey, n / 8, rh, rl); 550 550 - rh &= m62; 551 551 - if (dctx->first_block_processed) 552 552 - poly_step(ch, cl, tctx->polykey[0], tctx->polykey[1], 553 553 - rh, rl); 554 554 - else 555 555 - ADD128(ch, cl, rh, rl); 556 556 - } 557 557 - 558 558 - /* L3-hash the 128-bit output of L2-hash */ 559 559 - return l3hash(ch, cl, tctx->l3key[0], tctx->l3key[1], partial * 8); 560 560 - } 561 561 - 562 562 - static int vmac_final(struct shash_desc *desc, u8 *out) 563 563 - { 564 564 - const struct vmac_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); 565 565 - struct vmac_desc_ctx *dctx = shash_desc_ctx(desc); 566 566 - int index; 567 567 - u64 hash, pad; 568 568 - 569 569 - if (dctx->nonce_size != VMAC_NONCEBYTES) 570 570 - return -EINVAL; 571 571 - 572 572 - /* 573 573 - * The VMAC specification requires a nonce at least 1 bit shorter than 574 574 - * the block cipher's block length, so we actually only accept a 127-bit 575 575 - * nonce. We define the unused bit to be the first one and require that 576 576 - * it be 0, so the needed prepending of a 0 bit is implicit. 577 577 - */ 578 578 - if (dctx->nonce.bytes[0] & 0x80) 579 579 - return -EINVAL; 580 580 - 581 581 - /* Finish calculating the VHASH of the message */ 582 582 - hash = vhash_final(tctx, dctx); 583 583 - 584 584 - /* Generate pseudorandom pad by encrypting the nonce */ 585 585 - BUILD_BUG_ON(VMAC_NONCEBYTES != 2 * (VMAC_TAG_LEN / 8)); 586 586 - index = dctx->nonce.bytes[VMAC_NONCEBYTES - 1] & 1; 587 587 - dctx->nonce.bytes[VMAC_NONCEBYTES - 1] &= ~1; 588 588 - crypto_cipher_encrypt_one(tctx->cipher, dctx->nonce.bytes, 589 589 - dctx->nonce.bytes); 590 590 - pad = be64_to_cpu(dctx->nonce.pads[index]); 591 591 - 592 592 - /* The VMAC is the sum of VHASH and the pseudorandom pad */ 593 593 - put_unaligned_be64(hash + pad, out); 594 594 - return 0; 595 595 - } 596 596 - 597 597 - static int vmac_init_tfm(struct crypto_tfm *tfm) 598 598 - { 599 599 - struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); 600 600 - struct crypto_cipher_spawn *spawn = crypto_instance_ctx(inst); 601 601 - struct vmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm); 602 602 - struct crypto_cipher *cipher; 603 603 - 604 604 - cipher = crypto_spawn_cipher(spawn); 605 605 - if (IS_ERR(cipher)) 606 606 - return PTR_ERR(cipher); 607 607 - 608 608 - tctx->cipher = cipher; 609 609 - return 0; 610 610 - } 611 611 - 612 612 - static void vmac_exit_tfm(struct crypto_tfm *tfm) 613 613 - { 614 614 - struct vmac_tfm_ctx *tctx = crypto_tfm_ctx(tfm); 615 615 - 616 616 - crypto_free_cipher(tctx->cipher); 617 617 - } 618 618 - 619 619 - static int vmac_create(struct crypto_template *tmpl, struct rtattr **tb) 620 620 - { 621 621 - struct shash_instance *inst; 622 622 - struct crypto_cipher_spawn *spawn; 623 623 - struct crypto_alg *alg; 624 624 - u32 mask; 625 625 - int err; 626 626 - 627 627 - err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH, &mask); 628 628 - if (err) 629 629 - return err; 630 630 - 631 631 - inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); 632 632 - if (!inst) 633 633 - return -ENOMEM; 634 634 - spawn = shash_instance_ctx(inst); 635 635 - 636 636 - err = crypto_grab_cipher(spawn, shash_crypto_instance(inst), 637 637 - crypto_attr_alg_name(tb[1]), 0, mask); 638 638 - if (err) 639 639 - goto err_free_inst; 640 640 - alg = crypto_spawn_cipher_alg(spawn); 641 641 - 642 642 - err = -EINVAL; 643 643 - if (alg->cra_blocksize != VMAC_NONCEBYTES) 644 644 - goto err_free_inst; 645 645 - 646 646 - err = crypto_inst_setname(shash_crypto_instance(inst), tmpl->name, alg); 647 647 - if (err) 648 648 - goto err_free_inst; 649 649 - 650 650 - inst->alg.base.cra_priority = alg->cra_priority; 651 651 - inst->alg.base.cra_blocksize = alg->cra_blocksize; 652 652 - 653 653 - inst->alg.base.cra_ctxsize = sizeof(struct vmac_tfm_ctx); 654 654 - inst->alg.base.cra_init = vmac_init_tfm; 655 655 - inst->alg.base.cra_exit = vmac_exit_tfm; 656 656 - 657 657 - inst->alg.descsize = sizeof(struct vmac_desc_ctx); 658 658 - inst->alg.digestsize = VMAC_TAG_LEN / 8; 659 659 - inst->alg.init = vmac_init; 660 660 - inst->alg.update = vmac_update; 661 661 - inst->alg.final = vmac_final; 662 662 - inst->alg.setkey = vmac_setkey; 663 663 - 664 664 - inst->free = shash_free_singlespawn_instance; 665 665 - 666 666 - err = shash_register_instance(tmpl, inst); 667 667 - if (err) { 668 668 - err_free_inst: 669 669 - shash_free_singlespawn_instance(inst); 670 670 - } 671 671 - return err; 672 672 - } 673 673 - 674 674 - static struct crypto_template vmac64_tmpl = { 675 675 - .name = "vmac64", 676 676 - .create = vmac_create, 677 677 - .module = THIS_MODULE, 678 678 - }; 679 679 - 680 680 - static int __init vmac_module_init(void) 681 681 - { 682 682 - return crypto_register_template(&vmac64_tmpl); 683 683 - } 684 684 - 685 685 - static void __exit vmac_module_exit(void) 686 686 - { 687 687 - crypto_unregister_template(&vmac64_tmpl); 688 688 - } 689 689 - 690 690 - subsys_initcall(vmac_module_init); 691 691 - module_exit(vmac_module_exit); 692 692 - 693 693 - MODULE_LICENSE("GPL"); 694 694 - MODULE_DESCRIPTION("VMAC hash algorithm"); 695 695 - MODULE_ALIAS_CRYPTO("vmac64"); 696 696 - MODULE_IMPORT_NS("CRYPTO_INTERNAL");

-17

drivers/crypto/Kconfig

reviewed

··· 200 200 201 201 It is available as of z9. 202 202 203 203 - config CRYPTO_DEV_NIAGARA2 204 204 - tristate "Niagara2 Stream Processing Unit driver" 205 205 - select CRYPTO_LIB_DES 206 206 - select CRYPTO_SKCIPHER 207 207 - select CRYPTO_HASH 208 208 - select CRYPTO_MD5 209 209 - select CRYPTO_SHA1 210 210 - select CRYPTO_SHA256 211 211 - depends on SPARC64 212 212 - help 213 213 - Each core of a Niagara2 processor contains a Stream 214 214 - Processing Unit, which itself contains several cryptographic 215 215 - sub-units. One set provides the Modular Arithmetic Unit, 216 216 - used for SSL offload. The other set provides the Cipher 217 217 - Group, which can perform encryption, decryption, hashing, 218 218 - checksumming, and raw copies. 219 219 - 220 203 config CRYPTO_DEV_SL3516 221 204 tristate "Storlink SL3516 crypto offloader" 222 205 depends on ARCH_GEMINI || COMPILE_TEST

-2

drivers/crypto/Makefile

reviewed

··· 21 21 obj-$(CONFIG_CRYPTO_DEV_IMGTEC_HASH) += img-hash.o 22 22 obj-$(CONFIG_CRYPTO_DEV_MARVELL) += marvell/ 23 23 obj-$(CONFIG_CRYPTO_DEV_MXS_DCP) += mxs-dcp.o 24 24 - obj-$(CONFIG_CRYPTO_DEV_NIAGARA2) += n2_crypto.o 25 25 - n2_crypto-y := n2_core.o n2_asm.o 26 24 obj-$(CONFIG_CRYPTO_DEV_NX) += nx/ 27 25 obj-$(CONFIG_CRYPTO_DEV_OMAP) += omap-crypto.o 28 26 obj-$(CONFIG_CRYPTO_DEV_OMAP_AES) += omap-aes-driver.o

+1 -6

drivers/crypto/bcm/spu.c

reviewed

··· 836 836 u32 cipher_bits = 0; 837 837 u32 ecf_bits = 0; 838 838 u8 sctx_words = 0; 839 839 - u8 *ptr = spu_hdr; 840 839 841 840 flow_log("%s()\n", __func__); 842 841 flow_log(" cipher alg:%u mode:%u type %u\n", cipher_parms->alg, ··· 846 847 847 848 /* starting out: zero the header (plus some) */ 848 849 memset(spu_hdr, 0, sizeof(struct SPUHEADER)); 849 849 - ptr += sizeof(struct SPUHEADER); 850 850 851 851 /* format master header word */ 852 852 /* Do not set the next bit even though the datasheet says to */ ··· 859 861 860 862 /* copy the encryption keys in the SAD entry */ 861 863 if (cipher_parms->alg) { 862 862 - if (cipher_parms->key_len) { 863 863 - ptr += cipher_parms->key_len; 864 864 + if (cipher_parms->key_len) 864 865 sctx_words += cipher_parms->key_len / 4; 865 865 - } 866 866 867 867 /* 868 868 * if encrypting then set IV size, use SCTX IV unless no IV ··· 869 873 if (cipher_parms->iv_len) { 870 874 /* Use SCTX IV */ 871 875 ecf_bits |= SCTX_IV; 872 872 - ptr += cipher_parms->iv_len; 873 876 sctx_words += cipher_parms->iv_len / 4; 874 877 } 875 878 }

+2 -1

drivers/crypto/caam/blob_gen.c

reviewed

··· 2 2 /* 3 3 * Copyright (C) 2015 Pengutronix, Steffen Trumtrar <kernel@pengutronix.de> 4 4 * Copyright (C) 2021 Pengutronix, Ahmad Fatoum <kernel@pengutronix.de> 5 5 + * Copyright 2024 NXP 5 6 */ 6 7 7 8 #define pr_fmt(fmt) "caam blob_gen: " fmt ··· 105 104 } 106 105 107 106 ctrlpriv = dev_get_drvdata(jrdev->parent); 108 108 - moo = FIELD_GET(CSTA_MOO, rd_reg32(&ctrlpriv->ctrl->perfmon.status)); 107 107 + moo = FIELD_GET(CSTA_MOO, rd_reg32(&ctrlpriv->jr[0]->perfmon.status)); 109 108 if (moo != CSTA_MOO_SECURE && moo != CSTA_MOO_TRUSTED) 110 109 dev_warn(jrdev, 111 110 "using insecure test key, enable HAB to use unique device key!\n");

+20 -33

drivers/crypto/ccp/dbc.c

reviewed

··· 7 7 * Author: Mario Limonciello <mario.limonciello@amd.com> 8 8 */ 9 9 10 10 + #include <linux/mutex.h> 11 11 + 10 12 #include "dbc.h" 11 13 12 14 #define DBC_DEFAULT_TIMEOUT (10 * MSEC_PER_SEC) ··· 139 137 return -ENODEV; 140 138 dbc_dev = psp_master->dbc_data; 141 139 142 142 - mutex_lock(&dbc_dev->ioctl_mutex); 140 140 + guard(mutex)(&dbc_dev->ioctl_mutex); 143 141 144 142 switch (cmd) { 145 143 case DBCIOCNONCE: 146 146 - if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_nonce))) { 147 147 - ret = -EFAULT; 148 148 - goto unlock; 149 149 - } 144 144 + if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_nonce))) 145 145 + return -EFAULT; 150 146 151 147 ret = send_dbc_nonce(dbc_dev); 152 148 if (ret) 153 153 - goto unlock; 149 149 + return ret; 154 150 155 155 - if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_nonce))) { 156 156 - ret = -EFAULT; 157 157 - goto unlock; 158 158 - } 151 151 + if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_nonce))) 152 152 + return -EFAULT; 159 153 break; 160 154 case DBCIOCUID: 161 161 - if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_setuid))) { 162 162 - ret = -EFAULT; 163 163 - goto unlock; 164 164 - } 155 155 + if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_setuid))) 156 156 + return -EFAULT; 165 157 166 158 *dbc_dev->payload_size = dbc_dev->header_size + sizeof(struct dbc_user_setuid); 167 159 ret = send_dbc_cmd(dbc_dev, PSP_DYNAMIC_BOOST_SET_UID); 168 160 if (ret) 169 169 - goto unlock; 161 161 + return ret; 170 162 171 171 - if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_setuid))) { 172 172 - ret = -EFAULT; 173 173 - goto unlock; 174 174 - } 163 163 + if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_setuid))) 164 164 + return -EFAULT; 175 165 break; 176 166 case DBCIOCPARAM: 177 177 - if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_param))) { 178 178 - ret = -EFAULT; 179 179 - goto unlock; 180 180 - } 167 167 + if (copy_from_user(dbc_dev->payload, argp, sizeof(struct dbc_user_param))) 168 168 + return -EFAULT; 181 169 182 170 *dbc_dev->payload_size = dbc_dev->header_size + sizeof(struct dbc_user_param); 183 171 ret = send_dbc_parameter(dbc_dev); 184 172 if (ret) 185 185 - goto unlock; 173 173 + return ret; 186 174 187 187 - if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_param))) { 188 188 - ret = -EFAULT; 189 189 - goto unlock; 190 190 - } 175 175 + if (copy_to_user(argp, dbc_dev->payload, sizeof(struct dbc_user_param))) 176 176 + return -EFAULT; 191 177 break; 192 178 default: 193 193 - ret = -EINVAL; 194 194 - 179 179 + return -EINVAL; 195 180 } 196 196 - unlock: 197 197 - mutex_unlock(&dbc_dev->ioctl_mutex); 198 181 199 199 - return ret; 182 182 + return 0; 200 183 } 201 184 202 185 static const struct file_operations dbc_fops = {

+12 -1

drivers/crypto/hisilicon/hpre/hpre_main.c

reviewed

··· 1209 1209 1210 1210 qm->mode = uacce_mode; 1211 1211 qm->pdev = pdev; 1212 1212 - qm->ver = pdev->revision; 1213 1212 qm->sqe_size = HPRE_SQE_SIZE; 1214 1213 qm->dev_name = hpre_name; 1215 1214 ··· 1395 1396 return ACC_ERR_RECOVERED; 1396 1397 } 1397 1398 1399 1399 + static bool hpre_dev_is_abnormal(struct hisi_qm *qm) 1400 1400 + { 1401 1401 + u32 err_status; 1402 1402 + 1403 1403 + err_status = hpre_get_hw_err_status(qm); 1404 1404 + if (err_status & qm->err_info.dev_shutdown_mask) 1405 1405 + return true; 1406 1406 + 1407 1407 + return false; 1408 1408 + } 1409 1409 + 1398 1410 static void hpre_err_info_init(struct hisi_qm *qm) 1399 1411 { 1400 1412 struct hisi_qm_err_info *err_info = &qm->err_info; ··· 1438 1428 .show_last_dfx_regs = hpre_show_last_dfx_regs, 1439 1429 .err_info_init = hpre_err_info_init, 1440 1430 .get_err_result = hpre_get_err_result, 1431 1431 + .dev_is_abnormal = hpre_dev_is_abnormal, 1441 1432 }; 1442 1433 1443 1434 static int hpre_pf_probe_init(struct hpre *hpre)

+215 -76

drivers/crypto/hisilicon/qm.c

reviewed

··· 30 30 31 31 /* mailbox */ 32 32 #define QM_MB_PING_ALL_VFS 0xffff 33 33 - #define QM_MB_CMD_DATA_SHIFT 32 34 34 - #define QM_MB_CMD_DATA_MASK GENMASK(31, 0) 35 33 #define QM_MB_STATUS_MASK GENMASK(12, 9) 36 34 37 35 /* sqc shift */ ··· 100 102 #define QM_PM_CTRL 0x100148 101 103 #define QM_IDLE_DISABLE BIT(9) 102 104 105 105 + #define QM_SUB_VERSION_ID 0x210 106 106 + 103 107 #define QM_VFT_CFG_DATA_L 0x100064 104 108 #define QM_VFT_CFG_DATA_H 0x100068 105 109 #define QM_SQC_VFT_BUF_SIZE (7ULL << 8) ··· 119 119 #define QM_SQC_VFT_BASE_MASK_V2 GENMASK(15, 0) 120 120 #define QM_SQC_VFT_NUM_SHIFT_V2 45 121 121 #define QM_SQC_VFT_NUM_MASK_V2 GENMASK(9, 0) 122 122 + #define QM_MAX_QC_TYPE 2 122 123 123 124 #define QM_ABNORMAL_INT_SOURCE 0x100000 124 125 #define QM_ABNORMAL_INT_MASK 0x100004 ··· 177 176 #define QM_IFC_INT_MASK 0x0024 178 177 #define QM_IFC_INT_STATUS 0x0028 179 178 #define QM_IFC_INT_SET_V 0x002C 179 179 + #define QM_PF2VF_PF_W 0x104700 180 180 + #define QM_VF2PF_PF_R 0x104800 181 181 + #define QM_VF2PF_VF_W 0x320 182 182 + #define QM_PF2VF_VF_R 0x380 180 183 #define QM_IFC_SEND_ALL_VFS GENMASK(6, 0) 181 184 #define QM_IFC_INT_SOURCE_CLR GENMASK(63, 0) 182 185 #define QM_IFC_INT_SOURCE_MASK BIT(0) ··· 190 185 #define QM_WAIT_DST_ACK 10 191 186 #define QM_MAX_PF_WAIT_COUNT 10 192 187 #define QM_MAX_VF_WAIT_COUNT 40 193 193 - #define QM_VF_RESET_WAIT_US 20000 194 194 - #define QM_VF_RESET_WAIT_CNT 3000 188 188 + #define QM_VF_RESET_WAIT_US 20000 189 189 + #define QM_VF_RESET_WAIT_CNT 3000 190 190 + #define QM_VF2PF_REG_SIZE 4 191 191 + #define QM_IFC_CMD_MASK GENMASK(31, 0) 192 192 + #define QM_IFC_DATA_SHIFT 32 195 193 #define QM_VF_RESET_WAIT_TIMEOUT_US \ 196 194 (QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT) 197 195 ··· 242 234 #define QM_QOS_MAX_CIR_U 6 243 235 #define QM_AUTOSUSPEND_DELAY 3000 244 236 245 245 - #define QM_DEV_ALG_MAX_LEN 256 246 246 - 247 237 /* abnormal status value for stopping queue */ 248 238 #define QM_STOP_QUEUE_FAIL 1 249 239 #define QM_DUMP_SQC_FAIL 3 ··· 282 276 ALG_TYPE_1, 283 277 }; 284 278 285 285 - enum qm_mb_cmd { 279 279 + enum qm_ifc_cmd { 286 280 QM_PF_FLR_PREPARE = 0x01, 287 281 QM_PF_SRST_PREPARE, 288 282 QM_PF_RESET_DONE, ··· 339 333 {QM_SUPPORT_STOP_FUNC, 0x3100, 0, BIT(10), 0x0, 0x0, 0x1}, 340 334 {QM_SUPPORT_MB_COMMAND, 0x3100, 0, BIT(11), 0x0, 0x0, 0x1}, 341 335 {QM_SUPPORT_SVA_PREFETCH, 0x3100, 0, BIT(14), 0x0, 0x0, 0x1}, 336 336 + {QM_SUPPORT_DAE, 0x3100, 0, BIT(15), 0x0, 0x0, 0x0}, 342 337 }; 343 338 344 339 static const struct hisi_qm_cap_info qm_cap_info_pf[] = { ··· 403 396 void (*hw_error_uninit)(struct hisi_qm *qm); 404 397 enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); 405 398 int (*set_msi)(struct hisi_qm *qm, bool set); 399 399 + 400 400 + /* (u64)msg = (u32)data << 32 | (enum qm_ifc_cmd)cmd */ 401 401 + int (*set_ifc_begin)(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num); 402 402 + void (*set_ifc_end)(struct hisi_qm *qm); 403 403 + int (*get_ifc)(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num); 406 404 }; 407 405 408 406 struct hisi_qm_hw_error { ··· 513 501 /* Check if the error causes the master ooo block */ 514 502 static bool qm_check_dev_error(struct hisi_qm *qm) 515 503 { 516 516 - u32 val, dev_val; 504 504 + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); 505 505 + u32 err_status; 517 506 518 518 - if (qm->fun_type == QM_HW_VF) 507 507 + if (pf_qm->fun_type == QM_HW_VF) 519 508 return false; 520 509 521 521 - val = qm_get_hw_error_status(qm) & qm->err_info.qm_shutdown_mask; 522 522 - dev_val = qm_get_dev_err_status(qm) & qm->err_info.dev_shutdown_mask; 510 510 + err_status = qm_get_hw_error_status(pf_qm); 511 511 + if (err_status & pf_qm->err_info.qm_shutdown_mask) 512 512 + return true; 523 513 524 524 - return val || dev_val; 514 514 + if (pf_qm->err_ini->dev_is_abnormal) 515 515 + return pf_qm->err_ini->dev_is_abnormal(pf_qm); 516 516 + 517 517 + return false; 525 518 } 526 519 527 520 static int qm_wait_reset_finish(struct hisi_qm *qm) ··· 671 654 /* op 0: set xqc information to hardware, 1: get xqc information from hardware. */ 672 655 int qm_set_and_get_xqc(struct hisi_qm *qm, u8 cmd, void *xqc, u32 qp_id, bool op) 673 656 { 674 674 - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); 675 657 struct qm_mailbox mailbox; 676 658 dma_addr_t xqc_dma; 677 659 void *tmp_xqc; ··· 704 688 } 705 689 706 690 /* Setting xqc will fail if master OOO is blocked. */ 707 707 - if (qm_check_dev_error(pf_qm)) { 691 691 + if (qm_check_dev_error(qm)) { 708 692 dev_err(&qm->pdev->dev, "failed to send mailbox since qm is stop!\n"); 709 693 return -EIO; 710 694 } ··· 871 855 strcat(algs, dev_algs[i].alg); 872 856 873 857 ptr = strrchr(algs, '\n'); 874 874 - if (ptr) { 858 858 + if (ptr) 875 859 *ptr = '\0'; 876 876 - qm->uacce->algs = algs; 877 877 - } 860 860 + 861 861 + qm->uacce->algs = algs; 878 862 879 863 return 0; 880 864 } ··· 1068 1052 1069 1053 static void qm_reset_function(struct hisi_qm *qm) 1070 1054 { 1071 1071 - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); 1072 1055 struct device *dev = &qm->pdev->dev; 1073 1056 int ret; 1074 1057 1075 1075 - if (qm_check_dev_error(pf_qm)) 1058 1058 + if (qm_check_dev_error(qm)) 1076 1059 return; 1077 1060 1078 1061 ret = qm_reset_prepare_ready(qm); ··· 1555 1540 static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) 1556 1541 { 1557 1542 struct device *dev = &qm->pdev->dev; 1558 1558 - u32 cmd; 1559 1559 - u64 msg; 1543 1543 + enum qm_ifc_cmd cmd; 1560 1544 int ret; 1561 1545 1562 1562 - ret = qm_get_mb_cmd(qm, &msg, vf_id); 1546 1546 + ret = qm->ops->get_ifc(qm, &cmd, NULL, vf_id); 1563 1547 if (ret) { 1564 1564 - dev_err(dev, "failed to get msg from VF(%u)!\n", vf_id); 1548 1548 + dev_err(dev, "failed to get command from VF(%u)!\n", vf_id); 1565 1549 return; 1566 1550 } 1567 1551 1568 1568 - cmd = msg & QM_MB_CMD_DATA_MASK; 1569 1552 switch (cmd) { 1570 1553 case QM_VF_PREPARE_FAIL: 1571 1554 dev_err(dev, "failed to stop VF(%u)!\n", vf_id); ··· 1575 1562 case QM_VF_START_DONE: 1576 1563 break; 1577 1564 default: 1578 1578 - dev_err(dev, "unsupported cmd %u sent by VF(%u)!\n", cmd, vf_id); 1565 1565 + dev_err(dev, "unsupported command(0x%x) sent by VF(%u)!\n", cmd, vf_id); 1579 1566 break; 1580 1567 } 1581 1568 } ··· 1643 1630 writel(val, qm->io_base + QM_IFC_INT_SET_V); 1644 1631 } 1645 1632 1646 1646 - static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num) 1633 1633 + static int qm_ping_single_vf(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) 1647 1634 { 1648 1635 struct device *dev = &qm->pdev->dev; 1649 1649 - struct qm_mailbox mailbox; 1650 1636 int cnt = 0; 1651 1637 u64 val; 1652 1638 int ret; 1653 1639 1654 1654 - qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, fun_num, 0); 1655 1655 - mutex_lock(&qm->mailbox_lock); 1656 1656 - ret = qm_mb_nolock(qm, &mailbox); 1640 1640 + ret = qm->ops->set_ifc_begin(qm, cmd, data, fun_num); 1657 1641 if (ret) { 1658 1642 dev_err(dev, "failed to send command to vf(%u)!\n", fun_num); 1659 1643 goto err_unlock; ··· 1672 1662 } 1673 1663 1674 1664 err_unlock: 1675 1675 - mutex_unlock(&qm->mailbox_lock); 1665 1665 + qm->ops->set_ifc_end(qm); 1676 1666 return ret; 1677 1667 } 1678 1668 1679 1679 - static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) 1669 1669 + static int qm_ping_all_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) 1680 1670 { 1681 1671 struct device *dev = &qm->pdev->dev; 1682 1672 u32 vfs_num = qm->vfs_num; 1683 1683 - struct qm_mailbox mailbox; 1684 1673 u64 val = 0; 1685 1674 int cnt = 0; 1686 1675 int ret; 1687 1676 u32 i; 1688 1677 1689 1689 - qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, QM_MB_PING_ALL_VFS, 0); 1690 1690 - mutex_lock(&qm->mailbox_lock); 1691 1691 - /* PF sends command to all VFs by mailbox */ 1692 1692 - ret = qm_mb_nolock(qm, &mailbox); 1678 1678 + ret = qm->ops->set_ifc_begin(qm, cmd, 0, QM_MB_PING_ALL_VFS); 1693 1679 if (ret) { 1694 1694 - dev_err(dev, "failed to send command to VFs!\n"); 1695 1695 - mutex_unlock(&qm->mailbox_lock); 1680 1680 + dev_err(dev, "failed to send command(0x%x) to all vfs!\n", cmd); 1681 1681 + qm->ops->set_ifc_end(qm); 1696 1682 return ret; 1697 1683 } 1698 1684 ··· 1698 1692 val = readq(qm->io_base + QM_IFC_READY_STATUS); 1699 1693 /* If all VFs acked, PF notifies VFs successfully. */ 1700 1694 if (!(val & GENMASK(vfs_num, 1))) { 1701 1701 - mutex_unlock(&qm->mailbox_lock); 1695 1695 + qm->ops->set_ifc_end(qm); 1702 1696 return 0; 1703 1697 } 1704 1698 ··· 1706 1700 break; 1707 1701 } 1708 1702 1709 1709 - mutex_unlock(&qm->mailbox_lock); 1703 1703 + qm->ops->set_ifc_end(qm); 1710 1704 1711 1705 /* Check which vf respond timeout. */ 1712 1706 for (i = 1; i <= vfs_num; i++) { ··· 1717 1711 return -ETIMEDOUT; 1718 1712 } 1719 1713 1720 1720 - static int qm_ping_pf(struct hisi_qm *qm, u64 cmd) 1714 1714 + static int qm_ping_pf(struct hisi_qm *qm, enum qm_ifc_cmd cmd) 1721 1715 { 1722 1722 - struct qm_mailbox mailbox; 1723 1716 int cnt = 0; 1724 1717 u32 val; 1725 1718 int ret; 1726 1719 1727 1727 - qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, 0, 0); 1728 1728 - mutex_lock(&qm->mailbox_lock); 1729 1729 - ret = qm_mb_nolock(qm, &mailbox); 1720 1720 + ret = qm->ops->set_ifc_begin(qm, cmd, 0, 0); 1730 1721 if (ret) { 1731 1731 - dev_err(&qm->pdev->dev, "failed to send command to PF!\n"); 1722 1722 + dev_err(&qm->pdev->dev, "failed to send command(0x%x) to PF!\n", cmd); 1732 1723 goto unlock; 1733 1724 } 1734 1725 ··· 1744 1741 } 1745 1742 1746 1743 unlock: 1747 1747 - mutex_unlock(&qm->mailbox_lock); 1744 1744 + qm->ops->set_ifc_end(qm); 1745 1745 + 1748 1746 return ret; 1749 1747 } 1750 1748 ··· 1846 1842 return ret; 1847 1843 } 1848 1844 1845 1845 + static int qm_set_ifc_begin_v3(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) 1846 1846 + { 1847 1847 + struct qm_mailbox mailbox; 1848 1848 + u64 msg; 1849 1849 + 1850 1850 + msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; 1851 1851 + 1852 1852 + qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, msg, fun_num, 0); 1853 1853 + mutex_lock(&qm->mailbox_lock); 1854 1854 + return qm_mb_nolock(qm, &mailbox); 1855 1855 + } 1856 1856 + 1857 1857 + static void qm_set_ifc_end_v3(struct hisi_qm *qm) 1858 1858 + { 1859 1859 + mutex_unlock(&qm->mailbox_lock); 1860 1860 + } 1861 1861 + 1862 1862 + static int qm_get_ifc_v3(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) 1863 1863 + { 1864 1864 + u64 msg; 1865 1865 + int ret; 1866 1866 + 1867 1867 + ret = qm_get_mb_cmd(qm, &msg, fun_num); 1868 1868 + if (ret) 1869 1869 + return ret; 1870 1870 + 1871 1871 + *cmd = msg & QM_IFC_CMD_MASK; 1872 1872 + 1873 1873 + if (data) 1874 1874 + *data = msg >> QM_IFC_DATA_SHIFT; 1875 1875 + 1876 1876 + return 0; 1877 1877 + } 1878 1878 + 1879 1879 + static int qm_set_ifc_begin_v4(struct hisi_qm *qm, enum qm_ifc_cmd cmd, u32 data, u32 fun_num) 1880 1880 + { 1881 1881 + uintptr_t offset; 1882 1882 + u64 msg; 1883 1883 + 1884 1884 + if (qm->fun_type == QM_HW_PF) 1885 1885 + offset = QM_PF2VF_PF_W; 1886 1886 + else 1887 1887 + offset = QM_VF2PF_VF_W; 1888 1888 + 1889 1889 + msg = cmd | (u64)data << QM_IFC_DATA_SHIFT; 1890 1890 + 1891 1891 + mutex_lock(&qm->ifc_lock); 1892 1892 + writeq(msg, qm->io_base + offset); 1893 1893 + 1894 1894 + return 0; 1895 1895 + } 1896 1896 + 1897 1897 + static void qm_set_ifc_end_v4(struct hisi_qm *qm) 1898 1898 + { 1899 1899 + mutex_unlock(&qm->ifc_lock); 1900 1900 + } 1901 1901 + 1902 1902 + static u64 qm_get_ifc_pf(struct hisi_qm *qm, u32 fun_num) 1903 1903 + { 1904 1904 + uintptr_t offset; 1905 1905 + 1906 1906 + offset = QM_VF2PF_PF_R + QM_VF2PF_REG_SIZE * fun_num; 1907 1907 + 1908 1908 + return (u64)readl(qm->io_base + offset); 1909 1909 + } 1910 1910 + 1911 1911 + static u64 qm_get_ifc_vf(struct hisi_qm *qm) 1912 1912 + { 1913 1913 + return readq(qm->io_base + QM_PF2VF_VF_R); 1914 1914 + } 1915 1915 + 1916 1916 + static int qm_get_ifc_v4(struct hisi_qm *qm, enum qm_ifc_cmd *cmd, u32 *data, u32 fun_num) 1917 1917 + { 1918 1918 + u64 msg; 1919 1919 + 1920 1920 + if (qm->fun_type == QM_HW_PF) 1921 1921 + msg = qm_get_ifc_pf(qm, fun_num); 1922 1922 + else 1923 1923 + msg = qm_get_ifc_vf(qm); 1924 1924 + 1925 1925 + *cmd = msg & QM_IFC_CMD_MASK; 1926 1926 + 1927 1927 + if (data) 1928 1928 + *data = msg >> QM_IFC_DATA_SHIFT; 1929 1929 + 1930 1930 + return 0; 1931 1931 + } 1932 1932 + 1849 1933 static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { 1850 1934 .qm_db = qm_db_v1, 1851 1935 .hw_error_init = qm_hw_error_init_v1, ··· 1956 1864 .hw_error_uninit = qm_hw_error_uninit_v3, 1957 1865 .hw_error_handle = qm_hw_error_handle_v2, 1958 1866 .set_msi = qm_set_msi_v3, 1867 1867 + .set_ifc_begin = qm_set_ifc_begin_v3, 1868 1868 + .set_ifc_end = qm_set_ifc_end_v3, 1869 1869 + .get_ifc = qm_get_ifc_v3, 1870 1870 + }; 1871 1871 + 1872 1872 + static const struct hisi_qm_hw_ops qm_hw_ops_v4 = { 1873 1873 + .get_vft = qm_get_vft_v2, 1874 1874 + .qm_db = qm_db_v2, 1875 1875 + .hw_error_init = qm_hw_error_init_v3, 1876 1876 + .hw_error_uninit = qm_hw_error_uninit_v3, 1877 1877 + .hw_error_handle = qm_hw_error_handle_v2, 1878 1878 + .set_msi = qm_set_msi_v3, 1879 1879 + .set_ifc_begin = qm_set_ifc_begin_v4, 1880 1880 + .set_ifc_end = qm_set_ifc_end_v4, 1881 1881 + .get_ifc = qm_get_ifc_v4, 1959 1882 }; 1960 1883 1961 1884 static void *qm_get_avail_sqe(struct hisi_qp *qp) ··· 2263 2156 static int qm_drain_qp(struct hisi_qp *qp) 2264 2157 { 2265 2158 struct hisi_qm *qm = qp->qm; 2266 2266 - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); 2267 2159 u32 state = 0; 2268 2160 int ret; 2269 2161 2270 2162 /* No need to judge if master OOO is blocked. */ 2271 2271 - if (qm_check_dev_error(pf_qm)) 2163 2163 + if (qm_check_dev_error(qm)) 2272 2164 return 0; 2273 2165 2274 2166 /* HW V3 supports drain qp by device */ ··· 2581 2475 sizeof(struct hisi_qp_ctx))) 2582 2476 return -EFAULT; 2583 2477 2584 2584 - if (qp_ctx.qc_type != 0 && qp_ctx.qc_type != 1) 2478 2478 + if (qp_ctx.qc_type > QM_MAX_QC_TYPE) 2585 2479 return -EINVAL; 2586 2480 2587 2481 qm_set_sqctype(q, qp_ctx.qc_type); ··· 2949 2843 qm->ops = &qm_hw_ops_v1; 2950 2844 else if (qm->ver == QM_HW_V2) 2951 2845 qm->ops = &qm_hw_ops_v2; 2952 2952 - else 2846 2846 + else if (qm->ver == QM_HW_V3) 2953 2847 qm->ops = &qm_hw_ops_v3; 2848 2848 + else 2849 2849 + qm->ops = &qm_hw_ops_v4; 2954 2850 2955 2851 pci_set_drvdata(pdev, qm); 2956 2852 mutex_init(&qm->mailbox_lock); 2853 2853 + mutex_init(&qm->ifc_lock); 2957 2854 init_rwsem(&qm->qps_lock); 2958 2855 qm->qp_in_used = 0; 2959 2856 if (test_bit(QM_SUPPORT_RPM, &qm->caps)) { ··· 3716 3607 static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) 3717 3608 { 3718 3609 struct device *dev = &qm->pdev->dev; 3719 3719 - u64 mb_cmd; 3720 3610 u32 qos; 3721 3611 int ret; 3722 3612 ··· 3725 3617 return; 3726 3618 } 3727 3619 3728 3728 - mb_cmd = QM_PF_SET_QOS | (u64)qos << QM_MB_CMD_DATA_SHIFT; 3729 3729 - ret = qm_ping_single_vf(qm, mb_cmd, fun_num); 3620 3620 + ret = qm_ping_single_vf(qm, QM_PF_SET_QOS, qos, fun_num); 3730 3621 if (ret) 3731 3731 - dev_err(dev, "failed to send cmd to VF(%u)!\n", fun_num); 3622 3622 + dev_err(dev, "failed to send command(0x%x) to VF(%u)!\n", QM_PF_SET_QOS, fun_num); 3732 3623 } 3733 3624 3734 3625 static int qm_vf_read_qos(struct hisi_qm *qm) ··· 4216 4109 return ret; 4217 4110 } 4218 4111 4219 4219 - static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, 4112 4112 + static int qm_try_stop_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd, 4220 4113 enum qm_stop_reason stop_reason) 4221 4114 { 4222 4115 struct pci_dev *pdev = qm->pdev; ··· 4229 4122 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { 4230 4123 ret = qm_ping_all_vfs(qm, cmd); 4231 4124 if (ret) 4232 4232 - pci_err(pdev, "failed to send cmd to all VFs before PF reset!\n"); 4125 4125 + pci_err(pdev, "failed to send command to all VFs before PF reset!\n"); 4233 4126 } else { 4234 4127 ret = qm_vf_reset_prepare(qm, stop_reason); 4235 4128 if (ret) ··· 4243 4136 { 4244 4137 struct pci_dev *pdev = qm->pdev; 4245 4138 int ret; 4139 4139 + 4140 4140 + if (qm->err_ini->set_priv_status) { 4141 4141 + ret = qm->err_ini->set_priv_status(qm); 4142 4142 + if (ret) 4143 4143 + return ret; 4144 4144 + } 4246 4145 4247 4146 ret = qm_reset_prepare_ready(qm); 4248 4147 if (ret) { ··· 4411 4298 return ret; 4412 4299 } 4413 4300 4414 4414 - static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd) 4301 4301 + static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_ifc_cmd cmd) 4415 4302 { 4416 4303 struct pci_dev *pdev = qm->pdev; 4417 4304 int ret; ··· 4640 4527 * Check whether there is an ECC mbit error, If it occurs, need to 4641 4528 * wait for soft reset to fix it. 4642 4529 */ 4643 4643 - while (qm_check_dev_error(pf_qm)) { 4530 4530 + while (qm_check_dev_error(qm)) { 4644 4531 msleep(++delay); 4645 4532 if (delay > QM_RESET_WAIT_TIMEOUT) 4646 4533 return; ··· 4788 4675 static void qm_pf_reset_vf_prepare(struct hisi_qm *qm, 4789 4676 enum qm_stop_reason stop_reason) 4790 4677 { 4791 4791 - enum qm_mb_cmd cmd = QM_VF_PREPARE_DONE; 4678 4678 + enum qm_ifc_cmd cmd = QM_VF_PREPARE_DONE; 4792 4679 struct pci_dev *pdev = qm->pdev; 4793 4680 int ret; 4794 4681 ··· 4822 4709 4823 4710 static void qm_pf_reset_vf_done(struct hisi_qm *qm) 4824 4711 { 4825 4825 - enum qm_mb_cmd cmd = QM_VF_START_DONE; 4712 4712 + enum qm_ifc_cmd cmd = QM_VF_START_DONE; 4826 4713 struct pci_dev *pdev = qm->pdev; 4827 4714 int ret; 4828 4715 ··· 4845 4732 { 4846 4733 struct device *dev = &qm->pdev->dev; 4847 4734 u32 val, cmd; 4848 4848 - u64 msg; 4849 4735 int ret; 4850 4736 4851 4737 /* Wait for reset to finish */ ··· 4861 4749 * Whether message is got successfully, 4862 4750 * VF needs to ack PF by clearing the interrupt. 4863 4751 */ 4864 4864 - ret = qm_get_mb_cmd(qm, &msg, 0); 4752 4752 + ret = qm->ops->get_ifc(qm, &cmd, NULL, 0); 4865 4753 qm_clear_cmd_interrupt(qm, 0); 4866 4754 if (ret) { 4867 4867 - dev_err(dev, "failed to get msg from PF in reset done!\n"); 4755 4755 + dev_err(dev, "failed to get command from PF in reset done!\n"); 4868 4756 return ret; 4869 4757 } 4870 4758 4871 4871 - cmd = msg & QM_MB_CMD_DATA_MASK; 4872 4759 if (cmd != QM_PF_RESET_DONE) { 4873 4873 - dev_err(dev, "the cmd(%u) is not reset done!\n", cmd); 4760 4760 + dev_err(dev, "the command(0x%x) is not reset done!\n", cmd); 4874 4761 ret = -EINVAL; 4875 4762 } 4876 4763 ··· 4906 4795 static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) 4907 4796 { 4908 4797 struct device *dev = &qm->pdev->dev; 4909 4909 - u64 msg; 4910 4910 - u32 cmd; 4798 4798 + enum qm_ifc_cmd cmd; 4799 4799 + u32 data; 4911 4800 int ret; 4912 4801 4913 4802 /* 4914 4803 * Get the msg from source by sending mailbox. Whether message is got 4915 4804 * successfully, destination needs to ack source by clearing the interrupt. 4916 4805 */ 4917 4917 - ret = qm_get_mb_cmd(qm, &msg, fun_num); 4806 4806 + ret = qm->ops->get_ifc(qm, &cmd, &data, fun_num); 4918 4807 qm_clear_cmd_interrupt(qm, BIT(fun_num)); 4919 4808 if (ret) { 4920 4920 - dev_err(dev, "failed to get msg from source!\n"); 4809 4809 + dev_err(dev, "failed to get command from source!\n"); 4921 4810 return; 4922 4811 } 4923 4812 4924 4924 - cmd = msg & QM_MB_CMD_DATA_MASK; 4925 4813 switch (cmd) { 4926 4814 case QM_PF_FLR_PREPARE: 4927 4815 qm_pf_reset_vf_process(qm, QM_DOWN); ··· 4932 4822 qm_vf_get_qos(qm, fun_num); 4933 4823 break; 4934 4824 case QM_PF_SET_QOS: 4935 4935 - qm->mb_qos = msg >> QM_MB_CMD_DATA_SHIFT; 4825 4825 + qm->mb_qos = data; 4936 4826 break; 4937 4827 default: 4938 4938 - dev_err(dev, "unsupported cmd %u sent by function(%u)!\n", cmd, fun_num); 4828 4828 + dev_err(dev, "unsupported command(0x%x) sent by function(%u)!\n", cmd, fun_num); 4939 4829 break; 4940 4830 } 4941 4831 } ··· 5277 5167 return qm_pre_store_caps(qm); 5278 5168 } 5279 5169 5170 5170 + static void qm_get_version(struct hisi_qm *qm) 5171 5171 + { 5172 5172 + struct pci_dev *pdev = qm->pdev; 5173 5173 + u32 sub_version_id; 5174 5174 + 5175 5175 + qm->ver = pdev->revision; 5176 5176 + 5177 5177 + if (pdev->revision == QM_HW_V3) { 5178 5178 + sub_version_id = readl(qm->io_base + QM_SUB_VERSION_ID); 5179 5179 + if (sub_version_id) 5180 5180 + qm->ver = sub_version_id; 5181 5181 + } 5182 5182 + } 5183 5183 + 5280 5184 static int qm_get_pci_res(struct hisi_qm *qm) 5281 5185 { 5282 5186 struct pci_dev *pdev = qm->pdev; ··· 5309 5185 ret = -EIO; 5310 5186 goto err_request_mem_regions; 5311 5187 } 5188 5188 + 5189 5189 + qm_get_version(qm); 5312 5190 5313 5191 ret = qm_get_hw_caps(qm); 5314 5192 if (ret) ··· 5331 5205 qm->db_interval = 0; 5332 5206 } 5333 5207 5208 5208 + hisi_qm_pre_init(qm); 5334 5209 ret = qm_get_qp_num(qm); 5335 5210 if (ret) 5336 5211 goto err_db_ioremap; ··· 5372 5245 if (ret) { 5373 5246 writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); 5374 5247 return ret; 5248 5248 + } 5249 5249 + 5250 5250 + if (qm->err_ini->set_priv_status) { 5251 5251 + ret = qm->err_ini->set_priv_status(qm); 5252 5252 + if (ret) { 5253 5253 + writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); 5254 5254 + return ret; 5255 5255 + } 5375 5256 } 5376 5257 5377 5258 return qm_reset_device(qm); ··· 5596 5461 struct device *dev = &pdev->dev; 5597 5462 int ret; 5598 5463 5599 5599 - hisi_qm_pre_init(qm); 5600 5600 - 5601 5464 ret = hisi_qm_pci_init(qm); 5602 5465 if (ret) 5603 5466 return ret; ··· 5730 5597 ret = qm_master_ooo_check(qm); 5731 5598 if (ret) 5732 5599 return ret; 5600 5600 + 5601 5601 + if (qm->err_ini->set_priv_status) { 5602 5602 + ret = qm->err_ini->set_priv_status(qm); 5603 5603 + if (ret) 5604 5604 + return ret; 5605 5605 + } 5733 5606 5734 5607 ret = qm_set_pf_mse(qm, false); 5735 5608 if (ret)

+1 -2

drivers/crypto/hisilicon/sec2/sec.h

reviewed

··· 37 37 u8 *a_ivin; 38 38 dma_addr_t a_ivin_dma; 39 39 struct aead_request *aead_req; 40 40 + bool fallback; 40 41 }; 41 42 42 43 /* SEC request of Crypto */ ··· 91 90 dma_addr_t a_key_dma; 92 91 u8 *a_key; 93 92 u8 a_key_len; 94 94 - u8 mac_len; 95 93 u8 a_alg; 96 96 - bool fallback; 97 94 struct crypto_shash *hash_tfm; 98 95 struct crypto_aead *fallback_aead_tfm; 99 96 };

+73 -84

drivers/crypto/hisilicon/sec2/sec_crypto.c

reviewed

··· 948 948 struct aead_request *aead_req = req->aead_req; 949 949 struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 950 950 size_t authsize = crypto_aead_authsize(tfm); 951 951 - u8 *mac_out = req->out_mac; 952 951 struct scatterlist *sgl = aead_req->src; 952 952 + u8 *mac_out = req->out_mac; 953 953 size_t copy_size; 954 954 off_t skip_size; 955 955 956 956 /* Copy input mac */ 957 957 skip_size = aead_req->assoclen + aead_req->cryptlen - authsize; 958 958 - copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out, 959 959 - authsize, skip_size); 958 958 + copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out, authsize, skip_size); 960 959 if (unlikely(copy_size != authsize)) 961 960 return -EINVAL; 962 961 ··· 1119 1120 struct sec_ctx *ctx = crypto_tfm_ctx(tfm); 1120 1121 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 1121 1122 1122 1122 - if (unlikely(a_ctx->fallback_aead_tfm)) 1123 1123 - return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize); 1124 1124 - 1125 1125 - return 0; 1123 1123 + return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize); 1126 1124 } 1127 1125 1128 1126 static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx, ··· 1135 1139 static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, 1136 1140 const u32 keylen, const enum sec_hash_alg a_alg, 1137 1141 const enum sec_calg c_alg, 1138 1138 - const enum sec_mac_len mac_len, 1139 1142 const enum sec_cmode c_mode) 1140 1143 { 1141 1144 struct sec_ctx *ctx = crypto_aead_ctx(tfm); ··· 1146 1151 1147 1152 ctx->a_ctx.a_alg = a_alg; 1148 1153 ctx->c_ctx.c_alg = c_alg; 1149 1149 - ctx->a_ctx.mac_len = mac_len; 1150 1154 c_ctx->c_mode = c_mode; 1151 1155 1152 1156 if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) { ··· 1156 1162 } 1157 1163 memcpy(c_ctx->c_key, key, keylen); 1158 1164 1159 1159 - if (unlikely(a_ctx->fallback_aead_tfm)) { 1160 1160 - ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); 1161 1161 - if (ret) 1162 1162 - return ret; 1163 1163 - } 1164 1164 - 1165 1165 - return 0; 1165 1165 + return sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); 1166 1166 } 1167 1167 1168 1168 ret = crypto_authenc_extractkeys(&keys, key, keylen); ··· 1175 1187 goto bad_key; 1176 1188 } 1177 1189 1178 1178 - if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) || 1179 1179 - (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) { 1190 1190 + if (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK) { 1180 1191 ret = -EINVAL; 1181 1181 - dev_err(dev, "MAC or AUTH key length error!\n"); 1192 1192 + dev_err(dev, "AUTH key length error!\n"); 1193 1193 + goto bad_key; 1194 1194 + } 1195 1195 + 1196 1196 + ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); 1197 1197 + if (ret) { 1198 1198 + dev_err(dev, "set sec fallback key err!\n"); 1182 1199 goto bad_key; 1183 1200 } 1184 1201 ··· 1195 1202 } 1196 1203 1197 1204 1198 1198 - #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \ 1199 1199 - static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \ 1200 1200 - u32 keylen) \ 1201 1201 - { \ 1202 1202 - return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\ 1205 1205 + #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, cmode) \ 1206 1206 + static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, u32 keylen) \ 1207 1207 + { \ 1208 1208 + return sec_aead_setkey(tfm, key, keylen, aalg, calg, cmode); \ 1203 1209 } 1204 1210 1205 1205 - GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, 1206 1206 - SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC) 1207 1207 - GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, 1208 1208 - SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC) 1209 1209 - GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, 1210 1210 - SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC) 1211 1211 - GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, 1212 1212 - SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) 1213 1213 - GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, 1214 1214 - SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) 1215 1215 - GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, 1216 1216 - SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) 1217 1217 - GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, 1218 1218 - SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) 1211 1211 + GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, SEC_CALG_AES, SEC_CMODE_CBC) 1212 1212 + GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, SEC_CALG_AES, SEC_CMODE_CBC) 1213 1213 + GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, SEC_CALG_AES, SEC_CMODE_CBC) 1214 1214 + GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, SEC_CMODE_CCM) 1215 1215 + GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, SEC_CMODE_GCM) 1216 1216 + GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, SEC_CMODE_CCM) 1217 1217 + GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, SEC_CMODE_GCM) 1219 1218 1220 1219 static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 1221 1220 { ··· 1455 1470 static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) 1456 1471 { 1457 1472 struct aead_request *aead_req = req->aead_req.aead_req; 1458 1458 - struct sec_cipher_req *c_req = &req->c_req; 1473 1473 + struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 1474 1474 + size_t authsize = crypto_aead_authsize(tfm); 1459 1475 struct sec_aead_req *a_req = &req->aead_req; 1460 1460 - size_t authsize = ctx->a_ctx.mac_len; 1476 1476 + struct sec_cipher_req *c_req = &req->c_req; 1461 1477 u32 data_size = aead_req->cryptlen; 1462 1478 u8 flage = 0; 1463 1479 u8 cm, cl; ··· 1499 1513 static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) 1500 1514 { 1501 1515 struct aead_request *aead_req = req->aead_req.aead_req; 1502 1502 - struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 1503 1503 - size_t authsize = crypto_aead_authsize(tfm); 1504 1504 - struct sec_cipher_req *c_req = &req->c_req; 1505 1516 struct sec_aead_req *a_req = &req->aead_req; 1517 1517 + struct sec_cipher_req *c_req = &req->c_req; 1506 1518 1507 1519 memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); 1508 1520 ··· 1508 1524 /* 1509 1525 * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter}, 1510 1526 * the counter must set to 0x01 1527 1527 + * CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} 1511 1528 */ 1512 1512 - ctx->a_ctx.mac_len = authsize; 1513 1513 - /* CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */ 1514 1529 set_aead_auth_iv(ctx, req); 1515 1515 - } 1516 1516 - 1517 1517 - /* GCM 12Byte Cipher_IV == Auth_IV */ 1518 1518 - if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) { 1519 1519 - ctx->a_ctx.mac_len = authsize; 1530 1530 + } else if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) { 1531 1531 + /* GCM 12Byte Cipher_IV == Auth_IV */ 1520 1532 memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE); 1521 1533 } 1522 1534 } ··· 1522 1542 { 1523 1543 struct sec_aead_req *a_req = &req->aead_req; 1524 1544 struct aead_request *aq = a_req->aead_req; 1545 1545 + struct crypto_aead *tfm = crypto_aead_reqtfm(aq); 1546 1546 + size_t authsize = crypto_aead_authsize(tfm); 1525 1547 1526 1548 /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ 1527 1527 - sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)ctx->mac_len); 1549 1549 + sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)authsize); 1528 1550 1529 1551 /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ 1530 1552 sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr; ··· 1550 1568 { 1551 1569 struct sec_aead_req *a_req = &req->aead_req; 1552 1570 struct aead_request *aq = a_req->aead_req; 1571 1571 + struct crypto_aead *tfm = crypto_aead_reqtfm(aq); 1572 1572 + size_t authsize = crypto_aead_authsize(tfm); 1553 1573 1554 1574 /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ 1555 1555 - sqe3->c_icv_key |= cpu_to_le16((u16)ctx->mac_len << SEC_MAC_OFFSET_V3); 1575 1575 + sqe3->c_icv_key |= cpu_to_le16((u16)authsize << SEC_MAC_OFFSET_V3); 1556 1576 1557 1577 /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ 1558 1578 sqe3->a_key_addr = sqe3->c_key_addr; ··· 1578 1594 struct sec_aead_req *a_req = &req->aead_req; 1579 1595 struct sec_cipher_req *c_req = &req->c_req; 1580 1596 struct aead_request *aq = a_req->aead_req; 1597 1597 + struct crypto_aead *tfm = crypto_aead_reqtfm(aq); 1598 1598 + size_t authsize = crypto_aead_authsize(tfm); 1581 1599 1582 1600 sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); 1583 1601 1584 1584 - sec_sqe->type2.mac_key_alg = 1585 1585 - cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE); 1602 1602 + sec_sqe->type2.mac_key_alg = cpu_to_le32(authsize / SEC_SQE_LEN_RATE); 1586 1603 1587 1604 sec_sqe->type2.mac_key_alg |= 1588 1605 cpu_to_le32((u32)((ctx->a_key_len) / ··· 1633 1648 struct sec_aead_req *a_req = &req->aead_req; 1634 1649 struct sec_cipher_req *c_req = &req->c_req; 1635 1650 struct aead_request *aq = a_req->aead_req; 1651 1651 + struct crypto_aead *tfm = crypto_aead_reqtfm(aq); 1652 1652 + size_t authsize = crypto_aead_authsize(tfm); 1636 1653 1637 1654 sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); 1638 1655 1639 1656 sqe3->auth_mac_key |= 1640 1640 - cpu_to_le32((u32)(ctx->mac_len / 1657 1657 + cpu_to_le32((u32)(authsize / 1641 1658 SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3); 1642 1659 1643 1660 sqe3->auth_mac_key |= ··· 1690 1703 { 1691 1704 struct aead_request *a_req = req->aead_req.aead_req; 1692 1705 struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 1706 1706 + size_t authsize = crypto_aead_authsize(tfm); 1693 1707 struct sec_aead_req *aead_req = &req->aead_req; 1694 1708 struct sec_cipher_req *c_req = &req->c_req; 1695 1695 - size_t authsize = crypto_aead_authsize(tfm); 1696 1709 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 1697 1710 struct aead_request *backlog_aead_req; 1698 1711 struct sec_req *backlog_req; ··· 1705 1718 if (!err && c_req->encrypt) { 1706 1719 struct scatterlist *sgl = a_req->dst; 1707 1720 1708 1708 - sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), 1709 1709 - aead_req->out_mac, 1710 1710 - authsize, a_req->cryptlen + 1711 1711 - a_req->assoclen); 1721 1721 + sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), aead_req->out_mac, 1722 1722 + authsize, a_req->cryptlen + a_req->assoclen); 1712 1723 if (unlikely(sz != authsize)) { 1713 1724 dev_err(c->dev, "copy out mac err!\n"); 1714 1725 err = -EINVAL; ··· 1914 1929 1915 1930 static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) 1916 1931 { 1932 1932 + struct aead_alg *alg = crypto_aead_alg(tfm); 1917 1933 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1918 1918 - struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 1934 1934 + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 1935 1935 + const char *aead_name = alg->base.cra_name; 1919 1936 int ret; 1920 1937 1921 1938 ret = sec_aead_init(tfm); ··· 1926 1939 return ret; 1927 1940 } 1928 1941 1929 1929 - auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); 1930 1930 - if (IS_ERR(auth_ctx->hash_tfm)) { 1942 1942 + a_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); 1943 1943 + if (IS_ERR(a_ctx->hash_tfm)) { 1931 1944 dev_err(ctx->dev, "aead alloc shash error!\n"); 1932 1945 sec_aead_exit(tfm); 1933 1933 - return PTR_ERR(auth_ctx->hash_tfm); 1946 1946 + return PTR_ERR(a_ctx->hash_tfm); 1947 1947 + } 1948 1948 + 1949 1949 + a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, 1950 1950 + CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC); 1951 1951 + if (IS_ERR(a_ctx->fallback_aead_tfm)) { 1952 1952 + dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); 1953 1953 + crypto_free_shash(ctx->a_ctx.hash_tfm); 1954 1954 + sec_aead_exit(tfm); 1955 1955 + return PTR_ERR(a_ctx->fallback_aead_tfm); 1934 1956 } 1935 1957 1936 1958 return 0; ··· 1949 1953 { 1950 1954 struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1951 1955 1956 1956 + crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); 1952 1957 crypto_free_shash(ctx->a_ctx.hash_tfm); 1953 1958 sec_aead_exit(tfm); 1954 1959 } ··· 1976 1979 sec_aead_exit(tfm); 1977 1980 return PTR_ERR(a_ctx->fallback_aead_tfm); 1978 1981 } 1979 1979 - a_ctx->fallback = false; 1980 1982 1981 1983 return 0; 1982 1984 } ··· 2229 2233 { 2230 2234 struct aead_request *req = sreq->aead_req.aead_req; 2231 2235 struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2232 2232 - size_t authsize = crypto_aead_authsize(tfm); 2236 2236 + size_t sz = crypto_aead_authsize(tfm); 2233 2237 u8 c_mode = ctx->c_ctx.c_mode; 2234 2238 struct device *dev = ctx->dev; 2235 2239 int ret; 2236 2240 2237 2237 - if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || 2238 2238 - req->assoclen > SEC_MAX_AAD_LEN)) { 2239 2239 - dev_err(dev, "aead input spec error!\n"); 2241 2241 + /* Hardware does not handle cases where authsize is less than 4 bytes */ 2242 2242 + if (unlikely(sz < MIN_MAC_LEN)) { 2243 2243 + sreq->aead_req.fallback = true; 2240 2244 return -EINVAL; 2241 2245 } 2242 2246 2243 2243 - if (unlikely((c_mode == SEC_CMODE_GCM && authsize < DES_BLOCK_SIZE) || 2244 2244 - (c_mode == SEC_CMODE_CCM && (authsize < MIN_MAC_LEN || 2245 2245 - authsize & MAC_LEN_MASK)))) { 2246 2246 - dev_err(dev, "aead input mac length error!\n"); 2247 2247 + if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || 2248 2248 + req->assoclen > SEC_MAX_AAD_LEN)) { 2249 2249 + dev_err(dev, "aead input spec error!\n"); 2247 2250 return -EINVAL; 2248 2251 } 2249 2252 ··· 2261 2266 if (sreq->c_req.encrypt) 2262 2267 sreq->c_req.c_len = req->cryptlen; 2263 2268 else 2264 2264 - sreq->c_req.c_len = req->cryptlen - authsize; 2269 2269 + sreq->c_req.c_len = req->cryptlen - sz; 2265 2270 if (c_mode == SEC_CMODE_CBC) { 2266 2271 if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { 2267 2272 dev_err(dev, "aead crypto length error!\n"); ··· 2287 2292 2288 2293 if (ctx->sec->qm.ver == QM_HW_V2) { 2289 2294 if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt && 2290 2290 - req->cryptlen <= authsize))) { 2291 2291 - ctx->a_ctx.fallback = true; 2295 2295 + req->cryptlen <= authsize))) { 2296 2296 + sreq->aead_req.fallback = true; 2292 2297 return -EINVAL; 2293 2298 } 2294 2299 } ··· 2316 2321 bool encrypt) 2317 2322 { 2318 2323 struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 2319 2319 - struct device *dev = ctx->dev; 2320 2324 struct aead_request *subreq; 2321 2325 int ret; 2322 2322 - 2323 2323 - /* Kunpeng920 aead mode not support input 0 size */ 2324 2324 - if (!a_ctx->fallback_aead_tfm) { 2325 2325 - dev_err(dev, "aead fallback tfm is NULL!\n"); 2326 2326 - return -EINVAL; 2327 2327 - } 2328 2326 2329 2327 subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL); 2330 2328 if (!subreq) ··· 2350 2362 req->aead_req.aead_req = a_req; 2351 2363 req->c_req.encrypt = encrypt; 2352 2364 req->ctx = ctx; 2365 2365 + req->aead_req.fallback = false; 2353 2366 2354 2367 ret = sec_aead_param_check(ctx, req); 2355 2368 if (unlikely(ret)) { 2356 2356 - if (ctx->a_ctx.fallback) 2369 2369 + if (req->aead_req.fallback) 2357 2370 return sec_aead_soft_crypto(ctx, a_req, encrypt); 2358 2371 return -EINVAL; 2359 2372 }

-11

drivers/crypto/hisilicon/sec2/sec_crypto.h

reviewed

··· 23 23 SEC_A_HMAC_SHA512 = 0x15, 24 24 }; 25 25 26 26 - enum sec_mac_len { 27 27 - SEC_HMAC_CCM_MAC = 16, 28 28 - SEC_HMAC_GCM_MAC = 16, 29 29 - SEC_SM3_MAC = 32, 30 30 - SEC_HMAC_SM3_MAC = 32, 31 31 - SEC_HMAC_MD5_MAC = 16, 32 32 - SEC_HMAC_SHA1_MAC = 20, 33 33 - SEC_HMAC_SHA256_MAC = 32, 34 34 - SEC_HMAC_SHA512_MAC = 64, 35 35 - }; 36 36 - 37 26 enum sec_cmode { 38 27 SEC_CMODE_ECB = 0x0, 39 28 SEC_CMODE_CBC = 0x1,

+12 -1

drivers/crypto/hisilicon/sec2/sec_main.c

reviewed

··· 1097 1097 return ACC_ERR_RECOVERED; 1098 1098 } 1099 1099 1100 1100 + static bool sec_dev_is_abnormal(struct hisi_qm *qm) 1101 1101 + { 1102 1102 + u32 err_status; 1103 1103 + 1104 1104 + err_status = sec_get_hw_err_status(qm); 1105 1105 + if (err_status & qm->err_info.dev_shutdown_mask) 1106 1106 + return true; 1107 1107 + 1108 1108 + return false; 1109 1109 + } 1110 1110 + 1100 1111 static void sec_err_info_init(struct hisi_qm *qm) 1101 1112 { 1102 1113 struct hisi_qm_err_info *err_info = &qm->err_info; ··· 1140 1129 .show_last_dfx_regs = sec_show_last_dfx_regs, 1141 1130 .err_info_init = sec_err_info_init, 1142 1131 .get_err_result = sec_get_err_result, 1132 1132 + .dev_is_abnormal = sec_dev_is_abnormal, 1143 1133 }; 1144 1134 1145 1135 static int sec_pf_probe_init(struct sec_dev *sec) ··· 1192 1180 int ret; 1193 1181 1194 1182 qm->pdev = pdev; 1195 1195 - qm->ver = pdev->revision; 1196 1183 qm->mode = uacce_mode; 1197 1184 qm->sqe_size = SEC_SQE_SIZE; 1198 1185 qm->dev_name = sec_name;

+1 -1

drivers/crypto/hisilicon/zip/Makefile

reviewed

··· 1 1 obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += hisi_zip.o 2 2 - hisi_zip-objs = zip_main.o zip_crypto.o 2 2 + hisi_zip-objs = zip_main.o zip_crypto.o dae_main.o

+262

drivers/crypto/hisilicon/zip/dae_main.c

reviewed

··· 1 1 + // SPDX-License-Identifier: GPL-2.0 2 2 + /* Copyright (c) 2024 HiSilicon Limited. */ 3 3 + 4 4 + #include <linux/bitops.h> 5 5 + #include <linux/io.h> 6 6 + #include <linux/uacce.h> 7 7 + #include "zip.h" 8 8 + 9 9 + /* memory */ 10 10 + #define DAE_MEM_START_OFFSET 0x331040 11 11 + #define DAE_MEM_DONE_OFFSET 0x331044 12 12 + #define DAE_MEM_START_MASK 0x1 13 13 + #define DAE_MEM_DONE_MASK 0x1 14 14 + #define DAE_REG_RD_INTVRL_US 10 15 15 + #define DAE_REG_RD_TMOUT_US USEC_PER_SEC 16 16 + 17 17 + #define DAE_ALG_NAME "hashagg" 18 18 + 19 19 + /* error */ 20 20 + #define DAE_AXI_CFG_OFFSET 0x331000 21 21 + #define DAE_AXI_SHUTDOWN_MASK (BIT(0) | BIT(5)) 22 22 + #define DAE_ERR_SOURCE_OFFSET 0x331C84 23 23 + #define DAE_ERR_STATUS_OFFSET 0x331C88 24 24 + #define DAE_ERR_CE_OFFSET 0x331CA0 25 25 + #define DAE_ERR_CE_MASK BIT(3) 26 26 + #define DAE_ERR_NFE_OFFSET 0x331CA4 27 27 + #define DAE_ERR_NFE_MASK 0x17 28 28 + #define DAE_ERR_FE_OFFSET 0x331CA8 29 29 + #define DAE_ERR_FE_MASK 0 30 30 + #define DAE_ECC_MBIT_MASK BIT(2) 31 31 + #define DAE_ECC_INFO_OFFSET 0x33400C 32 32 + #define DAE_ERR_SHUTDOWN_OFFSET 0x331CAC 33 33 + #define DAE_ERR_SHUTDOWN_MASK 0x17 34 34 + #define DAE_ERR_ENABLE_OFFSET 0x331C80 35 35 + #define DAE_ERR_ENABLE_MASK (DAE_ERR_FE_MASK | DAE_ERR_NFE_MASK | DAE_ERR_CE_MASK) 36 36 + #define DAE_AM_CTRL_GLOBAL_OFFSET 0x330000 37 37 + #define DAE_AM_RETURN_OFFSET 0x330150 38 38 + #define DAE_AM_RETURN_MASK 0x3 39 39 + #define DAE_AXI_CFG_OFFSET 0x331000 40 40 + #define DAE_AXI_SHUTDOWN_EN_MASK (BIT(0) | BIT(5)) 41 41 + 42 42 + struct hisi_dae_hw_error { 43 43 + u32 int_msk; 44 44 + const char *msg; 45 45 + }; 46 46 + 47 47 + static const struct hisi_dae_hw_error dae_hw_error[] = { 48 48 + { .int_msk = BIT(0), .msg = "dae_axi_bus_err" }, 49 49 + { .int_msk = BIT(1), .msg = "dae_axi_poison_err" }, 50 50 + { .int_msk = BIT(2), .msg = "dae_ecc_2bit_err" }, 51 51 + { .int_msk = BIT(3), .msg = "dae_ecc_1bit_err" }, 52 52 + { .int_msk = BIT(4), .msg = "dae_fsm_hbeat_err" }, 53 53 + }; 54 54 + 55 55 + static inline bool dae_is_support(struct hisi_qm *qm) 56 56 + { 57 57 + if (test_bit(QM_SUPPORT_DAE, &qm->caps)) 58 58 + return true; 59 59 + 60 60 + return false; 61 61 + } 62 62 + 63 63 + int hisi_dae_set_user_domain(struct hisi_qm *qm) 64 64 + { 65 65 + u32 val; 66 66 + int ret; 67 67 + 68 68 + if (!dae_is_support(qm)) 69 69 + return 0; 70 70 + 71 71 + val = readl(qm->io_base + DAE_MEM_START_OFFSET); 72 72 + val |= DAE_MEM_START_MASK; 73 73 + writel(val, qm->io_base + DAE_MEM_START_OFFSET); 74 74 + ret = readl_relaxed_poll_timeout(qm->io_base + DAE_MEM_DONE_OFFSET, val, 75 75 + val & DAE_MEM_DONE_MASK, 76 76 + DAE_REG_RD_INTVRL_US, DAE_REG_RD_TMOUT_US); 77 77 + if (ret) 78 78 + pci_err(qm->pdev, "failed to init dae memory!\n"); 79 79 + 80 80 + return ret; 81 81 + } 82 82 + 83 83 + int hisi_dae_set_alg(struct hisi_qm *qm) 84 84 + { 85 85 + size_t len; 86 86 + 87 87 + if (!dae_is_support(qm)) 88 88 + return 0; 89 89 + 90 90 + if (!qm->uacce) 91 91 + return 0; 92 92 + 93 93 + len = strlen(qm->uacce->algs); 94 94 + /* A line break may be required */ 95 95 + if (len + strlen(DAE_ALG_NAME) + 1 >= QM_DEV_ALG_MAX_LEN) { 96 96 + pci_err(qm->pdev, "algorithm name is too long!\n"); 97 97 + return -EINVAL; 98 98 + } 99 99 + 100 100 + if (len) 101 101 + strcat((char *)qm->uacce->algs, "\n"); 102 102 + 103 103 + strcat((char *)qm->uacce->algs, DAE_ALG_NAME); 104 104 + 105 105 + return 0; 106 106 + } 107 107 + 108 108 + static void hisi_dae_master_ooo_ctrl(struct hisi_qm *qm, bool enable) 109 109 + { 110 110 + u32 axi_val, err_val; 111 111 + 112 112 + axi_val = readl(qm->io_base + DAE_AXI_CFG_OFFSET); 113 113 + if (enable) { 114 114 + axi_val |= DAE_AXI_SHUTDOWN_MASK; 115 115 + err_val = DAE_ERR_SHUTDOWN_MASK; 116 116 + } else { 117 117 + axi_val &= ~DAE_AXI_SHUTDOWN_MASK; 118 118 + err_val = 0; 119 119 + } 120 120 + 121 121 + writel(axi_val, qm->io_base + DAE_AXI_CFG_OFFSET); 122 122 + writel(err_val, qm->io_base + DAE_ERR_SHUTDOWN_OFFSET); 123 123 + } 124 124 + 125 125 + void hisi_dae_hw_error_enable(struct hisi_qm *qm) 126 126 + { 127 127 + if (!dae_is_support(qm)) 128 128 + return; 129 129 + 130 130 + /* clear dae hw error source if having */ 131 131 + writel(DAE_ERR_ENABLE_MASK, qm->io_base + DAE_ERR_SOURCE_OFFSET); 132 132 + 133 133 + /* configure error type */ 134 134 + writel(DAE_ERR_CE_MASK, qm->io_base + DAE_ERR_CE_OFFSET); 135 135 + writel(DAE_ERR_NFE_MASK, qm->io_base + DAE_ERR_NFE_OFFSET); 136 136 + writel(DAE_ERR_FE_MASK, qm->io_base + DAE_ERR_FE_OFFSET); 137 137 + 138 138 + hisi_dae_master_ooo_ctrl(qm, true); 139 139 + 140 140 + /* enable dae hw error interrupts */ 141 141 + writel(DAE_ERR_ENABLE_MASK, qm->io_base + DAE_ERR_ENABLE_OFFSET); 142 142 + } 143 143 + 144 144 + void hisi_dae_hw_error_disable(struct hisi_qm *qm) 145 145 + { 146 146 + if (!dae_is_support(qm)) 147 147 + return; 148 148 + 149 149 + writel(0, qm->io_base + DAE_ERR_ENABLE_OFFSET); 150 150 + hisi_dae_master_ooo_ctrl(qm, false); 151 151 + } 152 152 + 153 153 + static u32 hisi_dae_get_hw_err_status(struct hisi_qm *qm) 154 154 + { 155 155 + return readl(qm->io_base + DAE_ERR_STATUS_OFFSET); 156 156 + } 157 157 + 158 158 + static void hisi_dae_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) 159 159 + { 160 160 + if (!dae_is_support(qm)) 161 161 + return; 162 162 + 163 163 + writel(err_sts, qm->io_base + DAE_ERR_SOURCE_OFFSET); 164 164 + } 165 165 + 166 166 + static void hisi_dae_disable_error_report(struct hisi_qm *qm, u32 err_type) 167 167 + { 168 168 + writel(DAE_ERR_NFE_MASK & (~err_type), qm->io_base + DAE_ERR_NFE_OFFSET); 169 169 + } 170 170 + 171 171 + static void hisi_dae_log_hw_error(struct hisi_qm *qm, u32 err_type) 172 172 + { 173 173 + const struct hisi_dae_hw_error *err = dae_hw_error; 174 174 + struct device *dev = &qm->pdev->dev; 175 175 + u32 ecc_info; 176 176 + size_t i; 177 177 + 178 178 + for (i = 0; i < ARRAY_SIZE(dae_hw_error); i++) { 179 179 + err = &dae_hw_error[i]; 180 180 + if (!(err->int_msk & err_type)) 181 181 + continue; 182 182 + 183 183 + dev_err(dev, "%s [error status=0x%x] found\n", 184 184 + err->msg, err->int_msk); 185 185 + 186 186 + if (err->int_msk & DAE_ECC_MBIT_MASK) { 187 187 + ecc_info = readl(qm->io_base + DAE_ECC_INFO_OFFSET); 188 188 + dev_err(dev, "dae multi ecc sram info 0x%x\n", ecc_info); 189 189 + } 190 190 + } 191 191 + } 192 192 + 193 193 + enum acc_err_result hisi_dae_get_err_result(struct hisi_qm *qm) 194 194 + { 195 195 + u32 err_status; 196 196 + 197 197 + if (!dae_is_support(qm)) 198 198 + return ACC_ERR_NONE; 199 199 + 200 200 + err_status = hisi_dae_get_hw_err_status(qm); 201 201 + if (!err_status) 202 202 + return ACC_ERR_NONE; 203 203 + 204 204 + hisi_dae_log_hw_error(qm, err_status); 205 205 + 206 206 + if (err_status & DAE_ERR_NFE_MASK) { 207 207 + /* Disable the same error reporting until device is recovered. */ 208 208 + hisi_dae_disable_error_report(qm, err_status); 209 209 + return ACC_ERR_NEED_RESET; 210 210 + } 211 211 + hisi_dae_clear_hw_err_status(qm, err_status); 212 212 + 213 213 + return ACC_ERR_RECOVERED; 214 214 + } 215 215 + 216 216 + bool hisi_dae_dev_is_abnormal(struct hisi_qm *qm) 217 217 + { 218 218 + u32 err_status; 219 219 + 220 220 + if (!dae_is_support(qm)) 221 221 + return false; 222 222 + 223 223 + err_status = hisi_dae_get_hw_err_status(qm); 224 224 + if (err_status & DAE_ERR_NFE_MASK) 225 225 + return true; 226 226 + 227 227 + return false; 228 228 + } 229 229 + 230 230 + int hisi_dae_close_axi_master_ooo(struct hisi_qm *qm) 231 231 + { 232 232 + u32 val; 233 233 + int ret; 234 234 + 235 235 + if (!dae_is_support(qm)) 236 236 + return 0; 237 237 + 238 238 + val = readl(qm->io_base + DAE_AM_CTRL_GLOBAL_OFFSET); 239 239 + val |= BIT(0); 240 240 + writel(val, qm->io_base + DAE_AM_CTRL_GLOBAL_OFFSET); 241 241 + 242 242 + ret = readl_relaxed_poll_timeout(qm->io_base + DAE_AM_RETURN_OFFSET, 243 243 + val, (val == DAE_AM_RETURN_MASK), 244 244 + DAE_REG_RD_INTVRL_US, DAE_REG_RD_TMOUT_US); 245 245 + if (ret) 246 246 + dev_err(&qm->pdev->dev, "failed to close dae axi ooo!\n"); 247 247 + 248 248 + return ret; 249 249 + } 250 250 + 251 251 + void hisi_dae_open_axi_master_ooo(struct hisi_qm *qm) 252 252 + { 253 253 + u32 val; 254 254 + 255 255 + if (!dae_is_support(qm)) 256 256 + return; 257 257 + 258 258 + val = readl(qm->io_base + DAE_AXI_CFG_OFFSET); 259 259 + 260 260 + writel(val & ~DAE_AXI_SHUTDOWN_EN_MASK, qm->io_base + DAE_AXI_CFG_OFFSET); 261 261 + writel(val | DAE_AXI_SHUTDOWN_EN_MASK, qm->io_base + DAE_AXI_CFG_OFFSET); 262 262 + }

+8

drivers/crypto/hisilicon/zip/zip.h

reviewed

··· 103 103 int hisi_zip_register_to_crypto(struct hisi_qm *qm); 104 104 void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); 105 105 bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg); 106 106 + int hisi_dae_set_user_domain(struct hisi_qm *qm); 107 107 + int hisi_dae_set_alg(struct hisi_qm *qm); 108 108 + void hisi_dae_hw_error_disable(struct hisi_qm *qm); 109 109 + void hisi_dae_hw_error_enable(struct hisi_qm *qm); 110 110 + void hisi_dae_open_axi_master_ooo(struct hisi_qm *qm); 111 111 + int hisi_dae_close_axi_master_ooo(struct hisi_qm *qm); 112 112 + bool hisi_dae_dev_is_abnormal(struct hisi_qm *qm); 113 113 + enum acc_err_result hisi_dae_get_err_result(struct hisi_qm *qm); 106 114 #endif

+45 -7

drivers/crypto/hisilicon/zip/zip_main.c

reviewed

··· 582 582 583 583 hisi_zip_enable_clock_gate(qm); 584 584 585 585 - return 0; 585 585 + return hisi_dae_set_user_domain(qm); 586 586 } 587 587 588 588 static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) ··· 631 631 632 632 /* enable ZIP hw error interrupts */ 633 633 writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG); 634 634 + 635 635 + hisi_dae_hw_error_enable(qm); 634 636 } 635 637 636 638 static void hisi_zip_hw_error_disable(struct hisi_qm *qm) ··· 645 643 writel(ce | nfe | HZIP_CORE_INT_RAS_FE_ENB_MASK, qm->io_base + HZIP_CORE_INT_MASK_REG); 646 644 647 645 hisi_zip_master_ooo_ctrl(qm, false); 646 646 + 647 647 + hisi_dae_hw_error_disable(qm); 648 648 } 649 649 650 650 static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file) ··· 1133 1129 1134 1130 writel(val | HZIP_AXI_SHUTDOWN_ENABLE, 1135 1131 qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); 1132 1132 + 1133 1133 + hisi_dae_open_axi_master_ooo(qm); 1136 1134 } 1137 1135 1138 1136 static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm) ··· 1153 1147 1154 1148 static enum acc_err_result hisi_zip_get_err_result(struct hisi_qm *qm) 1155 1149 { 1150 1150 + enum acc_err_result zip_result = ACC_ERR_NONE; 1151 1151 + enum acc_err_result dae_result; 1156 1152 u32 err_status; 1157 1153 1154 1154 + /* Get device hardware new error status */ 1158 1155 err_status = hisi_zip_get_hw_err_status(qm); 1159 1156 if (err_status) { 1160 1157 if (err_status & qm->err_info.ecc_2bits_mask) ··· 1168 1159 /* Disable the same error reporting until device is recovered. */ 1169 1160 hisi_zip_disable_error_report(qm, err_status); 1170 1161 return ACC_ERR_NEED_RESET; 1162 1162 + } else { 1163 1163 + hisi_zip_clear_hw_err_status(qm, err_status); 1171 1164 } 1172 1172 - hisi_zip_clear_hw_err_status(qm, err_status); 1173 1165 } 1174 1166 1175 1175 - return ACC_ERR_RECOVERED; 1167 1167 + dae_result = hisi_dae_get_err_result(qm); 1168 1168 + 1169 1169 + return (zip_result == ACC_ERR_NEED_RESET || 1170 1170 + dae_result == ACC_ERR_NEED_RESET) ? 1171 1171 + ACC_ERR_NEED_RESET : ACC_ERR_RECOVERED; 1172 1172 + } 1173 1173 + 1174 1174 + static bool hisi_zip_dev_is_abnormal(struct hisi_qm *qm) 1175 1175 + { 1176 1176 + u32 err_status; 1177 1177 + 1178 1178 + err_status = hisi_zip_get_hw_err_status(qm); 1179 1179 + if (err_status & qm->err_info.dev_shutdown_mask) 1180 1180 + return true; 1181 1181 + 1182 1182 + return hisi_dae_dev_is_abnormal(qm); 1183 1183 + } 1184 1184 + 1185 1185 + static int hisi_zip_set_priv_status(struct hisi_qm *qm) 1186 1186 + { 1187 1187 + return hisi_dae_close_axi_master_ooo(qm); 1176 1188 } 1177 1189 1178 1190 static void hisi_zip_err_info_init(struct hisi_qm *qm) ··· 1230 1200 .show_last_dfx_regs = hisi_zip_show_last_dfx_regs, 1231 1201 .err_info_init = hisi_zip_err_info_init, 1232 1202 .get_err_result = hisi_zip_get_err_result, 1203 1203 + .set_priv_status = hisi_zip_set_priv_status, 1204 1204 + .dev_is_abnormal = hisi_zip_dev_is_abnormal, 1233 1205 }; 1234 1206 1235 1207 static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) ··· 1296 1264 int ret; 1297 1265 1298 1266 qm->pdev = pdev; 1299 1299 - qm->ver = pdev->revision; 1300 1267 qm->mode = uacce_mode; 1301 1268 qm->sqe_size = HZIP_SQE_SIZE; 1302 1269 qm->dev_name = hisi_zip_name; ··· 1332 1301 ret = zip_pre_store_cap_reg(qm); 1333 1302 if (ret) { 1334 1303 pci_err(qm->pdev, "Failed to pre-store capability registers!\n"); 1335 1335 - hisi_qm_uninit(qm); 1336 1336 - return ret; 1304 1304 + goto err_qm_uninit; 1337 1305 } 1338 1306 1339 1307 alg_msk = qm->cap_tables.dev_cap_table[ZIP_ALG_BITMAP].cap_val; 1340 1308 ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs)); 1341 1309 if (ret) { 1342 1310 pci_err(qm->pdev, "Failed to set zip algs!\n"); 1343 1343 - hisi_qm_uninit(qm); 1311 1311 + goto err_qm_uninit; 1344 1312 } 1345 1313 1314 1314 + ret = hisi_dae_set_alg(qm); 1315 1315 + if (ret) 1316 1316 + goto err_qm_uninit; 1317 1317 + 1318 1318 + return 0; 1319 1319 + 1320 1320 + err_qm_uninit: 1321 1321 + hisi_qm_uninit(qm); 1346 1322 return ret; 1347 1323 } 1348 1324

+1 -1

drivers/crypto/intel/iaa/iaa_crypto_main.c

reviewed

··· 173 173 async_mode = false; 174 174 use_irq = false; 175 175 } else if (sysfs_streq(name, "async")) { 176 176 - async_mode = true; 176 176 + async_mode = false; 177 177 use_irq = false; 178 178 } else if (sysfs_streq(name, "async_irq")) { 179 179 async_mode = true;

+3

drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c

reviewed

··· 471 471 return -ENODEV; 472 472 } 473 473 npe_id = npe_spec.args[0]; 474 474 + of_node_put(npe_spec.np); 474 475 475 476 ret = of_parse_phandle_with_fixed_args(np, "queue-rx", 1, 0, 476 477 &queue_spec); ··· 480 479 return -ENODEV; 481 480 } 482 481 recv_qid = queue_spec.args[0]; 482 482 + of_node_put(queue_spec.np); 483 483 484 484 ret = of_parse_phandle_with_fixed_args(np, "queue-txready", 1, 0, 485 485 &queue_spec); ··· 489 487 return -ENODEV; 490 488 } 491 489 send_qid = queue_spec.args[0]; 490 490 + of_node_put(queue_spec.np); 492 491 } else { 493 492 /* 494 493 * Hardcoded engine when using platform data, this goes away

-96

drivers/crypto/n2_asm.S

reviewed

··· 1 1 - /* SPDX-License-Identifier: GPL-2.0 */ 2 2 - /* n2_asm.S: Hypervisor calls for NCS support. 3 3 - * 4 4 - * Copyright (C) 2009 David S. Miller <davem@davemloft.net> 5 5 - */ 6 6 - 7 7 - #include <linux/linkage.h> 8 8 - #include <asm/hypervisor.h> 9 9 - #include "n2_core.h" 10 10 - 11 11 - /* o0: queue type 12 12 - * o1: RA of queue 13 13 - * o2: num entries in queue 14 14 - * o3: address of queue handle return 15 15 - */ 16 16 - ENTRY(sun4v_ncs_qconf) 17 17 - mov HV_FAST_NCS_QCONF, %o5 18 18 - ta HV_FAST_TRAP 19 19 - stx %o1, [%o3] 20 20 - retl 21 21 - nop 22 22 - ENDPROC(sun4v_ncs_qconf) 23 23 - 24 24 - /* %o0: queue handle 25 25 - * %o1: address of queue type return 26 26 - * %o2: address of queue base address return 27 27 - * %o3: address of queue num entries return 28 28 - */ 29 29 - ENTRY(sun4v_ncs_qinfo) 30 30 - mov %o1, %g1 31 31 - mov %o2, %g2 32 32 - mov %o3, %g3 33 33 - mov HV_FAST_NCS_QINFO, %o5 34 34 - ta HV_FAST_TRAP 35 35 - stx %o1, [%g1] 36 36 - stx %o2, [%g2] 37 37 - stx %o3, [%g3] 38 38 - retl 39 39 - nop 40 40 - ENDPROC(sun4v_ncs_qinfo) 41 41 - 42 42 - /* %o0: queue handle 43 43 - * %o1: address of head offset return 44 44 - */ 45 45 - ENTRY(sun4v_ncs_gethead) 46 46 - mov %o1, %o2 47 47 - mov HV_FAST_NCS_GETHEAD, %o5 48 48 - ta HV_FAST_TRAP 49 49 - stx %o1, [%o2] 50 50 - retl 51 51 - nop 52 52 - ENDPROC(sun4v_ncs_gethead) 53 53 - 54 54 - /* %o0: queue handle 55 55 - * %o1: address of tail offset return 56 56 - */ 57 57 - ENTRY(sun4v_ncs_gettail) 58 58 - mov %o1, %o2 59 59 - mov HV_FAST_NCS_GETTAIL, %o5 60 60 - ta HV_FAST_TRAP 61 61 - stx %o1, [%o2] 62 62 - retl 63 63 - nop 64 64 - ENDPROC(sun4v_ncs_gettail) 65 65 - 66 66 - /* %o0: queue handle 67 67 - * %o1: new tail offset 68 68 - */ 69 69 - ENTRY(sun4v_ncs_settail) 70 70 - mov HV_FAST_NCS_SETTAIL, %o5 71 71 - ta HV_FAST_TRAP 72 72 - retl 73 73 - nop 74 74 - ENDPROC(sun4v_ncs_settail) 75 75 - 76 76 - /* %o0: queue handle 77 77 - * %o1: address of devino return 78 78 - */ 79 79 - ENTRY(sun4v_ncs_qhandle_to_devino) 80 80 - mov %o1, %o2 81 81 - mov HV_FAST_NCS_QHANDLE_TO_DEVINO, %o5 82 82 - ta HV_FAST_TRAP 83 83 - stx %o1, [%o2] 84 84 - retl 85 85 - nop 86 86 - ENDPROC(sun4v_ncs_qhandle_to_devino) 87 87 - 88 88 - /* %o0: queue handle 89 89 - * %o1: new head offset 90 90 - */ 91 91 - ENTRY(sun4v_ncs_sethead_marker) 92 92 - mov HV_FAST_NCS_SETHEAD_MARKER, %o5 93 93 - ta HV_FAST_TRAP 94 94 - retl 95 95 - nop 96 96 - ENDPROC(sun4v_ncs_sethead_marker)

-2168

drivers/crypto/n2_core.c

reviewed

··· 1 1 - // SPDX-License-Identifier: GPL-2.0-only 2 2 - /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support. 3 3 - * 4 4 - * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net> 5 5 - */ 6 6 - 7 7 - #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 8 8 - 9 9 - #include <linux/kernel.h> 10 10 - #include <linux/module.h> 11 11 - #include <linux/of.h> 12 12 - #include <linux/of_address.h> 13 13 - #include <linux/platform_device.h> 14 14 - #include <linux/cpumask.h> 15 15 - #include <linux/slab.h> 16 16 - #include <linux/interrupt.h> 17 17 - #include <linux/crypto.h> 18 18 - #include <crypto/md5.h> 19 19 - #include <crypto/sha1.h> 20 20 - #include <crypto/sha2.h> 21 21 - #include <crypto/aes.h> 22 22 - #include <crypto/internal/des.h> 23 23 - #include <linux/mutex.h> 24 24 - #include <linux/delay.h> 25 25 - #include <linux/sched.h> 26 26 - 27 27 - #include <crypto/internal/hash.h> 28 28 - #include <crypto/internal/skcipher.h> 29 29 - #include <crypto/scatterwalk.h> 30 30 - #include <crypto/algapi.h> 31 31 - 32 32 - #include <asm/hypervisor.h> 33 33 - #include <asm/mdesc.h> 34 34 - 35 35 - #include "n2_core.h" 36 36 - 37 37 - #define DRV_MODULE_NAME "n2_crypto" 38 38 - #define DRV_MODULE_VERSION "0.2" 39 39 - #define DRV_MODULE_RELDATE "July 28, 2011" 40 40 - 41 41 - static const char version[] = 42 42 - DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 43 43 - 44 44 - MODULE_AUTHOR("David S. Miller <davem@davemloft.net>"); 45 45 - MODULE_DESCRIPTION("Niagara2 Crypto driver"); 46 46 - MODULE_LICENSE("GPL"); 47 47 - MODULE_VERSION(DRV_MODULE_VERSION); 48 48 - 49 49 - #define N2_CRA_PRIORITY 200 50 50 - 51 51 - static DEFINE_MUTEX(spu_lock); 52 52 - 53 53 - struct spu_queue { 54 54 - cpumask_t sharing; 55 55 - unsigned long qhandle; 56 56 - 57 57 - spinlock_t lock; 58 58 - u8 q_type; 59 59 - void *q; 60 60 - unsigned long head; 61 61 - unsigned long tail; 62 62 - struct list_head jobs; 63 63 - 64 64 - unsigned long devino; 65 65 - 66 66 - char irq_name[32]; 67 67 - unsigned int irq; 68 68 - 69 69 - struct list_head list; 70 70 - }; 71 71 - 72 72 - struct spu_qreg { 73 73 - struct spu_queue *queue; 74 74 - unsigned long type; 75 75 - }; 76 76 - 77 77 - static struct spu_queue **cpu_to_cwq; 78 78 - static struct spu_queue **cpu_to_mau; 79 79 - 80 80 - static unsigned long spu_next_offset(struct spu_queue *q, unsigned long off) 81 81 - { 82 82 - if (q->q_type == HV_NCS_QTYPE_MAU) { 83 83 - off += MAU_ENTRY_SIZE; 84 84 - if (off == (MAU_ENTRY_SIZE * MAU_NUM_ENTRIES)) 85 85 - off = 0; 86 86 - } else { 87 87 - off += CWQ_ENTRY_SIZE; 88 88 - if (off == (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES)) 89 89 - off = 0; 90 90 - } 91 91 - return off; 92 92 - } 93 93 - 94 94 - struct n2_request_common { 95 95 - struct list_head entry; 96 96 - unsigned int offset; 97 97 - }; 98 98 - #define OFFSET_NOT_RUNNING (~(unsigned int)0) 99 99 - 100 100 - /* An async job request records the final tail value it used in 101 101 - * n2_request_common->offset, test to see if that offset is in 102 102 - * the range old_head, new_head, inclusive. 103 103 - */ 104 104 - static inline bool job_finished(struct spu_queue *q, unsigned int offset, 105 105 - unsigned long old_head, unsigned long new_head) 106 106 - { 107 107 - if (old_head <= new_head) { 108 108 - if (offset > old_head && offset <= new_head) 109 109 - return true; 110 110 - } else { 111 111 - if (offset > old_head || offset <= new_head) 112 112 - return true; 113 113 - } 114 114 - return false; 115 115 - } 116 116 - 117 117 - /* When the HEAD marker is unequal to the actual HEAD, we get 118 118 - * a virtual device INO interrupt. We should process the 119 119 - * completed CWQ entries and adjust the HEAD marker to clear 120 120 - * the IRQ. 121 121 - */ 122 122 - static irqreturn_t cwq_intr(int irq, void *dev_id) 123 123 - { 124 124 - unsigned long off, new_head, hv_ret; 125 125 - struct spu_queue *q = dev_id; 126 126 - 127 127 - pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n", 128 128 - smp_processor_id(), q->qhandle); 129 129 - 130 130 - spin_lock(&q->lock); 131 131 - 132 132 - hv_ret = sun4v_ncs_gethead(q->qhandle, &new_head); 133 133 - 134 134 - pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n", 135 135 - smp_processor_id(), new_head, hv_ret); 136 136 - 137 137 - for (off = q->head; off != new_head; off = spu_next_offset(q, off)) { 138 138 - /* XXX ... XXX */ 139 139 - } 140 140 - 141 141 - hv_ret = sun4v_ncs_sethead_marker(q->qhandle, new_head); 142 142 - if (hv_ret == HV_EOK) 143 143 - q->head = new_head; 144 144 - 145 145 - spin_unlock(&q->lock); 146 146 - 147 147 - return IRQ_HANDLED; 148 148 - } 149 149 - 150 150 - static irqreturn_t mau_intr(int irq, void *dev_id) 151 151 - { 152 152 - struct spu_queue *q = dev_id; 153 153 - unsigned long head, hv_ret; 154 154 - 155 155 - spin_lock(&q->lock); 156 156 - 157 157 - pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n", 158 158 - smp_processor_id(), q->qhandle); 159 159 - 160 160 - hv_ret = sun4v_ncs_gethead(q->qhandle, &head); 161 161 - 162 162 - pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n", 163 163 - smp_processor_id(), head, hv_ret); 164 164 - 165 165 - sun4v_ncs_sethead_marker(q->qhandle, head); 166 166 - 167 167 - spin_unlock(&q->lock); 168 168 - 169 169 - return IRQ_HANDLED; 170 170 - } 171 171 - 172 172 - static void *spu_queue_next(struct spu_queue *q, void *cur) 173 173 - { 174 174 - return q->q + spu_next_offset(q, cur - q->q); 175 175 - } 176 176 - 177 177 - static int spu_queue_num_free(struct spu_queue *q) 178 178 - { 179 179 - unsigned long head = q->head; 180 180 - unsigned long tail = q->tail; 181 181 - unsigned long end = (CWQ_ENTRY_SIZE * CWQ_NUM_ENTRIES); 182 182 - unsigned long diff; 183 183 - 184 184 - if (head > tail) 185 185 - diff = head - tail; 186 186 - else 187 187 - diff = (end - tail) + head; 188 188 - 189 189 - return (diff / CWQ_ENTRY_SIZE) - 1; 190 190 - } 191 191 - 192 192 - static void *spu_queue_alloc(struct spu_queue *q, int num_entries) 193 193 - { 194 194 - int avail = spu_queue_num_free(q); 195 195 - 196 196 - if (avail >= num_entries) 197 197 - return q->q + q->tail; 198 198 - 199 199 - return NULL; 200 200 - } 201 201 - 202 202 - static unsigned long spu_queue_submit(struct spu_queue *q, void *last) 203 203 - { 204 204 - unsigned long hv_ret, new_tail; 205 205 - 206 206 - new_tail = spu_next_offset(q, last - q->q); 207 207 - 208 208 - hv_ret = sun4v_ncs_settail(q->qhandle, new_tail); 209 209 - if (hv_ret == HV_EOK) 210 210 - q->tail = new_tail; 211 211 - return hv_ret; 212 212 - } 213 213 - 214 214 - static u64 control_word_base(unsigned int len, unsigned int hmac_key_len, 215 215 - int enc_type, int auth_type, 216 216 - unsigned int hash_len, 217 217 - bool sfas, bool sob, bool eob, bool encrypt, 218 218 - int opcode) 219 219 - { 220 220 - u64 word = (len - 1) & CONTROL_LEN; 221 221 - 222 222 - word |= ((u64) opcode << CONTROL_OPCODE_SHIFT); 223 223 - word |= ((u64) enc_type << CONTROL_ENC_TYPE_SHIFT); 224 224 - word |= ((u64) auth_type << CONTROL_AUTH_TYPE_SHIFT); 225 225 - if (sfas) 226 226 - word |= CONTROL_STORE_FINAL_AUTH_STATE; 227 227 - if (sob) 228 228 - word |= CONTROL_START_OF_BLOCK; 229 229 - if (eob) 230 230 - word |= CONTROL_END_OF_BLOCK; 231 231 - if (encrypt) 232 232 - word |= CONTROL_ENCRYPT; 233 233 - if (hmac_key_len) 234 234 - word |= ((u64) (hmac_key_len - 1)) << CONTROL_HMAC_KEY_LEN_SHIFT; 235 235 - if (hash_len) 236 236 - word |= ((u64) (hash_len - 1)) << CONTROL_HASH_LEN_SHIFT; 237 237 - 238 238 - return word; 239 239 - } 240 240 - 241 241 - #if 0 242 242 - static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) 243 243 - { 244 244 - if (this_len >= 64 || 245 245 - qp->head != qp->tail) 246 246 - return true; 247 247 - return false; 248 248 - } 249 249 - #endif 250 250 - 251 251 - struct n2_ahash_alg { 252 252 - struct list_head entry; 253 253 - const u8 *hash_zero; 254 254 - const u8 *hash_init; 255 255 - u8 hw_op_hashsz; 256 256 - u8 digest_size; 257 257 - u8 auth_type; 258 258 - u8 hmac_type; 259 259 - struct ahash_alg alg; 260 260 - }; 261 261 - 262 262 - static inline struct n2_ahash_alg *n2_ahash_alg(struct crypto_tfm *tfm) 263 263 - { 264 264 - struct crypto_alg *alg = tfm->__crt_alg; 265 265 - struct ahash_alg *ahash_alg; 266 266 - 267 267 - ahash_alg = container_of(alg, struct ahash_alg, halg.base); 268 268 - 269 269 - return container_of(ahash_alg, struct n2_ahash_alg, alg); 270 270 - } 271 271 - 272 272 - struct n2_hmac_alg { 273 273 - const char *child_alg; 274 274 - struct n2_ahash_alg derived; 275 275 - }; 276 276 - 277 277 - static inline struct n2_hmac_alg *n2_hmac_alg(struct crypto_tfm *tfm) 278 278 - { 279 279 - struct crypto_alg *alg = tfm->__crt_alg; 280 280 - struct ahash_alg *ahash_alg; 281 281 - 282 282 - ahash_alg = container_of(alg, struct ahash_alg, halg.base); 283 283 - 284 284 - return container_of(ahash_alg, struct n2_hmac_alg, derived.alg); 285 285 - } 286 286 - 287 287 - struct n2_hash_ctx { 288 288 - struct crypto_ahash *fallback_tfm; 289 289 - }; 290 290 - 291 291 - #define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */ 292 292 - 293 293 - struct n2_hmac_ctx { 294 294 - struct n2_hash_ctx base; 295 295 - 296 296 - struct crypto_shash *child_shash; 297 297 - 298 298 - int hash_key_len; 299 299 - unsigned char hash_key[N2_HASH_KEY_MAX]; 300 300 - }; 301 301 - 302 302 - struct n2_hash_req_ctx { 303 303 - union { 304 304 - struct md5_state md5; 305 305 - struct sha1_state sha1; 306 306 - struct sha256_state sha256; 307 307 - } u; 308 308 - 309 309 - struct ahash_request fallback_req; 310 310 - }; 311 311 - 312 312 - static int n2_hash_async_init(struct ahash_request *req) 313 313 - { 314 314 - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 315 315 - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 316 316 - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 317 317 - 318 318 - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 319 319 - rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 320 320 - 321 321 - return crypto_ahash_init(&rctx->fallback_req); 322 322 - } 323 323 - 324 324 - static int n2_hash_async_update(struct ahash_request *req) 325 325 - { 326 326 - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 327 327 - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 328 328 - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 329 329 - 330 330 - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 331 331 - rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 332 332 - rctx->fallback_req.nbytes = req->nbytes; 333 333 - rctx->fallback_req.src = req->src; 334 334 - 335 335 - return crypto_ahash_update(&rctx->fallback_req); 336 336 - } 337 337 - 338 338 - static int n2_hash_async_final(struct ahash_request *req) 339 339 - { 340 340 - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 341 341 - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 342 342 - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 343 343 - 344 344 - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 345 345 - rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 346 346 - rctx->fallback_req.result = req->result; 347 347 - 348 348 - return crypto_ahash_final(&rctx->fallback_req); 349 349 - } 350 350 - 351 351 - static int n2_hash_async_finup(struct ahash_request *req) 352 352 - { 353 353 - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 354 354 - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 355 355 - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 356 356 - 357 357 - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 358 358 - rctx->fallback_req.base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 359 359 - rctx->fallback_req.nbytes = req->nbytes; 360 360 - rctx->fallback_req.src = req->src; 361 361 - rctx->fallback_req.result = req->result; 362 362 - 363 363 - return crypto_ahash_finup(&rctx->fallback_req); 364 364 - } 365 365 - 366 366 - static int n2_hash_async_noimport(struct ahash_request *req, const void *in) 367 367 - { 368 368 - return -ENOSYS; 369 369 - } 370 370 - 371 371 - static int n2_hash_async_noexport(struct ahash_request *req, void *out) 372 372 - { 373 373 - return -ENOSYS; 374 374 - } 375 375 - 376 376 - static int n2_hash_cra_init(struct crypto_tfm *tfm) 377 377 - { 378 378 - const char *fallback_driver_name = crypto_tfm_alg_name(tfm); 379 379 - struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 380 380 - struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); 381 381 - struct crypto_ahash *fallback_tfm; 382 382 - int err; 383 383 - 384 384 - fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, 385 385 - CRYPTO_ALG_NEED_FALLBACK); 386 386 - if (IS_ERR(fallback_tfm)) { 387 387 - pr_warn("Fallback driver '%s' could not be loaded!\n", 388 388 - fallback_driver_name); 389 389 - err = PTR_ERR(fallback_tfm); 390 390 - goto out; 391 391 - } 392 392 - 393 393 - crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) + 394 394 - crypto_ahash_reqsize(fallback_tfm))); 395 395 - 396 396 - ctx->fallback_tfm = fallback_tfm; 397 397 - return 0; 398 398 - 399 399 - out: 400 400 - return err; 401 401 - } 402 402 - 403 403 - static void n2_hash_cra_exit(struct crypto_tfm *tfm) 404 404 - { 405 405 - struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 406 406 - struct n2_hash_ctx *ctx = crypto_ahash_ctx(ahash); 407 407 - 408 408 - crypto_free_ahash(ctx->fallback_tfm); 409 409 - } 410 410 - 411 411 - static int n2_hmac_cra_init(struct crypto_tfm *tfm) 412 412 - { 413 413 - const char *fallback_driver_name = crypto_tfm_alg_name(tfm); 414 414 - struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 415 415 - struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); 416 416 - struct n2_hmac_alg *n2alg = n2_hmac_alg(tfm); 417 417 - struct crypto_ahash *fallback_tfm; 418 418 - struct crypto_shash *child_shash; 419 419 - int err; 420 420 - 421 421 - fallback_tfm = crypto_alloc_ahash(fallback_driver_name, 0, 422 422 - CRYPTO_ALG_NEED_FALLBACK); 423 423 - if (IS_ERR(fallback_tfm)) { 424 424 - pr_warn("Fallback driver '%s' could not be loaded!\n", 425 425 - fallback_driver_name); 426 426 - err = PTR_ERR(fallback_tfm); 427 427 - goto out; 428 428 - } 429 429 - 430 430 - child_shash = crypto_alloc_shash(n2alg->child_alg, 0, 0); 431 431 - if (IS_ERR(child_shash)) { 432 432 - pr_warn("Child shash '%s' could not be loaded!\n", 433 433 - n2alg->child_alg); 434 434 - err = PTR_ERR(child_shash); 435 435 - goto out_free_fallback; 436 436 - } 437 437 - 438 438 - crypto_ahash_set_reqsize(ahash, (sizeof(struct n2_hash_req_ctx) + 439 439 - crypto_ahash_reqsize(fallback_tfm))); 440 440 - 441 441 - ctx->child_shash = child_shash; 442 442 - ctx->base.fallback_tfm = fallback_tfm; 443 443 - return 0; 444 444 - 445 445 - out_free_fallback: 446 446 - crypto_free_ahash(fallback_tfm); 447 447 - 448 448 - out: 449 449 - return err; 450 450 - } 451 451 - 452 452 - static void n2_hmac_cra_exit(struct crypto_tfm *tfm) 453 453 - { 454 454 - struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); 455 455 - struct n2_hmac_ctx *ctx = crypto_ahash_ctx(ahash); 456 456 - 457 457 - crypto_free_ahash(ctx->base.fallback_tfm); 458 458 - crypto_free_shash(ctx->child_shash); 459 459 - } 460 460 - 461 461 - static int n2_hmac_async_setkey(struct crypto_ahash *tfm, const u8 *key, 462 462 - unsigned int keylen) 463 463 - { 464 464 - struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); 465 465 - struct crypto_shash *child_shash = ctx->child_shash; 466 466 - struct crypto_ahash *fallback_tfm; 467 467 - int err, bs, ds; 468 468 - 469 469 - fallback_tfm = ctx->base.fallback_tfm; 470 470 - err = crypto_ahash_setkey(fallback_tfm, key, keylen); 471 471 - if (err) 472 472 - return err; 473 473 - 474 474 - bs = crypto_shash_blocksize(child_shash); 475 475 - ds = crypto_shash_digestsize(child_shash); 476 476 - BUG_ON(ds > N2_HASH_KEY_MAX); 477 477 - if (keylen > bs) { 478 478 - err = crypto_shash_tfm_digest(child_shash, key, keylen, 479 479 - ctx->hash_key); 480 480 - if (err) 481 481 - return err; 482 482 - keylen = ds; 483 483 - } else if (keylen <= N2_HASH_KEY_MAX) 484 484 - memcpy(ctx->hash_key, key, keylen); 485 485 - 486 486 - ctx->hash_key_len = keylen; 487 487 - 488 488 - return err; 489 489 - } 490 490 - 491 491 - static unsigned long wait_for_tail(struct spu_queue *qp) 492 492 - { 493 493 - unsigned long head, hv_ret; 494 494 - 495 495 - do { 496 496 - hv_ret = sun4v_ncs_gethead(qp->qhandle, &head); 497 497 - if (hv_ret != HV_EOK) { 498 498 - pr_err("Hypervisor error on gethead\n"); 499 499 - break; 500 500 - } 501 501 - if (head == qp->tail) { 502 502 - qp->head = head; 503 503 - break; 504 504 - } 505 505 - } while (1); 506 506 - return hv_ret; 507 507 - } 508 508 - 509 509 - static unsigned long submit_and_wait_for_tail(struct spu_queue *qp, 510 510 - struct cwq_initial_entry *ent) 511 511 - { 512 512 - unsigned long hv_ret = spu_queue_submit(qp, ent); 513 513 - 514 514 - if (hv_ret == HV_EOK) 515 515 - hv_ret = wait_for_tail(qp); 516 516 - 517 517 - return hv_ret; 518 518 - } 519 519 - 520 520 - static int n2_do_async_digest(struct ahash_request *req, 521 521 - unsigned int auth_type, unsigned int digest_size, 522 522 - unsigned int result_size, void *hash_loc, 523 523 - unsigned long auth_key, unsigned int auth_key_len) 524 524 - { 525 525 - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 526 526 - struct cwq_initial_entry *ent; 527 527 - struct crypto_hash_walk walk; 528 528 - struct spu_queue *qp; 529 529 - unsigned long flags; 530 530 - int err = -ENODEV; 531 531 - int nbytes, cpu; 532 532 - 533 533 - /* The total effective length of the operation may not 534 534 - * exceed 2^16. 535 535 - */ 536 536 - if (unlikely(req->nbytes > (1 << 16))) { 537 537 - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 538 538 - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 539 539 - 540 540 - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 541 541 - rctx->fallback_req.base.flags = 542 542 - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 543 543 - rctx->fallback_req.nbytes = req->nbytes; 544 544 - rctx->fallback_req.src = req->src; 545 545 - rctx->fallback_req.result = req->result; 546 546 - 547 547 - return crypto_ahash_digest(&rctx->fallback_req); 548 548 - } 549 549 - 550 550 - nbytes = crypto_hash_walk_first(req, &walk); 551 551 - 552 552 - cpu = get_cpu(); 553 553 - qp = cpu_to_cwq[cpu]; 554 554 - if (!qp) 555 555 - goto out; 556 556 - 557 557 - spin_lock_irqsave(&qp->lock, flags); 558 558 - 559 559 - /* XXX can do better, improve this later by doing a by-hand scatterlist 560 560 - * XXX walk, etc. 561 561 - */ 562 562 - ent = qp->q + qp->tail; 563 563 - 564 564 - ent->control = control_word_base(nbytes, auth_key_len, 0, 565 565 - auth_type, digest_size, 566 566 - false, true, false, false, 567 567 - OPCODE_INPLACE_BIT | 568 568 - OPCODE_AUTH_MAC); 569 569 - ent->src_addr = __pa(walk.data); 570 570 - ent->auth_key_addr = auth_key; 571 571 - ent->auth_iv_addr = __pa(hash_loc); 572 572 - ent->final_auth_state_addr = 0UL; 573 573 - ent->enc_key_addr = 0UL; 574 574 - ent->enc_iv_addr = 0UL; 575 575 - ent->dest_addr = __pa(hash_loc); 576 576 - 577 577 - nbytes = crypto_hash_walk_done(&walk, 0); 578 578 - while (nbytes > 0) { 579 579 - ent = spu_queue_next(qp, ent); 580 580 - 581 581 - ent->control = (nbytes - 1); 582 582 - ent->src_addr = __pa(walk.data); 583 583 - ent->auth_key_addr = 0UL; 584 584 - ent->auth_iv_addr = 0UL; 585 585 - ent->final_auth_state_addr = 0UL; 586 586 - ent->enc_key_addr = 0UL; 587 587 - ent->enc_iv_addr = 0UL; 588 588 - ent->dest_addr = 0UL; 589 589 - 590 590 - nbytes = crypto_hash_walk_done(&walk, 0); 591 591 - } 592 592 - ent->control |= CONTROL_END_OF_BLOCK; 593 593 - 594 594 - if (submit_and_wait_for_tail(qp, ent) != HV_EOK) 595 595 - err = -EINVAL; 596 596 - else 597 597 - err = 0; 598 598 - 599 599 - spin_unlock_irqrestore(&qp->lock, flags); 600 600 - 601 601 - if (!err) 602 602 - memcpy(req->result, hash_loc, result_size); 603 603 - out: 604 604 - put_cpu(); 605 605 - 606 606 - return err; 607 607 - } 608 608 - 609 609 - static int n2_hash_async_digest(struct ahash_request *req) 610 610 - { 611 611 - struct n2_ahash_alg *n2alg = n2_ahash_alg(req->base.tfm); 612 612 - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 613 613 - int ds; 614 614 - 615 615 - ds = n2alg->digest_size; 616 616 - if (unlikely(req->nbytes == 0)) { 617 617 - memcpy(req->result, n2alg->hash_zero, ds); 618 618 - return 0; 619 619 - } 620 620 - memcpy(&rctx->u, n2alg->hash_init, n2alg->hw_op_hashsz); 621 621 - 622 622 - return n2_do_async_digest(req, n2alg->auth_type, 623 623 - n2alg->hw_op_hashsz, ds, 624 624 - &rctx->u, 0UL, 0); 625 625 - } 626 626 - 627 627 - static int n2_hmac_async_digest(struct ahash_request *req) 628 628 - { 629 629 - struct n2_hmac_alg *n2alg = n2_hmac_alg(req->base.tfm); 630 630 - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 631 631 - struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 632 632 - struct n2_hmac_ctx *ctx = crypto_ahash_ctx(tfm); 633 633 - int ds; 634 634 - 635 635 - ds = n2alg->derived.digest_size; 636 636 - if (unlikely(req->nbytes == 0) || 637 637 - unlikely(ctx->hash_key_len > N2_HASH_KEY_MAX)) { 638 638 - struct n2_hash_req_ctx *rctx = ahash_request_ctx(req); 639 639 - struct n2_hash_ctx *ctx = crypto_ahash_ctx(tfm); 640 640 - 641 641 - ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm); 642 642 - rctx->fallback_req.base.flags = 643 643 - req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 644 644 - rctx->fallback_req.nbytes = req->nbytes; 645 645 - rctx->fallback_req.src = req->src; 646 646 - rctx->fallback_req.result = req->result; 647 647 - 648 648 - return crypto_ahash_digest(&rctx->fallback_req); 649 649 - } 650 650 - memcpy(&rctx->u, n2alg->derived.hash_init, 651 651 - n2alg->derived.hw_op_hashsz); 652 652 - 653 653 - return n2_do_async_digest(req, n2alg->derived.hmac_type, 654 654 - n2alg->derived.hw_op_hashsz, ds, 655 655 - &rctx->u, 656 656 - __pa(&ctx->hash_key), 657 657 - ctx->hash_key_len); 658 658 - } 659 659 - 660 660 - struct n2_skcipher_context { 661 661 - int key_len; 662 662 - int enc_type; 663 663 - union { 664 664 - u8 aes[AES_MAX_KEY_SIZE]; 665 665 - u8 des[DES_KEY_SIZE]; 666 666 - u8 des3[3 * DES_KEY_SIZE]; 667 667 - } key; 668 668 - }; 669 669 - 670 670 - #define N2_CHUNK_ARR_LEN 16 671 671 - 672 672 - struct n2_crypto_chunk { 673 673 - struct list_head entry; 674 674 - unsigned long iv_paddr : 44; 675 675 - unsigned long arr_len : 20; 676 676 - unsigned long dest_paddr; 677 677 - unsigned long dest_final; 678 678 - struct { 679 679 - unsigned long src_paddr : 44; 680 680 - unsigned long src_len : 20; 681 681 - } arr[N2_CHUNK_ARR_LEN]; 682 682 - }; 683 683 - 684 684 - struct n2_request_context { 685 685 - struct skcipher_walk walk; 686 686 - struct list_head chunk_list; 687 687 - struct n2_crypto_chunk chunk; 688 688 - u8 temp_iv[16]; 689 689 - }; 690 690 - 691 691 - /* The SPU allows some level of flexibility for partial cipher blocks 692 692 - * being specified in a descriptor. 693 693 - * 694 694 - * It merely requires that every descriptor's length field is at least 695 695 - * as large as the cipher block size. This means that a cipher block 696 696 - * can span at most 2 descriptors. However, this does not allow a 697 697 - * partial block to span into the final descriptor as that would 698 698 - * violate the rule (since every descriptor's length must be at lest 699 699 - * the block size). So, for example, assuming an 8 byte block size: 700 700 - * 701 701 - * 0xe --> 0xa --> 0x8 702 702 - * 703 703 - * is a valid length sequence, whereas: 704 704 - * 705 705 - * 0xe --> 0xb --> 0x7 706 706 - * 707 707 - * is not a valid sequence. 708 708 - */ 709 709 - 710 710 - struct n2_skcipher_alg { 711 711 - struct list_head entry; 712 712 - u8 enc_type; 713 713 - struct skcipher_alg skcipher; 714 714 - }; 715 715 - 716 716 - static inline struct n2_skcipher_alg *n2_skcipher_alg(struct crypto_skcipher *tfm) 717 717 - { 718 718 - struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 719 719 - 720 720 - return container_of(alg, struct n2_skcipher_alg, skcipher); 721 721 - } 722 722 - 723 723 - static int n2_aes_setkey(struct crypto_skcipher *skcipher, const u8 *key, 724 724 - unsigned int keylen) 725 725 - { 726 726 - struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); 727 727 - struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); 728 728 - struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher); 729 729 - 730 730 - ctx->enc_type = (n2alg->enc_type & ENC_TYPE_CHAINING_MASK); 731 731 - 732 732 - switch (keylen) { 733 733 - case AES_KEYSIZE_128: 734 734 - ctx->enc_type |= ENC_TYPE_ALG_AES128; 735 735 - break; 736 736 - case AES_KEYSIZE_192: 737 737 - ctx->enc_type |= ENC_TYPE_ALG_AES192; 738 738 - break; 739 739 - case AES_KEYSIZE_256: 740 740 - ctx->enc_type |= ENC_TYPE_ALG_AES256; 741 741 - break; 742 742 - default: 743 743 - return -EINVAL; 744 744 - } 745 745 - 746 746 - ctx->key_len = keylen; 747 747 - memcpy(ctx->key.aes, key, keylen); 748 748 - return 0; 749 749 - } 750 750 - 751 751 - static int n2_des_setkey(struct crypto_skcipher *skcipher, const u8 *key, 752 752 - unsigned int keylen) 753 753 - { 754 754 - struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); 755 755 - struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); 756 756 - struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher); 757 757 - int err; 758 758 - 759 759 - err = verify_skcipher_des_key(skcipher, key); 760 760 - if (err) 761 761 - return err; 762 762 - 763 763 - ctx->enc_type = n2alg->enc_type; 764 764 - 765 765 - ctx->key_len = keylen; 766 766 - memcpy(ctx->key.des, key, keylen); 767 767 - return 0; 768 768 - } 769 769 - 770 770 - static int n2_3des_setkey(struct crypto_skcipher *skcipher, const u8 *key, 771 771 - unsigned int keylen) 772 772 - { 773 773 - struct crypto_tfm *tfm = crypto_skcipher_tfm(skcipher); 774 774 - struct n2_skcipher_context *ctx = crypto_tfm_ctx(tfm); 775 775 - struct n2_skcipher_alg *n2alg = n2_skcipher_alg(skcipher); 776 776 - int err; 777 777 - 778 778 - err = verify_skcipher_des3_key(skcipher, key); 779 779 - if (err) 780 780 - return err; 781 781 - 782 782 - ctx->enc_type = n2alg->enc_type; 783 783 - 784 784 - ctx->key_len = keylen; 785 785 - memcpy(ctx->key.des3, key, keylen); 786 786 - return 0; 787 787 - } 788 788 - 789 789 - static inline int skcipher_descriptor_len(int nbytes, unsigned int block_size) 790 790 - { 791 791 - int this_len = nbytes; 792 792 - 793 793 - this_len -= (nbytes & (block_size - 1)); 794 794 - return this_len > (1 << 16) ? (1 << 16) : this_len; 795 795 - } 796 796 - 797 797 - static int __n2_crypt_chunk(struct crypto_skcipher *skcipher, 798 798 - struct n2_crypto_chunk *cp, 799 799 - struct spu_queue *qp, bool encrypt) 800 800 - { 801 801 - struct n2_skcipher_context *ctx = crypto_skcipher_ctx(skcipher); 802 802 - struct cwq_initial_entry *ent; 803 803 - bool in_place; 804 804 - int i; 805 805 - 806 806 - ent = spu_queue_alloc(qp, cp->arr_len); 807 807 - if (!ent) { 808 808 - pr_info("queue_alloc() of %d fails\n", 809 809 - cp->arr_len); 810 810 - return -EBUSY; 811 811 - } 812 812 - 813 813 - in_place = (cp->dest_paddr == cp->arr[0].src_paddr); 814 814 - 815 815 - ent->control = control_word_base(cp->arr[0].src_len, 816 816 - 0, ctx->enc_type, 0, 0, 817 817 - false, true, false, encrypt, 818 818 - OPCODE_ENCRYPT | 819 819 - (in_place ? OPCODE_INPLACE_BIT : 0)); 820 820 - ent->src_addr = cp->arr[0].src_paddr; 821 821 - ent->auth_key_addr = 0UL; 822 822 - ent->auth_iv_addr = 0UL; 823 823 - ent->final_auth_state_addr = 0UL; 824 824 - ent->enc_key_addr = __pa(&ctx->key); 825 825 - ent->enc_iv_addr = cp->iv_paddr; 826 826 - ent->dest_addr = (in_place ? 0UL : cp->dest_paddr); 827 827 - 828 828 - for (i = 1; i < cp->arr_len; i++) { 829 829 - ent = spu_queue_next(qp, ent); 830 830 - 831 831 - ent->control = cp->arr[i].src_len - 1; 832 832 - ent->src_addr = cp->arr[i].src_paddr; 833 833 - ent->auth_key_addr = 0UL; 834 834 - ent->auth_iv_addr = 0UL; 835 835 - ent->final_auth_state_addr = 0UL; 836 836 - ent->enc_key_addr = 0UL; 837 837 - ent->enc_iv_addr = 0UL; 838 838 - ent->dest_addr = 0UL; 839 839 - } 840 840 - ent->control |= CONTROL_END_OF_BLOCK; 841 841 - 842 842 - return (spu_queue_submit(qp, ent) != HV_EOK) ? -EINVAL : 0; 843 843 - } 844 844 - 845 845 - static int n2_compute_chunks(struct skcipher_request *req) 846 846 - { 847 847 - struct n2_request_context *rctx = skcipher_request_ctx(req); 848 848 - struct skcipher_walk *walk = &rctx->walk; 849 849 - struct n2_crypto_chunk *chunk; 850 850 - unsigned long dest_prev; 851 851 - unsigned int tot_len; 852 852 - bool prev_in_place; 853 853 - int err, nbytes; 854 854 - 855 855 - err = skcipher_walk_async(walk, req); 856 856 - if (err) 857 857 - return err; 858 858 - 859 859 - INIT_LIST_HEAD(&rctx->chunk_list); 860 860 - 861 861 - chunk = &rctx->chunk; 862 862 - INIT_LIST_HEAD(&chunk->entry); 863 863 - 864 864 - chunk->iv_paddr = 0UL; 865 865 - chunk->arr_len = 0; 866 866 - chunk->dest_paddr = 0UL; 867 867 - 868 868 - prev_in_place = false; 869 869 - dest_prev = ~0UL; 870 870 - tot_len = 0; 871 871 - 872 872 - while ((nbytes = walk->nbytes) != 0) { 873 873 - unsigned long dest_paddr, src_paddr; 874 874 - bool in_place; 875 875 - int this_len; 876 876 - 877 877 - src_paddr = (page_to_phys(walk->src.phys.page) + 878 878 - walk->src.phys.offset); 879 879 - dest_paddr = (page_to_phys(walk->dst.phys.page) + 880 880 - walk->dst.phys.offset); 881 881 - in_place = (src_paddr == dest_paddr); 882 882 - this_len = skcipher_descriptor_len(nbytes, walk->blocksize); 883 883 - 884 884 - if (chunk->arr_len != 0) { 885 885 - if (in_place != prev_in_place || 886 886 - (!prev_in_place && 887 887 - dest_paddr != dest_prev) || 888 888 - chunk->arr_len == N2_CHUNK_ARR_LEN || 889 889 - tot_len + this_len > (1 << 16)) { 890 890 - chunk->dest_final = dest_prev; 891 891 - list_add_tail(&chunk->entry, 892 892 - &rctx->chunk_list); 893 893 - chunk = kzalloc(sizeof(*chunk), GFP_ATOMIC); 894 894 - if (!chunk) { 895 895 - err = -ENOMEM; 896 896 - break; 897 897 - } 898 898 - INIT_LIST_HEAD(&chunk->entry); 899 899 - } 900 900 - } 901 901 - if (chunk->arr_len == 0) { 902 902 - chunk->dest_paddr = dest_paddr; 903 903 - tot_len = 0; 904 904 - } 905 905 - chunk->arr[chunk->arr_len].src_paddr = src_paddr; 906 906 - chunk->arr[chunk->arr_len].src_len = this_len; 907 907 - chunk->arr_len++; 908 908 - 909 909 - dest_prev = dest_paddr + this_len; 910 910 - prev_in_place = in_place; 911 911 - tot_len += this_len; 912 912 - 913 913 - err = skcipher_walk_done(walk, nbytes - this_len); 914 914 - if (err) 915 915 - break; 916 916 - } 917 917 - if (!err && chunk->arr_len != 0) { 918 918 - chunk->dest_final = dest_prev; 919 919 - list_add_tail(&chunk->entry, &rctx->chunk_list); 920 920 - } 921 921 - 922 922 - return err; 923 923 - } 924 924 - 925 925 - static void n2_chunk_complete(struct skcipher_request *req, void *final_iv) 926 926 - { 927 927 - struct n2_request_context *rctx = skcipher_request_ctx(req); 928 928 - struct n2_crypto_chunk *c, *tmp; 929 929 - 930 930 - if (final_iv) 931 931 - memcpy(rctx->walk.iv, final_iv, rctx->walk.blocksize); 932 932 - 933 933 - list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { 934 934 - list_del(&c->entry); 935 935 - if (unlikely(c != &rctx->chunk)) 936 936 - kfree(c); 937 937 - } 938 938 - 939 939 - } 940 940 - 941 941 - static int n2_do_ecb(struct skcipher_request *req, bool encrypt) 942 942 - { 943 943 - struct n2_request_context *rctx = skcipher_request_ctx(req); 944 944 - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 945 945 - int err = n2_compute_chunks(req); 946 946 - struct n2_crypto_chunk *c, *tmp; 947 947 - unsigned long flags, hv_ret; 948 948 - struct spu_queue *qp; 949 949 - 950 950 - if (err) 951 951 - return err; 952 952 - 953 953 - qp = cpu_to_cwq[get_cpu()]; 954 954 - err = -ENODEV; 955 955 - if (!qp) 956 956 - goto out; 957 957 - 958 958 - spin_lock_irqsave(&qp->lock, flags); 959 959 - 960 960 - list_for_each_entry_safe(c, tmp, &rctx->chunk_list, entry) { 961 961 - err = __n2_crypt_chunk(tfm, c, qp, encrypt); 962 962 - if (err) 963 963 - break; 964 964 - list_del(&c->entry); 965 965 - if (unlikely(c != &rctx->chunk)) 966 966 - kfree(c); 967 967 - } 968 968 - if (!err) { 969 969 - hv_ret = wait_for_tail(qp); 970 970 - if (hv_ret != HV_EOK) 971 971 - err = -EINVAL; 972 972 - } 973 973 - 974 974 - spin_unlock_irqrestore(&qp->lock, flags); 975 975 - 976 976 - out: 977 977 - put_cpu(); 978 978 - 979 979 - n2_chunk_complete(req, NULL); 980 980 - return err; 981 981 - } 982 982 - 983 983 - static int n2_encrypt_ecb(struct skcipher_request *req) 984 984 - { 985 985 - return n2_do_ecb(req, true); 986 986 - } 987 987 - 988 988 - static int n2_decrypt_ecb(struct skcipher_request *req) 989 989 - { 990 990 - return n2_do_ecb(req, false); 991 991 - } 992 992 - 993 993 - static int n2_do_chaining(struct skcipher_request *req, bool encrypt) 994 994 - { 995 995 - struct n2_request_context *rctx = skcipher_request_ctx(req); 996 996 - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); 997 997 - unsigned long flags, hv_ret, iv_paddr; 998 998 - int err = n2_compute_chunks(req); 999 999 - struct n2_crypto_chunk *c, *tmp; 1000 1000 - struct spu_queue *qp; 1001 1001 - void *final_iv_addr; 1002 1002 - 1003 1003 - final_iv_addr = NULL; 1004 1004 - 1005 1005 - if (err) 1006 1006 - return err; 1007 1007 - 1008 1008 - qp = cpu_to_cwq[get_cpu()]; 1009 1009 - err = -ENODEV; 1010 1010 - if (!qp) 1011 1011 - goto out; 1012 1012 - 1013 1013 - spin_lock_irqsave(&qp->lock, flags); 1014 1014 - 1015 1015 - if (encrypt) { 1016 1016 - iv_paddr = __pa(rctx->walk.iv); 1017 1017 - list_for_each_entry_safe(c, tmp, &rctx->chunk_list, 1018 1018 - entry) { 1019 1019 - c->iv_paddr = iv_paddr; 1020 1020 - err = __n2_crypt_chunk(tfm, c, qp, true); 1021 1021 - if (err) 1022 1022 - break; 1023 1023 - iv_paddr = c->dest_final - rctx->walk.blocksize; 1024 1024 - list_del(&c->entry); 1025 1025 - if (unlikely(c != &rctx->chunk)) 1026 1026 - kfree(c); 1027 1027 - } 1028 1028 - final_iv_addr = __va(iv_paddr); 1029 1029 - } else { 1030 1030 - list_for_each_entry_safe_reverse(c, tmp, &rctx->chunk_list, 1031 1031 - entry) { 1032 1032 - if (c == &rctx->chunk) { 1033 1033 - iv_paddr = __pa(rctx->walk.iv); 1034 1034 - } else { 1035 1035 - iv_paddr = (tmp->arr[tmp->arr_len-1].src_paddr + 1036 1036 - tmp->arr[tmp->arr_len-1].src_len - 1037 1037 - rctx->walk.blocksize); 1038 1038 - } 1039 1039 - if (!final_iv_addr) { 1040 1040 - unsigned long pa; 1041 1041 - 1042 1042 - pa = (c->arr[c->arr_len-1].src_paddr + 1043 1043 - c->arr[c->arr_len-1].src_len - 1044 1044 - rctx->walk.blocksize); 1045 1045 - final_iv_addr = rctx->temp_iv; 1046 1046 - memcpy(rctx->temp_iv, __va(pa), 1047 1047 - rctx->walk.blocksize); 1048 1048 - } 1049 1049 - c->iv_paddr = iv_paddr; 1050 1050 - err = __n2_crypt_chunk(tfm, c, qp, false); 1051 1051 - if (err) 1052 1052 - break; 1053 1053 - list_del(&c->entry); 1054 1054 - if (unlikely(c != &rctx->chunk)) 1055 1055 - kfree(c); 1056 1056 - } 1057 1057 - } 1058 1058 - if (!err) { 1059 1059 - hv_ret = wait_for_tail(qp); 1060 1060 - if (hv_ret != HV_EOK) 1061 1061 - err = -EINVAL; 1062 1062 - } 1063 1063 - 1064 1064 - spin_unlock_irqrestore(&qp->lock, flags); 1065 1065 - 1066 1066 - out: 1067 1067 - put_cpu(); 1068 1068 - 1069 1069 - n2_chunk_complete(req, err ? NULL : final_iv_addr); 1070 1070 - return err; 1071 1071 - } 1072 1072 - 1073 1073 - static int n2_encrypt_chaining(struct skcipher_request *req) 1074 1074 - { 1075 1075 - return n2_do_chaining(req, true); 1076 1076 - } 1077 1077 - 1078 1078 - static int n2_decrypt_chaining(struct skcipher_request *req) 1079 1079 - { 1080 1080 - return n2_do_chaining(req, false); 1081 1081 - } 1082 1082 - 1083 1083 - struct n2_skcipher_tmpl { 1084 1084 - const char *name; 1085 1085 - const char *drv_name; 1086 1086 - u8 block_size; 1087 1087 - u8 enc_type; 1088 1088 - struct skcipher_alg skcipher; 1089 1089 - }; 1090 1090 - 1091 1091 - static const struct n2_skcipher_tmpl skcipher_tmpls[] = { 1092 1092 - /* DES: ECB CBC and CFB are supported */ 1093 1093 - { .name = "ecb(des)", 1094 1094 - .drv_name = "ecb-des", 1095 1095 - .block_size = DES_BLOCK_SIZE, 1096 1096 - .enc_type = (ENC_TYPE_ALG_DES | 1097 1097 - ENC_TYPE_CHAINING_ECB), 1098 1098 - .skcipher = { 1099 1099 - .min_keysize = DES_KEY_SIZE, 1100 1100 - .max_keysize = DES_KEY_SIZE, 1101 1101 - .setkey = n2_des_setkey, 1102 1102 - .encrypt = n2_encrypt_ecb, 1103 1103 - .decrypt = n2_decrypt_ecb, 1104 1104 - }, 1105 1105 - }, 1106 1106 - { .name = "cbc(des)", 1107 1107 - .drv_name = "cbc-des", 1108 1108 - .block_size = DES_BLOCK_SIZE, 1109 1109 - .enc_type = (ENC_TYPE_ALG_DES | 1110 1110 - ENC_TYPE_CHAINING_CBC), 1111 1111 - .skcipher = { 1112 1112 - .ivsize = DES_BLOCK_SIZE, 1113 1113 - .min_keysize = DES_KEY_SIZE, 1114 1114 - .max_keysize = DES_KEY_SIZE, 1115 1115 - .setkey = n2_des_setkey, 1116 1116 - .encrypt = n2_encrypt_chaining, 1117 1117 - .decrypt = n2_decrypt_chaining, 1118 1118 - }, 1119 1119 - }, 1120 1120 - 1121 1121 - /* 3DES: ECB CBC and CFB are supported */ 1122 1122 - { .name = "ecb(des3_ede)", 1123 1123 - .drv_name = "ecb-3des", 1124 1124 - .block_size = DES_BLOCK_SIZE, 1125 1125 - .enc_type = (ENC_TYPE_ALG_3DES | 1126 1126 - ENC_TYPE_CHAINING_ECB), 1127 1127 - .skcipher = { 1128 1128 - .min_keysize = 3 * DES_KEY_SIZE, 1129 1129 - .max_keysize = 3 * DES_KEY_SIZE, 1130 1130 - .setkey = n2_3des_setkey, 1131 1131 - .encrypt = n2_encrypt_ecb, 1132 1132 - .decrypt = n2_decrypt_ecb, 1133 1133 - }, 1134 1134 - }, 1135 1135 - { .name = "cbc(des3_ede)", 1136 1136 - .drv_name = "cbc-3des", 1137 1137 - .block_size = DES_BLOCK_SIZE, 1138 1138 - .enc_type = (ENC_TYPE_ALG_3DES | 1139 1139 - ENC_TYPE_CHAINING_CBC), 1140 1140 - .skcipher = { 1141 1141 - .ivsize = DES_BLOCK_SIZE, 1142 1142 - .min_keysize = 3 * DES_KEY_SIZE, 1143 1143 - .max_keysize = 3 * DES_KEY_SIZE, 1144 1144 - .setkey = n2_3des_setkey, 1145 1145 - .encrypt = n2_encrypt_chaining, 1146 1146 - .decrypt = n2_decrypt_chaining, 1147 1147 - }, 1148 1148 - }, 1149 1149 - 1150 1150 - /* AES: ECB CBC and CTR are supported */ 1151 1151 - { .name = "ecb(aes)", 1152 1152 - .drv_name = "ecb-aes", 1153 1153 - .block_size = AES_BLOCK_SIZE, 1154 1154 - .enc_type = (ENC_TYPE_ALG_AES128 | 1155 1155 - ENC_TYPE_CHAINING_ECB), 1156 1156 - .skcipher = { 1157 1157 - .min_keysize = AES_MIN_KEY_SIZE, 1158 1158 - .max_keysize = AES_MAX_KEY_SIZE, 1159 1159 - .setkey = n2_aes_setkey, 1160 1160 - .encrypt = n2_encrypt_ecb, 1161 1161 - .decrypt = n2_decrypt_ecb, 1162 1162 - }, 1163 1163 - }, 1164 1164 - { .name = "cbc(aes)", 1165 1165 - .drv_name = "cbc-aes", 1166 1166 - .block_size = AES_BLOCK_SIZE, 1167 1167 - .enc_type = (ENC_TYPE_ALG_AES128 | 1168 1168 - ENC_TYPE_CHAINING_CBC), 1169 1169 - .skcipher = { 1170 1170 - .ivsize = AES_BLOCK_SIZE, 1171 1171 - .min_keysize = AES_MIN_KEY_SIZE, 1172 1172 - .max_keysize = AES_MAX_KEY_SIZE, 1173 1173 - .setkey = n2_aes_setkey, 1174 1174 - .encrypt = n2_encrypt_chaining, 1175 1175 - .decrypt = n2_decrypt_chaining, 1176 1176 - }, 1177 1177 - }, 1178 1178 - { .name = "ctr(aes)", 1179 1179 - .drv_name = "ctr-aes", 1180 1180 - .block_size = AES_BLOCK_SIZE, 1181 1181 - .enc_type = (ENC_TYPE_ALG_AES128 | 1182 1182 - ENC_TYPE_CHAINING_COUNTER), 1183 1183 - .skcipher = { 1184 1184 - .ivsize = AES_BLOCK_SIZE, 1185 1185 - .min_keysize = AES_MIN_KEY_SIZE, 1186 1186 - .max_keysize = AES_MAX_KEY_SIZE, 1187 1187 - .setkey = n2_aes_setkey, 1188 1188 - .encrypt = n2_encrypt_chaining, 1189 1189 - .decrypt = n2_encrypt_chaining, 1190 1190 - }, 1191 1191 - }, 1192 1192 - 1193 1193 - }; 1194 1194 - #define NUM_CIPHER_TMPLS ARRAY_SIZE(skcipher_tmpls) 1195 1195 - 1196 1196 - static LIST_HEAD(skcipher_algs); 1197 1197 - 1198 1198 - struct n2_hash_tmpl { 1199 1199 - const char *name; 1200 1200 - const u8 *hash_zero; 1201 1201 - const u8 *hash_init; 1202 1202 - u8 hw_op_hashsz; 1203 1203 - u8 digest_size; 1204 1204 - u8 statesize; 1205 1205 - u8 block_size; 1206 1206 - u8 auth_type; 1207 1207 - u8 hmac_type; 1208 1208 - }; 1209 1209 - 1210 1210 - static const __le32 n2_md5_init[MD5_HASH_WORDS] = { 1211 1211 - cpu_to_le32(MD5_H0), 1212 1212 - cpu_to_le32(MD5_H1), 1213 1213 - cpu_to_le32(MD5_H2), 1214 1214 - cpu_to_le32(MD5_H3), 1215 1215 - }; 1216 1216 - static const u32 n2_sha1_init[SHA1_DIGEST_SIZE / 4] = { 1217 1217 - SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 1218 1218 - }; 1219 1219 - static const u32 n2_sha256_init[SHA256_DIGEST_SIZE / 4] = { 1220 1220 - SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, 1221 1221 - SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, 1222 1222 - }; 1223 1223 - static const u32 n2_sha224_init[SHA256_DIGEST_SIZE / 4] = { 1224 1224 - SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, 1225 1225 - SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, 1226 1226 - }; 1227 1227 - 1228 1228 - static const struct n2_hash_tmpl hash_tmpls[] = { 1229 1229 - { .name = "md5", 1230 1230 - .hash_zero = md5_zero_message_hash, 1231 1231 - .hash_init = (u8 *)n2_md5_init, 1232 1232 - .auth_type = AUTH_TYPE_MD5, 1233 1233 - .hmac_type = AUTH_TYPE_HMAC_MD5, 1234 1234 - .hw_op_hashsz = MD5_DIGEST_SIZE, 1235 1235 - .digest_size = MD5_DIGEST_SIZE, 1236 1236 - .statesize = sizeof(struct md5_state), 1237 1237 - .block_size = MD5_HMAC_BLOCK_SIZE }, 1238 1238 - { .name = "sha1", 1239 1239 - .hash_zero = sha1_zero_message_hash, 1240 1240 - .hash_init = (u8 *)n2_sha1_init, 1241 1241 - .auth_type = AUTH_TYPE_SHA1, 1242 1242 - .hmac_type = AUTH_TYPE_HMAC_SHA1, 1243 1243 - .hw_op_hashsz = SHA1_DIGEST_SIZE, 1244 1244 - .digest_size = SHA1_DIGEST_SIZE, 1245 1245 - .statesize = sizeof(struct sha1_state), 1246 1246 - .block_size = SHA1_BLOCK_SIZE }, 1247 1247 - { .name = "sha256", 1248 1248 - .hash_zero = sha256_zero_message_hash, 1249 1249 - .hash_init = (u8 *)n2_sha256_init, 1250 1250 - .auth_type = AUTH_TYPE_SHA256, 1251 1251 - .hmac_type = AUTH_TYPE_HMAC_SHA256, 1252 1252 - .hw_op_hashsz = SHA256_DIGEST_SIZE, 1253 1253 - .digest_size = SHA256_DIGEST_SIZE, 1254 1254 - .statesize = sizeof(struct sha256_state), 1255 1255 - .block_size = SHA256_BLOCK_SIZE }, 1256 1256 - { .name = "sha224", 1257 1257 - .hash_zero = sha224_zero_message_hash, 1258 1258 - .hash_init = (u8 *)n2_sha224_init, 1259 1259 - .auth_type = AUTH_TYPE_SHA256, 1260 1260 - .hmac_type = AUTH_TYPE_RESERVED, 1261 1261 - .hw_op_hashsz = SHA256_DIGEST_SIZE, 1262 1262 - .digest_size = SHA224_DIGEST_SIZE, 1263 1263 - .statesize = sizeof(struct sha256_state), 1264 1264 - .block_size = SHA224_BLOCK_SIZE }, 1265 1265 - }; 1266 1266 - #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls) 1267 1267 - 1268 1268 - static LIST_HEAD(ahash_algs); 1269 1269 - static LIST_HEAD(hmac_algs); 1270 1270 - 1271 1271 - static int algs_registered; 1272 1272 - 1273 1273 - static void __n2_unregister_algs(void) 1274 1274 - { 1275 1275 - struct n2_skcipher_alg *skcipher, *skcipher_tmp; 1276 1276 - struct n2_ahash_alg *alg, *alg_tmp; 1277 1277 - struct n2_hmac_alg *hmac, *hmac_tmp; 1278 1278 - 1279 1279 - list_for_each_entry_safe(skcipher, skcipher_tmp, &skcipher_algs, entry) { 1280 1280 - crypto_unregister_skcipher(&skcipher->skcipher); 1281 1281 - list_del(&skcipher->entry); 1282 1282 - kfree(skcipher); 1283 1283 - } 1284 1284 - list_for_each_entry_safe(hmac, hmac_tmp, &hmac_algs, derived.entry) { 1285 1285 - crypto_unregister_ahash(&hmac->derived.alg); 1286 1286 - list_del(&hmac->derived.entry); 1287 1287 - kfree(hmac); 1288 1288 - } 1289 1289 - list_for_each_entry_safe(alg, alg_tmp, &ahash_algs, entry) { 1290 1290 - crypto_unregister_ahash(&alg->alg); 1291 1291 - list_del(&alg->entry); 1292 1292 - kfree(alg); 1293 1293 - } 1294 1294 - } 1295 1295 - 1296 1296 - static int n2_skcipher_init_tfm(struct crypto_skcipher *tfm) 1297 1297 - { 1298 1298 - crypto_skcipher_set_reqsize(tfm, sizeof(struct n2_request_context)); 1299 1299 - return 0; 1300 1300 - } 1301 1301 - 1302 1302 - static int __n2_register_one_skcipher(const struct n2_skcipher_tmpl *tmpl) 1303 1303 - { 1304 1304 - struct n2_skcipher_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); 1305 1305 - struct skcipher_alg *alg; 1306 1306 - int err; 1307 1307 - 1308 1308 - if (!p) 1309 1309 - return -ENOMEM; 1310 1310 - 1311 1311 - alg = &p->skcipher; 1312 1312 - *alg = tmpl->skcipher; 1313 1313 - 1314 1314 - snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 1315 1315 - snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->drv_name); 1316 1316 - alg->base.cra_priority = N2_CRA_PRIORITY; 1317 1317 - alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | 1318 1318 - CRYPTO_ALG_ALLOCATES_MEMORY; 1319 1319 - alg->base.cra_blocksize = tmpl->block_size; 1320 1320 - p->enc_type = tmpl->enc_type; 1321 1321 - alg->base.cra_ctxsize = sizeof(struct n2_skcipher_context); 1322 1322 - alg->base.cra_module = THIS_MODULE; 1323 1323 - alg->init = n2_skcipher_init_tfm; 1324 1324 - 1325 1325 - list_add(&p->entry, &skcipher_algs); 1326 1326 - err = crypto_register_skcipher(alg); 1327 1327 - if (err) { 1328 1328 - pr_err("%s alg registration failed\n", alg->base.cra_name); 1329 1329 - list_del(&p->entry); 1330 1330 - kfree(p); 1331 1331 - } else { 1332 1332 - pr_info("%s alg registered\n", alg->base.cra_name); 1333 1333 - } 1334 1334 - return err; 1335 1335 - } 1336 1336 - 1337 1337 - static int __n2_register_one_hmac(struct n2_ahash_alg *n2ahash) 1338 1338 - { 1339 1339 - struct n2_hmac_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); 1340 1340 - struct ahash_alg *ahash; 1341 1341 - struct crypto_alg *base; 1342 1342 - int err; 1343 1343 - 1344 1344 - if (!p) 1345 1345 - return -ENOMEM; 1346 1346 - 1347 1347 - p->child_alg = n2ahash->alg.halg.base.cra_name; 1348 1348 - memcpy(&p->derived, n2ahash, sizeof(struct n2_ahash_alg)); 1349 1349 - INIT_LIST_HEAD(&p->derived.entry); 1350 1350 - 1351 1351 - ahash = &p->derived.alg; 1352 1352 - ahash->digest = n2_hmac_async_digest; 1353 1353 - ahash->setkey = n2_hmac_async_setkey; 1354 1354 - 1355 1355 - base = &ahash->halg.base; 1356 1356 - err = -EINVAL; 1357 1357 - if (snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "hmac(%s)", 1358 1358 - p->child_alg) >= CRYPTO_MAX_ALG_NAME) 1359 1359 - goto out_free_p; 1360 1360 - if (snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "hmac-%s-n2", 1361 1361 - p->child_alg) >= CRYPTO_MAX_ALG_NAME) 1362 1362 - goto out_free_p; 1363 1363 - 1364 1364 - base->cra_ctxsize = sizeof(struct n2_hmac_ctx); 1365 1365 - base->cra_init = n2_hmac_cra_init; 1366 1366 - base->cra_exit = n2_hmac_cra_exit; 1367 1367 - 1368 1368 - list_add(&p->derived.entry, &hmac_algs); 1369 1369 - err = crypto_register_ahash(ahash); 1370 1370 - if (err) { 1371 1371 - pr_err("%s alg registration failed\n", base->cra_name); 1372 1372 - list_del(&p->derived.entry); 1373 1373 - out_free_p: 1374 1374 - kfree(p); 1375 1375 - } else { 1376 1376 - pr_info("%s alg registered\n", base->cra_name); 1377 1377 - } 1378 1378 - return err; 1379 1379 - } 1380 1380 - 1381 1381 - static int __n2_register_one_ahash(const struct n2_hash_tmpl *tmpl) 1382 1382 - { 1383 1383 - struct n2_ahash_alg *p = kzalloc(sizeof(*p), GFP_KERNEL); 1384 1384 - struct hash_alg_common *halg; 1385 1385 - struct crypto_alg *base; 1386 1386 - struct ahash_alg *ahash; 1387 1387 - int err; 1388 1388 - 1389 1389 - if (!p) 1390 1390 - return -ENOMEM; 1391 1391 - 1392 1392 - p->hash_zero = tmpl->hash_zero; 1393 1393 - p->hash_init = tmpl->hash_init; 1394 1394 - p->auth_type = tmpl->auth_type; 1395 1395 - p->hmac_type = tmpl->hmac_type; 1396 1396 - p->hw_op_hashsz = tmpl->hw_op_hashsz; 1397 1397 - p->digest_size = tmpl->digest_size; 1398 1398 - 1399 1399 - ahash = &p->alg; 1400 1400 - ahash->init = n2_hash_async_init; 1401 1401 - ahash->update = n2_hash_async_update; 1402 1402 - ahash->final = n2_hash_async_final; 1403 1403 - ahash->finup = n2_hash_async_finup; 1404 1404 - ahash->digest = n2_hash_async_digest; 1405 1405 - ahash->export = n2_hash_async_noexport; 1406 1406 - ahash->import = n2_hash_async_noimport; 1407 1407 - 1408 1408 - halg = &ahash->halg; 1409 1409 - halg->digestsize = tmpl->digest_size; 1410 1410 - halg->statesize = tmpl->statesize; 1411 1411 - 1412 1412 - base = &halg->base; 1413 1413 - snprintf(base->cra_name, CRYPTO_MAX_ALG_NAME, "%s", tmpl->name); 1414 1414 - snprintf(base->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-n2", tmpl->name); 1415 1415 - base->cra_priority = N2_CRA_PRIORITY; 1416 1416 - base->cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | 1417 1417 - CRYPTO_ALG_NEED_FALLBACK; 1418 1418 - base->cra_blocksize = tmpl->block_size; 1419 1419 - base->cra_ctxsize = sizeof(struct n2_hash_ctx); 1420 1420 - base->cra_module = THIS_MODULE; 1421 1421 - base->cra_init = n2_hash_cra_init; 1422 1422 - base->cra_exit = n2_hash_cra_exit; 1423 1423 - 1424 1424 - list_add(&p->entry, &ahash_algs); 1425 1425 - err = crypto_register_ahash(ahash); 1426 1426 - if (err) { 1427 1427 - pr_err("%s alg registration failed\n", base->cra_name); 1428 1428 - list_del(&p->entry); 1429 1429 - kfree(p); 1430 1430 - } else { 1431 1431 - pr_info("%s alg registered\n", base->cra_name); 1432 1432 - } 1433 1433 - if (!err && p->hmac_type != AUTH_TYPE_RESERVED) 1434 1434 - err = __n2_register_one_hmac(p); 1435 1435 - return err; 1436 1436 - } 1437 1437 - 1438 1438 - static int n2_register_algs(void) 1439 1439 - { 1440 1440 - int i, err = 0; 1441 1441 - 1442 1442 - mutex_lock(&spu_lock); 1443 1443 - if (algs_registered++) 1444 1444 - goto out; 1445 1445 - 1446 1446 - for (i = 0; i < NUM_HASH_TMPLS; i++) { 1447 1447 - err = __n2_register_one_ahash(&hash_tmpls[i]); 1448 1448 - if (err) { 1449 1449 - __n2_unregister_algs(); 1450 1450 - goto out; 1451 1451 - } 1452 1452 - } 1453 1453 - for (i = 0; i < NUM_CIPHER_TMPLS; i++) { 1454 1454 - err = __n2_register_one_skcipher(&skcipher_tmpls[i]); 1455 1455 - if (err) { 1456 1456 - __n2_unregister_algs(); 1457 1457 - goto out; 1458 1458 - } 1459 1459 - } 1460 1460 - 1461 1461 - out: 1462 1462 - mutex_unlock(&spu_lock); 1463 1463 - return err; 1464 1464 - } 1465 1465 - 1466 1466 - static void n2_unregister_algs(void) 1467 1467 - { 1468 1468 - mutex_lock(&spu_lock); 1469 1469 - if (!--algs_registered) 1470 1470 - __n2_unregister_algs(); 1471 1471 - mutex_unlock(&spu_lock); 1472 1472 - } 1473 1473 - 1474 1474 - /* To map CWQ queues to interrupt sources, the hypervisor API provides 1475 1475 - * a devino. This isn't very useful to us because all of the 1476 1476 - * interrupts listed in the device_node have been translated to 1477 1477 - * Linux virtual IRQ cookie numbers. 1478 1478 - * 1479 1479 - * So we have to back-translate, going through the 'intr' and 'ino' 1480 1480 - * property tables of the n2cp MDESC node, matching it with the OF 1481 1481 - * 'interrupts' property entries, in order to figure out which 1482 1482 - * devino goes to which already-translated IRQ. 1483 1483 - */ 1484 1484 - static int find_devino_index(struct platform_device *dev, struct spu_mdesc_info *ip, 1485 1485 - unsigned long dev_ino) 1486 1486 - { 1487 1487 - const unsigned int *dev_intrs; 1488 1488 - unsigned int intr; 1489 1489 - int i; 1490 1490 - 1491 1491 - for (i = 0; i < ip->num_intrs; i++) { 1492 1492 - if (ip->ino_table[i].ino == dev_ino) 1493 1493 - break; 1494 1494 - } 1495 1495 - if (i == ip->num_intrs) 1496 1496 - return -ENODEV; 1497 1497 - 1498 1498 - intr = ip->ino_table[i].intr; 1499 1499 - 1500 1500 - dev_intrs = of_get_property(dev->dev.of_node, "interrupts", NULL); 1501 1501 - if (!dev_intrs) 1502 1502 - return -ENODEV; 1503 1503 - 1504 1504 - for (i = 0; i < dev->archdata.num_irqs; i++) { 1505 1505 - if (dev_intrs[i] == intr) 1506 1506 - return i; 1507 1507 - } 1508 1508 - 1509 1509 - return -ENODEV; 1510 1510 - } 1511 1511 - 1512 1512 - static int spu_map_ino(struct platform_device *dev, struct spu_mdesc_info *ip, 1513 1513 - const char *irq_name, struct spu_queue *p, 1514 1514 - irq_handler_t handler) 1515 1515 - { 1516 1516 - unsigned long herr; 1517 1517 - int index; 1518 1518 - 1519 1519 - herr = sun4v_ncs_qhandle_to_devino(p->qhandle, &p->devino); 1520 1520 - if (herr) 1521 1521 - return -EINVAL; 1522 1522 - 1523 1523 - index = find_devino_index(dev, ip, p->devino); 1524 1524 - if (index < 0) 1525 1525 - return index; 1526 1526 - 1527 1527 - p->irq = dev->archdata.irqs[index]; 1528 1528 - 1529 1529 - sprintf(p->irq_name, "%s-%d", irq_name, index); 1530 1530 - 1531 1531 - return request_irq(p->irq, handler, 0, p->irq_name, p); 1532 1532 - } 1533 1533 - 1534 1534 - static struct kmem_cache *queue_cache[2]; 1535 1535 - 1536 1536 - static void *new_queue(unsigned long q_type) 1537 1537 - { 1538 1538 - return kmem_cache_zalloc(queue_cache[q_type - 1], GFP_KERNEL); 1539 1539 - } 1540 1540 - 1541 1541 - static void free_queue(void *p, unsigned long q_type) 1542 1542 - { 1543 1543 - kmem_cache_free(queue_cache[q_type - 1], p); 1544 1544 - } 1545 1545 - 1546 1546 - static int queue_cache_init(void) 1547 1547 - { 1548 1548 - if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) 1549 1549 - queue_cache[HV_NCS_QTYPE_MAU - 1] = 1550 1550 - kmem_cache_create("mau_queue", 1551 1551 - (MAU_NUM_ENTRIES * 1552 1552 - MAU_ENTRY_SIZE), 1553 1553 - MAU_ENTRY_SIZE, 0, NULL); 1554 1554 - if (!queue_cache[HV_NCS_QTYPE_MAU - 1]) 1555 1555 - return -ENOMEM; 1556 1556 - 1557 1557 - if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) 1558 1558 - queue_cache[HV_NCS_QTYPE_CWQ - 1] = 1559 1559 - kmem_cache_create("cwq_queue", 1560 1560 - (CWQ_NUM_ENTRIES * 1561 1561 - CWQ_ENTRY_SIZE), 1562 1562 - CWQ_ENTRY_SIZE, 0, NULL); 1563 1563 - if (!queue_cache[HV_NCS_QTYPE_CWQ - 1]) { 1564 1564 - kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]); 1565 1565 - queue_cache[HV_NCS_QTYPE_MAU - 1] = NULL; 1566 1566 - return -ENOMEM; 1567 1567 - } 1568 1568 - return 0; 1569 1569 - } 1570 1570 - 1571 1571 - static void queue_cache_destroy(void) 1572 1572 - { 1573 1573 - kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_MAU - 1]); 1574 1574 - kmem_cache_destroy(queue_cache[HV_NCS_QTYPE_CWQ - 1]); 1575 1575 - queue_cache[HV_NCS_QTYPE_MAU - 1] = NULL; 1576 1576 - queue_cache[HV_NCS_QTYPE_CWQ - 1] = NULL; 1577 1577 - } 1578 1578 - 1579 1579 - static long spu_queue_register_workfn(void *arg) 1580 1580 - { 1581 1581 - struct spu_qreg *qr = arg; 1582 1582 - struct spu_queue *p = qr->queue; 1583 1583 - unsigned long q_type = qr->type; 1584 1584 - unsigned long hv_ret; 1585 1585 - 1586 1586 - hv_ret = sun4v_ncs_qconf(q_type, __pa(p->q), 1587 1587 - CWQ_NUM_ENTRIES, &p->qhandle); 1588 1588 - if (!hv_ret) 1589 1589 - sun4v_ncs_sethead_marker(p->qhandle, 0); 1590 1590 - 1591 1591 - return hv_ret ? -EINVAL : 0; 1592 1592 - } 1593 1593 - 1594 1594 - static int spu_queue_register(struct spu_queue *p, unsigned long q_type) 1595 1595 - { 1596 1596 - int cpu = cpumask_any_and(&p->sharing, cpu_online_mask); 1597 1597 - struct spu_qreg qr = { .queue = p, .type = q_type }; 1598 1598 - 1599 1599 - return work_on_cpu_safe(cpu, spu_queue_register_workfn, &qr); 1600 1600 - } 1601 1601 - 1602 1602 - static int spu_queue_setup(struct spu_queue *p) 1603 1603 - { 1604 1604 - int err; 1605 1605 - 1606 1606 - p->q = new_queue(p->q_type); 1607 1607 - if (!p->q) 1608 1608 - return -ENOMEM; 1609 1609 - 1610 1610 - err = spu_queue_register(p, p->q_type); 1611 1611 - if (err) { 1612 1612 - free_queue(p->q, p->q_type); 1613 1613 - p->q = NULL; 1614 1614 - } 1615 1615 - 1616 1616 - return err; 1617 1617 - } 1618 1618 - 1619 1619 - static void spu_queue_destroy(struct spu_queue *p) 1620 1620 - { 1621 1621 - unsigned long hv_ret; 1622 1622 - 1623 1623 - if (!p->q) 1624 1624 - return; 1625 1625 - 1626 1626 - hv_ret = sun4v_ncs_qconf(p->q_type, p->qhandle, 0, &p->qhandle); 1627 1627 - 1628 1628 - if (!hv_ret) 1629 1629 - free_queue(p->q, p->q_type); 1630 1630 - } 1631 1631 - 1632 1632 - static void spu_list_destroy(struct list_head *list) 1633 1633 - { 1634 1634 - struct spu_queue *p, *n; 1635 1635 - 1636 1636 - list_for_each_entry_safe(p, n, list, list) { 1637 1637 - int i; 1638 1638 - 1639 1639 - for (i = 0; i < NR_CPUS; i++) { 1640 1640 - if (cpu_to_cwq[i] == p) 1641 1641 - cpu_to_cwq[i] = NULL; 1642 1642 - } 1643 1643 - 1644 1644 - if (p->irq) { 1645 1645 - free_irq(p->irq, p); 1646 1646 - p->irq = 0; 1647 1647 - } 1648 1648 - spu_queue_destroy(p); 1649 1649 - list_del(&p->list); 1650 1650 - kfree(p); 1651 1651 - } 1652 1652 - } 1653 1653 - 1654 1654 - /* Walk the backward arcs of a CWQ 'exec-unit' node, 1655 1655 - * gathering cpu membership information. 1656 1656 - */ 1657 1657 - static int spu_mdesc_walk_arcs(struct mdesc_handle *mdesc, 1658 1658 - struct platform_device *dev, 1659 1659 - u64 node, struct spu_queue *p, 1660 1660 - struct spu_queue **table) 1661 1661 - { 1662 1662 - u64 arc; 1663 1663 - 1664 1664 - mdesc_for_each_arc(arc, mdesc, node, MDESC_ARC_TYPE_BACK) { 1665 1665 - u64 tgt = mdesc_arc_target(mdesc, arc); 1666 1666 - const char *name = mdesc_node_name(mdesc, tgt); 1667 1667 - const u64 *id; 1668 1668 - 1669 1669 - if (strcmp(name, "cpu")) 1670 1670 - continue; 1671 1671 - id = mdesc_get_property(mdesc, tgt, "id", NULL); 1672 1672 - if (table[*id] != NULL) { 1673 1673 - dev_err(&dev->dev, "%pOF: SPU cpu slot already set.\n", 1674 1674 - dev->dev.of_node); 1675 1675 - return -EINVAL; 1676 1676 - } 1677 1677 - cpumask_set_cpu(*id, &p->sharing); 1678 1678 - table[*id] = p; 1679 1679 - } 1680 1680 - return 0; 1681 1681 - } 1682 1682 - 1683 1683 - /* Process an 'exec-unit' MDESC node of type 'cwq'. */ 1684 1684 - static int handle_exec_unit(struct spu_mdesc_info *ip, struct list_head *list, 1685 1685 - struct platform_device *dev, struct mdesc_handle *mdesc, 1686 1686 - u64 node, const char *iname, unsigned long q_type, 1687 1687 - irq_handler_t handler, struct spu_queue **table) 1688 1688 - { 1689 1689 - struct spu_queue *p; 1690 1690 - int err; 1691 1691 - 1692 1692 - p = kzalloc(sizeof(struct spu_queue), GFP_KERNEL); 1693 1693 - if (!p) { 1694 1694 - dev_err(&dev->dev, "%pOF: Could not allocate SPU queue.\n", 1695 1695 - dev->dev.of_node); 1696 1696 - return -ENOMEM; 1697 1697 - } 1698 1698 - 1699 1699 - cpumask_clear(&p->sharing); 1700 1700 - spin_lock_init(&p->lock); 1701 1701 - p->q_type = q_type; 1702 1702 - INIT_LIST_HEAD(&p->jobs); 1703 1703 - list_add(&p->list, list); 1704 1704 - 1705 1705 - err = spu_mdesc_walk_arcs(mdesc, dev, node, p, table); 1706 1706 - if (err) 1707 1707 - return err; 1708 1708 - 1709 1709 - err = spu_queue_setup(p); 1710 1710 - if (err) 1711 1711 - return err; 1712 1712 - 1713 1713 - return spu_map_ino(dev, ip, iname, p, handler); 1714 1714 - } 1715 1715 - 1716 1716 - static int spu_mdesc_scan(struct mdesc_handle *mdesc, struct platform_device *dev, 1717 1717 - struct spu_mdesc_info *ip, struct list_head *list, 1718 1718 - const char *exec_name, unsigned long q_type, 1719 1719 - irq_handler_t handler, struct spu_queue **table) 1720 1720 - { 1721 1721 - int err = 0; 1722 1722 - u64 node; 1723 1723 - 1724 1724 - mdesc_for_each_node_by_name(mdesc, node, "exec-unit") { 1725 1725 - const char *type; 1726 1726 - 1727 1727 - type = mdesc_get_property(mdesc, node, "type", NULL); 1728 1728 - if (!type || strcmp(type, exec_name)) 1729 1729 - continue; 1730 1730 - 1731 1731 - err = handle_exec_unit(ip, list, dev, mdesc, node, 1732 1732 - exec_name, q_type, handler, table); 1733 1733 - if (err) { 1734 1734 - spu_list_destroy(list); 1735 1735 - break; 1736 1736 - } 1737 1737 - } 1738 1738 - 1739 1739 - return err; 1740 1740 - } 1741 1741 - 1742 1742 - static int get_irq_props(struct mdesc_handle *mdesc, u64 node, 1743 1743 - struct spu_mdesc_info *ip) 1744 1744 - { 1745 1745 - const u64 *ino; 1746 1746 - int ino_len; 1747 1747 - int i; 1748 1748 - 1749 1749 - ino = mdesc_get_property(mdesc, node, "ino", &ino_len); 1750 1750 - if (!ino) { 1751 1751 - printk("NO 'ino'\n"); 1752 1752 - return -ENODEV; 1753 1753 - } 1754 1754 - 1755 1755 - ip->num_intrs = ino_len / sizeof(u64); 1756 1756 - ip->ino_table = kzalloc((sizeof(struct ino_blob) * 1757 1757 - ip->num_intrs), 1758 1758 - GFP_KERNEL); 1759 1759 - if (!ip->ino_table) 1760 1760 - return -ENOMEM; 1761 1761 - 1762 1762 - for (i = 0; i < ip->num_intrs; i++) { 1763 1763 - struct ino_blob *b = &ip->ino_table[i]; 1764 1764 - b->intr = i + 1; 1765 1765 - b->ino = ino[i]; 1766 1766 - } 1767 1767 - 1768 1768 - return 0; 1769 1769 - } 1770 1770 - 1771 1771 - static int grab_mdesc_irq_props(struct mdesc_handle *mdesc, 1772 1772 - struct platform_device *dev, 1773 1773 - struct spu_mdesc_info *ip, 1774 1774 - const char *node_name) 1775 1775 - { 1776 1776 - u64 node, reg; 1777 1777 - 1778 1778 - if (of_property_read_reg(dev->dev.of_node, 0, &reg, NULL) < 0) 1779 1779 - return -ENODEV; 1780 1780 - 1781 1781 - mdesc_for_each_node_by_name(mdesc, node, "virtual-device") { 1782 1782 - const char *name; 1783 1783 - const u64 *chdl; 1784 1784 - 1785 1785 - name = mdesc_get_property(mdesc, node, "name", NULL); 1786 1786 - if (!name || strcmp(name, node_name)) 1787 1787 - continue; 1788 1788 - chdl = mdesc_get_property(mdesc, node, "cfg-handle", NULL); 1789 1789 - if (!chdl || (*chdl != reg)) 1790 1790 - continue; 1791 1791 - ip->cfg_handle = *chdl; 1792 1792 - return get_irq_props(mdesc, node, ip); 1793 1793 - } 1794 1794 - 1795 1795 - return -ENODEV; 1796 1796 - } 1797 1797 - 1798 1798 - static unsigned long n2_spu_hvapi_major; 1799 1799 - static unsigned long n2_spu_hvapi_minor; 1800 1800 - 1801 1801 - static int n2_spu_hvapi_register(void) 1802 1802 - { 1803 1803 - int err; 1804 1804 - 1805 1805 - n2_spu_hvapi_major = 2; 1806 1806 - n2_spu_hvapi_minor = 0; 1807 1807 - 1808 1808 - err = sun4v_hvapi_register(HV_GRP_NCS, 1809 1809 - n2_spu_hvapi_major, 1810 1810 - &n2_spu_hvapi_minor); 1811 1811 - 1812 1812 - if (!err) 1813 1813 - pr_info("Registered NCS HVAPI version %lu.%lu\n", 1814 1814 - n2_spu_hvapi_major, 1815 1815 - n2_spu_hvapi_minor); 1816 1816 - 1817 1817 - return err; 1818 1818 - } 1819 1819 - 1820 1820 - static void n2_spu_hvapi_unregister(void) 1821 1821 - { 1822 1822 - sun4v_hvapi_unregister(HV_GRP_NCS); 1823 1823 - } 1824 1824 - 1825 1825 - static int global_ref; 1826 1826 - 1827 1827 - static int grab_global_resources(void) 1828 1828 - { 1829 1829 - int err = 0; 1830 1830 - 1831 1831 - mutex_lock(&spu_lock); 1832 1832 - 1833 1833 - if (global_ref++) 1834 1834 - goto out; 1835 1835 - 1836 1836 - err = n2_spu_hvapi_register(); 1837 1837 - if (err) 1838 1838 - goto out; 1839 1839 - 1840 1840 - err = queue_cache_init(); 1841 1841 - if (err) 1842 1842 - goto out_hvapi_release; 1843 1843 - 1844 1844 - err = -ENOMEM; 1845 1845 - cpu_to_cwq = kcalloc(NR_CPUS, sizeof(struct spu_queue *), 1846 1846 - GFP_KERNEL); 1847 1847 - if (!cpu_to_cwq) 1848 1848 - goto out_queue_cache_destroy; 1849 1849 - 1850 1850 - cpu_to_mau = kcalloc(NR_CPUS, sizeof(struct spu_queue *), 1851 1851 - GFP_KERNEL); 1852 1852 - if (!cpu_to_mau) 1853 1853 - goto out_free_cwq_table; 1854 1854 - 1855 1855 - err = 0; 1856 1856 - 1857 1857 - out: 1858 1858 - if (err) 1859 1859 - global_ref--; 1860 1860 - mutex_unlock(&spu_lock); 1861 1861 - return err; 1862 1862 - 1863 1863 - out_free_cwq_table: 1864 1864 - kfree(cpu_to_cwq); 1865 1865 - cpu_to_cwq = NULL; 1866 1866 - 1867 1867 - out_queue_cache_destroy: 1868 1868 - queue_cache_destroy(); 1869 1869 - 1870 1870 - out_hvapi_release: 1871 1871 - n2_spu_hvapi_unregister(); 1872 1872 - goto out; 1873 1873 - } 1874 1874 - 1875 1875 - static void release_global_resources(void) 1876 1876 - { 1877 1877 - mutex_lock(&spu_lock); 1878 1878 - if (!--global_ref) { 1879 1879 - kfree(cpu_to_cwq); 1880 1880 - cpu_to_cwq = NULL; 1881 1881 - 1882 1882 - kfree(cpu_to_mau); 1883 1883 - cpu_to_mau = NULL; 1884 1884 - 1885 1885 - queue_cache_destroy(); 1886 1886 - n2_spu_hvapi_unregister(); 1887 1887 - } 1888 1888 - mutex_unlock(&spu_lock); 1889 1889 - } 1890 1890 - 1891 1891 - static struct n2_crypto *alloc_n2cp(void) 1892 1892 - { 1893 1893 - struct n2_crypto *np = kzalloc(sizeof(struct n2_crypto), GFP_KERNEL); 1894 1894 - 1895 1895 - if (np) 1896 1896 - INIT_LIST_HEAD(&np->cwq_list); 1897 1897 - 1898 1898 - return np; 1899 1899 - } 1900 1900 - 1901 1901 - static void free_n2cp(struct n2_crypto *np) 1902 1902 - { 1903 1903 - kfree(np->cwq_info.ino_table); 1904 1904 - np->cwq_info.ino_table = NULL; 1905 1905 - 1906 1906 - kfree(np); 1907 1907 - } 1908 1908 - 1909 1909 - static void n2_spu_driver_version(void) 1910 1910 - { 1911 1911 - static int n2_spu_version_printed; 1912 1912 - 1913 1913 - if (n2_spu_version_printed++ == 0) 1914 1914 - pr_info("%s", version); 1915 1915 - } 1916 1916 - 1917 1917 - static int n2_crypto_probe(struct platform_device *dev) 1918 1918 - { 1919 1919 - struct mdesc_handle *mdesc; 1920 1920 - struct n2_crypto *np; 1921 1921 - int err; 1922 1922 - 1923 1923 - n2_spu_driver_version(); 1924 1924 - 1925 1925 - pr_info("Found N2CP at %pOF\n", dev->dev.of_node); 1926 1926 - 1927 1927 - np = alloc_n2cp(); 1928 1928 - if (!np) { 1929 1929 - dev_err(&dev->dev, "%pOF: Unable to allocate n2cp.\n", 1930 1930 - dev->dev.of_node); 1931 1931 - return -ENOMEM; 1932 1932 - } 1933 1933 - 1934 1934 - err = grab_global_resources(); 1935 1935 - if (err) { 1936 1936 - dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n", 1937 1937 - dev->dev.of_node); 1938 1938 - goto out_free_n2cp; 1939 1939 - } 1940 1940 - 1941 1941 - mdesc = mdesc_grab(); 1942 1942 - 1943 1943 - if (!mdesc) { 1944 1944 - dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n", 1945 1945 - dev->dev.of_node); 1946 1946 - err = -ENODEV; 1947 1947 - goto out_free_global; 1948 1948 - } 1949 1949 - err = grab_mdesc_irq_props(mdesc, dev, &np->cwq_info, "n2cp"); 1950 1950 - if (err) { 1951 1951 - dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n", 1952 1952 - dev->dev.of_node); 1953 1953 - mdesc_release(mdesc); 1954 1954 - goto out_free_global; 1955 1955 - } 1956 1956 - 1957 1957 - err = spu_mdesc_scan(mdesc, dev, &np->cwq_info, &np->cwq_list, 1958 1958 - "cwq", HV_NCS_QTYPE_CWQ, cwq_intr, 1959 1959 - cpu_to_cwq); 1960 1960 - mdesc_release(mdesc); 1961 1961 - 1962 1962 - if (err) { 1963 1963 - dev_err(&dev->dev, "%pOF: CWQ MDESC scan failed.\n", 1964 1964 - dev->dev.of_node); 1965 1965 - goto out_free_global; 1966 1966 - } 1967 1967 - 1968 1968 - err = n2_register_algs(); 1969 1969 - if (err) { 1970 1970 - dev_err(&dev->dev, "%pOF: Unable to register algorithms.\n", 1971 1971 - dev->dev.of_node); 1972 1972 - goto out_free_spu_list; 1973 1973 - } 1974 1974 - 1975 1975 - dev_set_drvdata(&dev->dev, np); 1976 1976 - 1977 1977 - return 0; 1978 1978 - 1979 1979 - out_free_spu_list: 1980 1980 - spu_list_destroy(&np->cwq_list); 1981 1981 - 1982 1982 - out_free_global: 1983 1983 - release_global_resources(); 1984 1984 - 1985 1985 - out_free_n2cp: 1986 1986 - free_n2cp(np); 1987 1987 - 1988 1988 - return err; 1989 1989 - } 1990 1990 - 1991 1991 - static void n2_crypto_remove(struct platform_device *dev) 1992 1992 - { 1993 1993 - struct n2_crypto *np = dev_get_drvdata(&dev->dev); 1994 1994 - 1995 1995 - n2_unregister_algs(); 1996 1996 - 1997 1997 - spu_list_destroy(&np->cwq_list); 1998 1998 - 1999 1999 - release_global_resources(); 2000 2000 - 2001 2001 - free_n2cp(np); 2002 2002 - } 2003 2003 - 2004 2004 - static struct n2_mau *alloc_ncp(void) 2005 2005 - { 2006 2006 - struct n2_mau *mp = kzalloc(sizeof(struct n2_mau), GFP_KERNEL); 2007 2007 - 2008 2008 - if (mp) 2009 2009 - INIT_LIST_HEAD(&mp->mau_list); 2010 2010 - 2011 2011 - return mp; 2012 2012 - } 2013 2013 - 2014 2014 - static void free_ncp(struct n2_mau *mp) 2015 2015 - { 2016 2016 - kfree(mp->mau_info.ino_table); 2017 2017 - mp->mau_info.ino_table = NULL; 2018 2018 - 2019 2019 - kfree(mp); 2020 2020 - } 2021 2021 - 2022 2022 - static int n2_mau_probe(struct platform_device *dev) 2023 2023 - { 2024 2024 - struct mdesc_handle *mdesc; 2025 2025 - struct n2_mau *mp; 2026 2026 - int err; 2027 2027 - 2028 2028 - n2_spu_driver_version(); 2029 2029 - 2030 2030 - pr_info("Found NCP at %pOF\n", dev->dev.of_node); 2031 2031 - 2032 2032 - mp = alloc_ncp(); 2033 2033 - if (!mp) { 2034 2034 - dev_err(&dev->dev, "%pOF: Unable to allocate ncp.\n", 2035 2035 - dev->dev.of_node); 2036 2036 - return -ENOMEM; 2037 2037 - } 2038 2038 - 2039 2039 - err = grab_global_resources(); 2040 2040 - if (err) { 2041 2041 - dev_err(&dev->dev, "%pOF: Unable to grab global resources.\n", 2042 2042 - dev->dev.of_node); 2043 2043 - goto out_free_ncp; 2044 2044 - } 2045 2045 - 2046 2046 - mdesc = mdesc_grab(); 2047 2047 - 2048 2048 - if (!mdesc) { 2049 2049 - dev_err(&dev->dev, "%pOF: Unable to grab MDESC.\n", 2050 2050 - dev->dev.of_node); 2051 2051 - err = -ENODEV; 2052 2052 - goto out_free_global; 2053 2053 - } 2054 2054 - 2055 2055 - err = grab_mdesc_irq_props(mdesc, dev, &mp->mau_info, "ncp"); 2056 2056 - if (err) { 2057 2057 - dev_err(&dev->dev, "%pOF: Unable to grab IRQ props.\n", 2058 2058 - dev->dev.of_node); 2059 2059 - mdesc_release(mdesc); 2060 2060 - goto out_free_global; 2061 2061 - } 2062 2062 - 2063 2063 - err = spu_mdesc_scan(mdesc, dev, &mp->mau_info, &mp->mau_list, 2064 2064 - "mau", HV_NCS_QTYPE_MAU, mau_intr, 2065 2065 - cpu_to_mau); 2066 2066 - mdesc_release(mdesc); 2067 2067 - 2068 2068 - if (err) { 2069 2069 - dev_err(&dev->dev, "%pOF: MAU MDESC scan failed.\n", 2070 2070 - dev->dev.of_node); 2071 2071 - goto out_free_global; 2072 2072 - } 2073 2073 - 2074 2074 - dev_set_drvdata(&dev->dev, mp); 2075 2075 - 2076 2076 - return 0; 2077 2077 - 2078 2078 - out_free_global: 2079 2079 - release_global_resources(); 2080 2080 - 2081 2081 - out_free_ncp: 2082 2082 - free_ncp(mp); 2083 2083 - 2084 2084 - return err; 2085 2085 - } 2086 2086 - 2087 2087 - static void n2_mau_remove(struct platform_device *dev) 2088 2088 - { 2089 2089 - struct n2_mau *mp = dev_get_drvdata(&dev->dev); 2090 2090 - 2091 2091 - spu_list_destroy(&mp->mau_list); 2092 2092 - 2093 2093 - release_global_resources(); 2094 2094 - 2095 2095 - free_ncp(mp); 2096 2096 - } 2097 2097 - 2098 2098 - static const struct of_device_id n2_crypto_match[] = { 2099 2099 - { 2100 2100 - .name = "n2cp", 2101 2101 - .compatible = "SUNW,n2-cwq", 2102 2102 - }, 2103 2103 - { 2104 2104 - .name = "n2cp", 2105 2105 - .compatible = "SUNW,vf-cwq", 2106 2106 - }, 2107 2107 - { 2108 2108 - .name = "n2cp", 2109 2109 - .compatible = "SUNW,kt-cwq", 2110 2110 - }, 2111 2111 - {}, 2112 2112 - }; 2113 2113 - 2114 2114 - MODULE_DEVICE_TABLE(of, n2_crypto_match); 2115 2115 - 2116 2116 - static struct platform_driver n2_crypto_driver = { 2117 2117 - .driver = { 2118 2118 - .name = "n2cp", 2119 2119 - .of_match_table = n2_crypto_match, 2120 2120 - }, 2121 2121 - .probe = n2_crypto_probe, 2122 2122 - .remove = n2_crypto_remove, 2123 2123 - }; 2124 2124 - 2125 2125 - static const struct of_device_id n2_mau_match[] = { 2126 2126 - { 2127 2127 - .name = "ncp", 2128 2128 - .compatible = "SUNW,n2-mau", 2129 2129 - }, 2130 2130 - { 2131 2131 - .name = "ncp", 2132 2132 - .compatible = "SUNW,vf-mau", 2133 2133 - }, 2134 2134 - { 2135 2135 - .name = "ncp", 2136 2136 - .compatible = "SUNW,kt-mau", 2137 2137 - }, 2138 2138 - {}, 2139 2139 - }; 2140 2140 - 2141 2141 - MODULE_DEVICE_TABLE(of, n2_mau_match); 2142 2142 - 2143 2143 - static struct platform_driver n2_mau_driver = { 2144 2144 - .driver = { 2145 2145 - .name = "ncp", 2146 2146 - .of_match_table = n2_mau_match, 2147 2147 - }, 2148 2148 - .probe = n2_mau_probe, 2149 2149 - .remove = n2_mau_remove, 2150 2150 - }; 2151 2151 - 2152 2152 - static struct platform_driver * const drivers[] = { 2153 2153 - &n2_crypto_driver, 2154 2154 - &n2_mau_driver, 2155 2155 - }; 2156 2156 - 2157 2157 - static int __init n2_init(void) 2158 2158 - { 2159 2159 - return platform_register_drivers(drivers, ARRAY_SIZE(drivers)); 2160 2160 - } 2161 2161 - 2162 2162 - static void __exit n2_exit(void) 2163 2163 - { 2164 2164 - platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); 2165 2165 - } 2166 2166 - 2167 2167 - module_init(n2_init); 2168 2168 - module_exit(n2_exit);

-232

drivers/crypto/n2_core.h

reviewed

··· 1 1 - /* SPDX-License-Identifier: GPL-2.0 */ 2 2 - #ifndef _N2_CORE_H 3 3 - #define _N2_CORE_H 4 4 - 5 5 - #ifndef __ASSEMBLY__ 6 6 - 7 7 - struct ino_blob { 8 8 - u64 intr; 9 9 - u64 ino; 10 10 - }; 11 11 - 12 12 - struct spu_mdesc_info { 13 13 - u64 cfg_handle; 14 14 - struct ino_blob *ino_table; 15 15 - int num_intrs; 16 16 - }; 17 17 - 18 18 - struct n2_crypto { 19 19 - struct spu_mdesc_info cwq_info; 20 20 - struct list_head cwq_list; 21 21 - }; 22 22 - 23 23 - struct n2_mau { 24 24 - struct spu_mdesc_info mau_info; 25 25 - struct list_head mau_list; 26 26 - }; 27 27 - 28 28 - #define CWQ_ENTRY_SIZE 64 29 29 - #define CWQ_NUM_ENTRIES 64 30 30 - 31 31 - #define MAU_ENTRY_SIZE 64 32 32 - #define MAU_NUM_ENTRIES 64 33 33 - 34 34 - struct cwq_initial_entry { 35 35 - u64 control; 36 36 - u64 src_addr; 37 37 - u64 auth_key_addr; 38 38 - u64 auth_iv_addr; 39 39 - u64 final_auth_state_addr; 40 40 - u64 enc_key_addr; 41 41 - u64 enc_iv_addr; 42 42 - u64 dest_addr; 43 43 - }; 44 44 - 45 45 - struct cwq_ext_entry { 46 46 - u64 len; 47 47 - u64 src_addr; 48 48 - u64 resv1; 49 49 - u64 resv2; 50 50 - u64 resv3; 51 51 - u64 resv4; 52 52 - u64 resv5; 53 53 - u64 resv6; 54 54 - }; 55 55 - 56 56 - struct cwq_final_entry { 57 57 - u64 control; 58 58 - u64 src_addr; 59 59 - u64 resv1; 60 60 - u64 resv2; 61 61 - u64 resv3; 62 62 - u64 resv4; 63 63 - u64 resv5; 64 64 - u64 resv6; 65 65 - }; 66 66 - 67 67 - #define CONTROL_LEN 0x000000000000ffffULL 68 68 - #define CONTROL_LEN_SHIFT 0 69 69 - #define CONTROL_HMAC_KEY_LEN 0x0000000000ff0000ULL 70 70 - #define CONTROL_HMAC_KEY_LEN_SHIFT 16 71 71 - #define CONTROL_ENC_TYPE 0x00000000ff000000ULL 72 72 - #define CONTROL_ENC_TYPE_SHIFT 24 73 73 - #define ENC_TYPE_ALG_RC4_STREAM 0x00ULL 74 74 - #define ENC_TYPE_ALG_RC4_NOSTREAM 0x04ULL 75 75 - #define ENC_TYPE_ALG_DES 0x08ULL 76 76 - #define ENC_TYPE_ALG_3DES 0x0cULL 77 77 - #define ENC_TYPE_ALG_AES128 0x10ULL 78 78 - #define ENC_TYPE_ALG_AES192 0x14ULL 79 79 - #define ENC_TYPE_ALG_AES256 0x18ULL 80 80 - #define ENC_TYPE_ALG_RESERVED 0x1cULL 81 81 - #define ENC_TYPE_ALG_MASK 0x1cULL 82 82 - #define ENC_TYPE_CHAINING_ECB 0x00ULL 83 83 - #define ENC_TYPE_CHAINING_CBC 0x01ULL 84 84 - #define ENC_TYPE_CHAINING_CFB 0x02ULL 85 85 - #define ENC_TYPE_CHAINING_COUNTER 0x03ULL 86 86 - #define ENC_TYPE_CHAINING_MASK 0x03ULL 87 87 - #define CONTROL_AUTH_TYPE 0x0000001f00000000ULL 88 88 - #define CONTROL_AUTH_TYPE_SHIFT 32 89 89 - #define AUTH_TYPE_RESERVED 0x00ULL 90 90 - #define AUTH_TYPE_MD5 0x01ULL 91 91 - #define AUTH_TYPE_SHA1 0x02ULL 92 92 - #define AUTH_TYPE_SHA256 0x03ULL 93 93 - #define AUTH_TYPE_CRC32 0x04ULL 94 94 - #define AUTH_TYPE_HMAC_MD5 0x05ULL 95 95 - #define AUTH_TYPE_HMAC_SHA1 0x06ULL 96 96 - #define AUTH_TYPE_HMAC_SHA256 0x07ULL 97 97 - #define AUTH_TYPE_TCP_CHECKSUM 0x08ULL 98 98 - #define AUTH_TYPE_SSL_HMAC_MD5 0x09ULL 99 99 - #define AUTH_TYPE_SSL_HMAC_SHA1 0x0aULL 100 100 - #define AUTH_TYPE_SSL_HMAC_SHA256 0x0bULL 101 101 - #define CONTROL_STRAND 0x000000e000000000ULL 102 102 - #define CONTROL_STRAND_SHIFT 37 103 103 - #define CONTROL_HASH_LEN 0x0000ff0000000000ULL 104 104 - #define CONTROL_HASH_LEN_SHIFT 40 105 105 - #define CONTROL_INTERRUPT 0x0001000000000000ULL 106 106 - #define CONTROL_STORE_FINAL_AUTH_STATE 0x0002000000000000ULL 107 107 - #define CONTROL_RESERVED 0x001c000000000000ULL 108 108 - #define CONTROL_HV_DONE 0x0004000000000000ULL 109 109 - #define CONTROL_HV_PROTOCOL_ERROR 0x0008000000000000ULL 110 110 - #define CONTROL_HV_HARDWARE_ERROR 0x0010000000000000ULL 111 111 - #define CONTROL_END_OF_BLOCK 0x0020000000000000ULL 112 112 - #define CONTROL_START_OF_BLOCK 0x0040000000000000ULL 113 113 - #define CONTROL_ENCRYPT 0x0080000000000000ULL 114 114 - #define CONTROL_OPCODE 0xff00000000000000ULL 115 115 - #define CONTROL_OPCODE_SHIFT 56 116 116 - #define OPCODE_INPLACE_BIT 0x80ULL 117 117 - #define OPCODE_SSL_KEYBLOCK 0x10ULL 118 118 - #define OPCODE_COPY 0x20ULL 119 119 - #define OPCODE_ENCRYPT 0x40ULL 120 120 - #define OPCODE_AUTH_MAC 0x41ULL 121 121 - 122 122 - #endif /* !(__ASSEMBLY__) */ 123 123 - 124 124 - /* NCS v2.0 hypervisor interfaces */ 125 125 - #define HV_NCS_QTYPE_MAU 0x01 126 126 - #define HV_NCS_QTYPE_CWQ 0x02 127 127 - 128 128 - /* ncs_qconf() 129 129 - * TRAP: HV_FAST_TRAP 130 130 - * FUNCTION: HV_FAST_NCS_QCONF 131 131 - * ARG0: Queue type (HV_NCS_QTYPE_{MAU,CWQ}) 132 132 - * ARG1: Real address of queue, or handle for unconfigure 133 133 - * ARG2: Number of entries in queue, zero for unconfigure 134 134 - * RET0: status 135 135 - * RET1: queue handle 136 136 - * 137 137 - * Configure a queue in the stream processing unit. 138 138 - * 139 139 - * The real address given as the base must be 64-byte 140 140 - * aligned. 141 141 - * 142 142 - * The queue size can range from a minimum of 2 to a maximum 143 143 - * of 64. The queue size must be a power of two. 144 144 - * 145 145 - * To unconfigure a queue, specify a length of zero and place 146 146 - * the queue handle into ARG1. 147 147 - * 148 148 - * On configure success the hypervisor will set the FIRST, HEAD, 149 149 - * and TAIL registers to the address of the first entry in the 150 150 - * queue. The LAST register will be set to point to the last 151 151 - * entry in the queue. 152 152 - */ 153 153 - #define HV_FAST_NCS_QCONF 0x111 154 154 - 155 155 - /* ncs_qinfo() 156 156 - * TRAP: HV_FAST_TRAP 157 157 - * FUNCTION: HV_FAST_NCS_QINFO 158 158 - * ARG0: Queue handle 159 159 - * RET0: status 160 160 - * RET1: Queue type (HV_NCS_QTYPE_{MAU,CWQ}) 161 161 - * RET2: Queue base address 162 162 - * RET3: Number of entries 163 163 - */ 164 164 - #define HV_FAST_NCS_QINFO 0x112 165 165 - 166 166 - /* ncs_gethead() 167 167 - * TRAP: HV_FAST_TRAP 168 168 - * FUNCTION: HV_FAST_NCS_GETHEAD 169 169 - * ARG0: Queue handle 170 170 - * RET0: status 171 171 - * RET1: queue head offset 172 172 - */ 173 173 - #define HV_FAST_NCS_GETHEAD 0x113 174 174 - 175 175 - /* ncs_gettail() 176 176 - * TRAP: HV_FAST_TRAP 177 177 - * FUNCTION: HV_FAST_NCS_GETTAIL 178 178 - * ARG0: Queue handle 179 179 - * RET0: status 180 180 - * RET1: queue tail offset 181 181 - */ 182 182 - #define HV_FAST_NCS_GETTAIL 0x114 183 183 - 184 184 - /* ncs_settail() 185 185 - * TRAP: HV_FAST_TRAP 186 186 - * FUNCTION: HV_FAST_NCS_SETTAIL 187 187 - * ARG0: Queue handle 188 188 - * ARG1: New tail offset 189 189 - * RET0: status 190 190 - */ 191 191 - #define HV_FAST_NCS_SETTAIL 0x115 192 192 - 193 193 - /* ncs_qhandle_to_devino() 194 194 - * TRAP: HV_FAST_TRAP 195 195 - * FUNCTION: HV_FAST_NCS_QHANDLE_TO_DEVINO 196 196 - * ARG0: Queue handle 197 197 - * RET0: status 198 198 - * RET1: devino 199 199 - */ 200 200 - #define HV_FAST_NCS_QHANDLE_TO_DEVINO 0x116 201 201 - 202 202 - /* ncs_sethead_marker() 203 203 - * TRAP: HV_FAST_TRAP 204 204 - * FUNCTION: HV_FAST_NCS_SETHEAD_MARKER 205 205 - * ARG0: Queue handle 206 206 - * ARG1: New head offset 207 207 - * RET0: status 208 208 - */ 209 209 - #define HV_FAST_NCS_SETHEAD_MARKER 0x117 210 210 - 211 211 - #ifndef __ASSEMBLY__ 212 212 - extern unsigned long sun4v_ncs_qconf(unsigned long queue_type, 213 213 - unsigned long queue_ra, 214 214 - unsigned long num_entries, 215 215 - unsigned long *qhandle); 216 216 - extern unsigned long sun4v_ncs_qinfo(unsigned long qhandle, 217 217 - unsigned long *queue_type, 218 218 - unsigned long *queue_ra, 219 219 - unsigned long *num_entries); 220 220 - extern unsigned long sun4v_ncs_gethead(unsigned long qhandle, 221 221 - unsigned long *head); 222 222 - extern unsigned long sun4v_ncs_gettail(unsigned long qhandle, 223 223 - unsigned long *tail); 224 224 - extern unsigned long sun4v_ncs_settail(unsigned long qhandle, 225 225 - unsigned long tail); 226 226 - extern unsigned long sun4v_ncs_qhandle_to_devino(unsigned long qhandle, 227 227 - unsigned long *devino); 228 228 - extern unsigned long sun4v_ncs_sethead_marker(unsigned long qhandle, 229 229 - unsigned long head); 230 230 - #endif /* !(__ASSEMBLY__) */ 231 231 - 232 232 - #endif /* _N2_CORE_H */

+14 -20

drivers/crypto/omap-aes.c

reviewed

··· 18 18 #include <crypto/internal/aead.h> 19 19 #include <crypto/internal/engine.h> 20 20 #include <crypto/internal/skcipher.h> 21 21 - #include <crypto/scatterwalk.h> 22 21 #include <linux/dma-mapping.h> 23 22 #include <linux/dmaengine.h> 24 23 #include <linux/err.h> ··· 271 272 int ret; 272 273 273 274 if (dd->pio_only) { 274 274 - scatterwalk_start(&dd->in_walk, dd->in_sg); 275 275 + dd->in_sg_offset = 0; 275 276 if (out_sg_len) 276 276 - scatterwalk_start(&dd->out_walk, dd->out_sg); 277 277 + dd->out_sg_offset = 0; 277 278 278 279 /* Enable DATAIN interrupt and let it take 279 280 care of the rest */ ··· 870 871 871 872 BUG_ON(!dd->in_sg); 872 873 873 873 - BUG_ON(_calc_walked(in) > dd->in_sg->length); 874 874 + BUG_ON(dd->in_sg_offset > dd->in_sg->length); 874 875 875 875 - src = sg_virt(dd->in_sg) + _calc_walked(in); 876 876 + src = sg_virt(dd->in_sg) + dd->in_sg_offset; 876 877 877 878 for (i = 0; i < AES_BLOCK_WORDS; i++) { 878 879 omap_aes_write(dd, AES_REG_DATA_N(dd, i), *src); 879 879 - 880 880 - scatterwalk_advance(&dd->in_walk, 4); 881 881 - if (dd->in_sg->length == _calc_walked(in)) { 880 880 + dd->in_sg_offset += 4; 881 881 + if (dd->in_sg_offset == dd->in_sg->length) { 882 882 dd->in_sg = sg_next(dd->in_sg); 883 883 if (dd->in_sg) { 884 884 - scatterwalk_start(&dd->in_walk, 885 885 - dd->in_sg); 886 886 - src = sg_virt(dd->in_sg) + 887 887 - _calc_walked(in); 884 884 + dd->in_sg_offset = 0; 885 885 + src = sg_virt(dd->in_sg); 888 886 } 889 887 } else { 890 888 src++; ··· 900 904 901 905 BUG_ON(!dd->out_sg); 902 906 903 903 - BUG_ON(_calc_walked(out) > dd->out_sg->length); 907 907 + BUG_ON(dd->out_sg_offset > dd->out_sg->length); 904 908 905 905 - dst = sg_virt(dd->out_sg) + _calc_walked(out); 909 909 + dst = sg_virt(dd->out_sg) + dd->out_sg_offset; 906 910 907 911 for (i = 0; i < AES_BLOCK_WORDS; i++) { 908 912 *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i)); 909 909 - scatterwalk_advance(&dd->out_walk, 4); 910 910 - if (dd->out_sg->length == _calc_walked(out)) { 913 913 + dd->out_sg_offset += 4; 914 914 + if (dd->out_sg_offset == dd->out_sg->length) { 911 915 dd->out_sg = sg_next(dd->out_sg); 912 916 if (dd->out_sg) { 913 913 - scatterwalk_start(&dd->out_walk, 914 914 - dd->out_sg); 915 915 - dst = sg_virt(dd->out_sg) + 916 916 - _calc_walked(out); 917 917 + dd->out_sg_offset = 0; 918 918 + dst = sg_virt(dd->out_sg); 917 919 } 918 920 } else { 919 921 dst++;

+2 -4

drivers/crypto/omap-aes.h

reviewed

··· 14 14 #define DST_MAXBURST 4 15 15 #define DMA_MIN (DST_MAXBURST * sizeof(u32)) 16 16 17 17 - #define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset) 18 18 - 19 17 /* 20 18 * OMAP TRM gives bitfields as start:end, where start is the higher bit 21 19 * number. For example 7:0 ··· 184 186 struct scatterlist out_sgl; 185 187 struct scatterlist *orig_out; 186 188 187 187 - struct scatter_walk in_walk; 188 188 - struct scatter_walk out_walk; 189 189 + unsigned int in_sg_offset; 190 190 + unsigned int out_sg_offset; 189 191 struct dma_chan *dma_lch_in; 190 192 struct dma_chan *dma_lch_out; 191 193 int in_sg_len;

+16 -24

drivers/crypto/omap-des.c

reviewed

··· 19 19 #include <crypto/engine.h> 20 20 #include <crypto/internal/des.h> 21 21 #include <crypto/internal/skcipher.h> 22 22 - #include <crypto/scatterwalk.h> 23 22 #include <linux/dma-mapping.h> 24 23 #include <linux/dmaengine.h> 25 24 #include <linux/err.h> ··· 38 39 #define DST_MAXBURST 2 39 40 40 41 #define DES_BLOCK_WORDS (DES_BLOCK_SIZE >> 2) 41 41 - 42 42 - #define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset) 43 42 44 43 #define DES_REG_KEY(dd, x) ((dd)->pdata->key_ofs - \ 45 44 ((x ^ 0x01) * 0x04)) ··· 149 152 struct scatterlist out_sgl; 150 153 struct scatterlist *orig_out; 151 154 152 152 - struct scatter_walk in_walk; 153 153 - struct scatter_walk out_walk; 155 155 + unsigned int in_sg_offset; 156 156 + unsigned int out_sg_offset; 154 157 struct dma_chan *dma_lch_in; 155 158 struct dma_chan *dma_lch_out; 156 159 int in_sg_len; ··· 376 379 int ret; 377 380 378 381 if (dd->pio_only) { 379 379 - scatterwalk_start(&dd->in_walk, dd->in_sg); 380 380 - scatterwalk_start(&dd->out_walk, dd->out_sg); 382 382 + dd->in_sg_offset = 0; 383 383 + dd->out_sg_offset = 0; 381 384 382 385 /* Enable DATAIN interrupt and let it take 383 386 care of the rest */ ··· 833 836 834 837 BUG_ON(!dd->in_sg); 835 838 836 836 - BUG_ON(_calc_walked(in) > dd->in_sg->length); 839 839 + BUG_ON(dd->in_sg_offset > dd->in_sg->length); 837 840 838 838 - src = sg_virt(dd->in_sg) + _calc_walked(in); 841 841 + src = sg_virt(dd->in_sg) + dd->in_sg_offset; 839 842 840 843 for (i = 0; i < DES_BLOCK_WORDS; i++) { 841 844 omap_des_write(dd, DES_REG_DATA_N(dd, i), *src); 842 842 - 843 843 - scatterwalk_advance(&dd->in_walk, 4); 844 844 - if (dd->in_sg->length == _calc_walked(in)) { 845 845 + dd->in_sg_offset += 4; 846 846 + if (dd->in_sg_offset == dd->in_sg->length) { 845 847 dd->in_sg = sg_next(dd->in_sg); 846 848 if (dd->in_sg) { 847 847 - scatterwalk_start(&dd->in_walk, 848 848 - dd->in_sg); 849 849 - src = sg_virt(dd->in_sg) + 850 850 - _calc_walked(in); 849 849 + dd->in_sg_offset = 0; 850 850 + src = sg_virt(dd->in_sg); 851 851 } 852 852 } else { 853 853 src++; ··· 863 869 864 870 BUG_ON(!dd->out_sg); 865 871 866 866 - BUG_ON(_calc_walked(out) > dd->out_sg->length); 872 872 + BUG_ON(dd->out_sg_offset > dd->out_sg->length); 867 873 868 868 - dst = sg_virt(dd->out_sg) + _calc_walked(out); 874 874 + dst = sg_virt(dd->out_sg) + dd->out_sg_offset; 869 875 870 876 for (i = 0; i < DES_BLOCK_WORDS; i++) { 871 877 *dst = omap_des_read(dd, DES_REG_DATA_N(dd, i)); 872 872 - scatterwalk_advance(&dd->out_walk, 4); 873 873 - if (dd->out_sg->length == _calc_walked(out)) { 878 878 + dd->out_sg_offset += 4; 879 879 + if (dd->out_sg_offset == dd->out_sg->length) { 874 880 dd->out_sg = sg_next(dd->out_sg); 875 881 if (dd->out_sg) { 876 876 - scatterwalk_start(&dd->out_walk, 877 877 - dd->out_sg); 878 878 - dst = sg_virt(dd->out_sg) + 879 879 - _calc_walked(out); 882 882 + dd->out_sg_offset = 0; 883 883 + dst = sg_virt(dd->out_sg); 880 884 } 881 885 } else { 882 886 dst++;

+1 -1

drivers/crypto/qce/aead.c

reviewed

··· 786 786 alg->init = qce_aead_init; 787 787 alg->exit = qce_aead_exit; 788 788 789 789 - alg->base.cra_priority = 300; 789 789 + alg->base.cra_priority = 275; 790 790 alg->base.cra_flags = CRYPTO_ALG_ASYNC | 791 791 CRYPTO_ALG_ALLOCATES_MEMORY | 792 792 CRYPTO_ALG_KERN_DRIVER_ONLY |

+45 -86

drivers/crypto/qce/core.c

reviewed

··· 3 3 * Copyright (c) 2010-2014, The Linux Foundation. All rights reserved. 4 4 */ 5 5 6 6 + #include <linux/cleanup.h> 6 7 #include <linux/clk.h> 8 8 + #include <linux/device.h> 7 9 #include <linux/dma-mapping.h> 8 10 #include <linux/interconnect.h> 9 11 #include <linux/interrupt.h> 10 12 #include <linux/module.h> 11 13 #include <linux/mod_devicetable.h> 12 14 #include <linux/platform_device.h> 13 13 - #include <linux/spinlock.h> 14 15 #include <linux/types.h> 15 16 #include <crypto/algapi.h> 16 17 #include <crypto/internal/hash.h> ··· 38 37 #endif 39 38 }; 40 39 41 41 - static void qce_unregister_algs(struct qce_device *qce) 40 40 + static void qce_unregister_algs(void *data) 42 41 { 43 42 const struct qce_algo_ops *ops; 43 43 + struct qce_device *qce = data; 44 44 int i; 45 45 46 46 for (i = 0; i < ARRAY_SIZE(qce_ops); i++) { ··· 50 48 } 51 49 } 52 50 53 53 - static int qce_register_algs(struct qce_device *qce) 51 51 + static int devm_qce_register_algs(struct qce_device *qce) 54 52 { 55 53 const struct qce_algo_ops *ops; 56 56 - int i, ret = -ENODEV; 54 54 + int i, j, ret = -ENODEV; 57 55 58 56 for (i = 0; i < ARRAY_SIZE(qce_ops); i++) { 59 57 ops = qce_ops[i]; 60 58 ret = ops->register_algs(qce); 61 61 - if (ret) 62 62 - break; 59 59 + if (ret) { 60 60 + for (j = i - 1; j >= 0; j--) 61 61 + ops->unregister_algs(qce); 62 62 + return ret; 63 63 + } 63 64 } 64 65 65 65 - return ret; 66 66 + return devm_add_action_or_reset(qce->dev, qce_unregister_algs, qce); 66 67 } 67 68 68 69 static int qce_handle_request(struct crypto_async_request *async_req) ··· 89 84 struct crypto_async_request *req) 90 85 { 91 86 struct crypto_async_request *async_req, *backlog; 92 92 - unsigned long flags; 93 87 int ret = 0, err; 94 88 95 95 - spin_lock_irqsave(&qce->lock, flags); 89 89 + scoped_guard(mutex, &qce->lock) { 90 90 + if (req) 91 91 + ret = crypto_enqueue_request(&qce->queue, req); 96 92 97 97 - if (req) 98 98 - ret = crypto_enqueue_request(&qce->queue, req); 93 93 + /* busy, do not dequeue request */ 94 94 + if (qce->req) 95 95 + return ret; 99 96 100 100 - /* busy, do not dequeue request */ 101 101 - if (qce->req) { 102 102 - spin_unlock_irqrestore(&qce->lock, flags); 103 103 - return ret; 97 97 + backlog = crypto_get_backlog(&qce->queue); 98 98 + async_req = crypto_dequeue_request(&qce->queue); 99 99 + if (async_req) 100 100 + qce->req = async_req; 104 101 } 105 105 - 106 106 - backlog = crypto_get_backlog(&qce->queue); 107 107 - async_req = crypto_dequeue_request(&qce->queue); 108 108 - if (async_req) 109 109 - qce->req = async_req; 110 110 - 111 111 - spin_unlock_irqrestore(&qce->lock, flags); 112 102 113 103 if (!async_req) 114 104 return ret; 115 105 116 106 if (backlog) { 117 117 - spin_lock_bh(&qce->lock); 118 118 - crypto_request_complete(backlog, -EINPROGRESS); 119 119 - spin_unlock_bh(&qce->lock); 107 107 + scoped_guard(mutex, &qce->lock) 108 108 + crypto_request_complete(backlog, -EINPROGRESS); 120 109 } 121 110 122 111 err = qce_handle_request(async_req); 123 112 if (err) { 124 113 qce->result = err; 125 125 - tasklet_schedule(&qce->done_tasklet); 114 114 + schedule_work(&qce->done_work); 126 115 } 127 116 128 117 return ret; 129 118 } 130 119 131 131 - static void qce_tasklet_req_done(unsigned long data) 120 120 + static void qce_req_done_work(struct work_struct *work) 132 121 { 133 133 - struct qce_device *qce = (struct qce_device *)data; 122 122 + struct qce_device *qce = container_of(work, struct qce_device, 123 123 + done_work); 134 124 struct crypto_async_request *req; 135 135 - unsigned long flags; 136 125 137 137 - spin_lock_irqsave(&qce->lock, flags); 138 138 - req = qce->req; 139 139 - qce->req = NULL; 140 140 - spin_unlock_irqrestore(&qce->lock, flags); 126 126 + scoped_guard(mutex, &qce->lock) { 127 127 + req = qce->req; 128 128 + qce->req = NULL; 129 129 + } 141 130 142 131 if (req) 143 132 crypto_request_complete(req, qce->result); ··· 148 149 static void qce_async_request_done(struct qce_device *qce, int ret) 149 150 { 150 151 qce->result = ret; 151 151 - tasklet_schedule(&qce->done_tasklet); 152 152 + schedule_work(&qce->done_work); 152 153 } 153 154 154 155 static int qce_check_version(struct qce_device *qce) ··· 208 209 if (ret < 0) 209 210 return ret; 210 211 211 211 - qce->core = devm_clk_get_optional(qce->dev, "core"); 212 212 + qce->core = devm_clk_get_optional_enabled(qce->dev, "core"); 212 213 if (IS_ERR(qce->core)) 213 214 return PTR_ERR(qce->core); 214 215 215 215 - qce->iface = devm_clk_get_optional(qce->dev, "iface"); 216 216 + qce->iface = devm_clk_get_optional_enabled(qce->dev, "iface"); 216 217 if (IS_ERR(qce->iface)) 217 218 return PTR_ERR(qce->iface); 218 219 219 219 - qce->bus = devm_clk_get_optional(qce->dev, "bus"); 220 220 + qce->bus = devm_clk_get_optional_enabled(qce->dev, "bus"); 220 221 if (IS_ERR(qce->bus)) 221 222 return PTR_ERR(qce->bus); 222 223 ··· 228 229 if (ret) 229 230 return ret; 230 231 231 231 - ret = clk_prepare_enable(qce->core); 232 232 + ret = devm_qce_dma_request(qce->dev, &qce->dma); 232 233 if (ret) 233 233 - goto err_mem_path_disable; 234 234 - 235 235 - ret = clk_prepare_enable(qce->iface); 236 236 - if (ret) 237 237 - goto err_clks_core; 238 238 - 239 239 - ret = clk_prepare_enable(qce->bus); 240 240 - if (ret) 241 241 - goto err_clks_iface; 242 242 - 243 243 - ret = qce_dma_request(qce->dev, &qce->dma); 244 244 - if (ret) 245 245 - goto err_clks; 234 234 + return ret; 246 235 247 236 ret = qce_check_version(qce); 248 237 if (ret) 249 249 - goto err_clks; 238 238 + return ret; 250 239 251 251 - spin_lock_init(&qce->lock); 252 252 - tasklet_init(&qce->done_tasklet, qce_tasklet_req_done, 253 253 - (unsigned long)qce); 240 240 + ret = devm_mutex_init(qce->dev, &qce->lock); 241 241 + if (ret) 242 242 + return ret; 243 243 + 244 244 + INIT_WORK(&qce->done_work, qce_req_done_work); 254 245 crypto_init_queue(&qce->queue, QCE_QUEUE_LENGTH); 255 246 256 247 qce->async_req_enqueue = qce_async_request_enqueue; 257 248 qce->async_req_done = qce_async_request_done; 258 249 259 259 - ret = qce_register_algs(qce); 260 260 - if (ret) 261 261 - goto err_dma; 262 262 - 263 263 - return 0; 264 264 - 265 265 - err_dma: 266 266 - qce_dma_release(&qce->dma); 267 267 - err_clks: 268 268 - clk_disable_unprepare(qce->bus); 269 269 - err_clks_iface: 270 270 - clk_disable_unprepare(qce->iface); 271 271 - err_clks_core: 272 272 - clk_disable_unprepare(qce->core); 273 273 - err_mem_path_disable: 274 274 - icc_set_bw(qce->mem_path, 0, 0); 275 275 - 276 276 - return ret; 277 277 - } 278 278 - 279 279 - static void qce_crypto_remove(struct platform_device *pdev) 280 280 - { 281 281 - struct qce_device *qce = platform_get_drvdata(pdev); 282 282 - 283 283 - tasklet_kill(&qce->done_tasklet); 284 284 - qce_unregister_algs(qce); 285 285 - qce_dma_release(&qce->dma); 286 286 - clk_disable_unprepare(qce->bus); 287 287 - clk_disable_unprepare(qce->iface); 288 288 - clk_disable_unprepare(qce->core); 250 250 + return devm_qce_register_algs(qce); 289 251 } 290 252 291 253 static const struct of_device_id qce_crypto_of_match[] = { ··· 259 299 260 300 static struct platform_driver qce_crypto_driver = { 261 301 .probe = qce_crypto_probe, 262 262 - .remove = qce_crypto_remove, 263 302 .driver = { 264 303 .name = KBUILD_MODNAME, 265 304 .of_match_table = qce_crypto_of_match,

+6 -3

drivers/crypto/qce/core.h

reviewed

··· 6 6 #ifndef _CORE_H_ 7 7 #define _CORE_H_ 8 8 9 9 + #include <linux/mutex.h> 10 10 + #include <linux/workqueue.h> 11 11 + 9 12 #include "dma.h" 10 13 11 14 /** 12 15 * struct qce_device - crypto engine device structure 13 16 * @queue: crypto request queue 14 17 * @lock: the lock protects queue and req 15 15 - * @done_tasklet: done tasklet object 18 18 + * @done_work: workqueue context 16 19 * @req: current active request 17 20 * @result: result of current transform 18 21 * @base: virtual IO base ··· 31 28 */ 32 29 struct qce_device { 33 30 struct crypto_queue queue; 34 34 - spinlock_t lock; 35 35 - struct tasklet_struct done_tasklet; 31 31 + struct mutex lock; 32 32 + struct work_struct done_work; 36 33 struct crypto_async_request *req; 37 34 int result; 38 35 void __iomem *base;

+13 -9

drivers/crypto/qce/dma.c

reviewed

··· 3 3 * Copyright (c) 2012-2014, The Linux Foundation. All rights reserved. 4 4 */ 5 5 6 6 + #include <linux/device.h> 6 7 #include <linux/dmaengine.h> 7 8 #include <crypto/scatterwalk.h> 8 9 9 10 #include "dma.h" 10 11 11 11 - int qce_dma_request(struct device *dev, struct qce_dma_data *dma) 12 12 + static void qce_dma_release(void *data) 13 13 + { 14 14 + struct qce_dma_data *dma = data; 15 15 + 16 16 + dma_release_channel(dma->txchan); 17 17 + dma_release_channel(dma->rxchan); 18 18 + kfree(dma->result_buf); 19 19 + } 20 20 + 21 21 + int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma) 12 22 { 13 23 int ret; 14 24 ··· 41 31 42 32 dma->ignore_buf = dma->result_buf + QCE_RESULT_BUF_SZ; 43 33 44 44 - return 0; 34 34 + return devm_add_action_or_reset(dev, qce_dma_release, dma); 35 35 + 45 36 error_nomem: 46 37 dma_release_channel(dma->rxchan); 47 38 error_rx: 48 39 dma_release_channel(dma->txchan); 49 40 return ret; 50 50 - } 51 51 - 52 52 - void qce_dma_release(struct qce_dma_data *dma) 53 53 - { 54 54 - dma_release_channel(dma->txchan); 55 55 - dma_release_channel(dma->rxchan); 56 56 - kfree(dma->result_buf); 57 41 } 58 42 59 43 struct scatterlist *

+1 -2

drivers/crypto/qce/dma.h

reviewed

··· 34 34 void *ignore_buf; 35 35 }; 36 36 37 37 - int qce_dma_request(struct device *dev, struct qce_dma_data *dma); 38 38 - void qce_dma_release(struct qce_dma_data *dma); 37 37 + int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma); 39 38 int qce_dma_prep_sgs(struct qce_dma_data *dma, struct scatterlist *sg_in, 40 39 int in_ents, struct scatterlist *sg_out, int out_ents, 41 40 dma_async_tx_callback cb, void *cb_param);

+1 -1

drivers/crypto/qce/sha.c

reviewed

··· 482 482 483 483 base = &alg->halg.base; 484 484 base->cra_blocksize = def->blocksize; 485 485 - base->cra_priority = 300; 485 485 + base->cra_priority = 175; 486 486 base->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; 487 487 base->cra_ctxsize = sizeof(struct qce_sha_ctx); 488 488 base->cra_alignmask = 0;

+1 -1

drivers/crypto/qce/skcipher.c

reviewed

··· 461 461 alg->encrypt = qce_skcipher_encrypt; 462 462 alg->decrypt = qce_skcipher_decrypt; 463 463 464 464 - alg->base.cra_priority = 300; 464 464 + alg->base.cra_priority = 275; 465 465 alg->base.cra_flags = CRYPTO_ALG_ASYNC | 466 466 CRYPTO_ALG_ALLOCATES_MEMORY | 467 467 CRYPTO_ALG_KERN_DRIVER_ONLY;

+5 -2

drivers/crypto/tegra/tegra-se-aes.c

reviewed

··· 1750 1750 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 1751 1751 struct tegra_cmac_ctx *ctx = crypto_ahash_ctx(tfm); 1752 1752 struct tegra_cmac_reqctx *rctx = ahash_request_ctx(req); 1753 1753 + int ret; 1753 1754 1754 1754 - tegra_cmac_init(req); 1755 1755 + ret = tegra_cmac_init(req); 1756 1756 + if (ret) 1757 1757 + return ret; 1758 1758 + 1755 1759 rctx->task |= SHA_UPDATE | SHA_FINAL; 1756 1756 - 1757 1760 return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); 1758 1761 } 1759 1762

+5 -2

drivers/crypto/tegra/tegra-se-hash.c

reviewed

··· 615 615 struct tegra_sha_reqctx *rctx = ahash_request_ctx(req); 616 616 struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 617 617 struct tegra_sha_ctx *ctx = crypto_ahash_ctx(tfm); 618 618 + int ret; 618 619 619 620 if (ctx->fallback) 620 621 return tegra_sha_fallback_digest(req); 621 622 622 622 - tegra_sha_init(req); 623 623 - rctx->task |= SHA_UPDATE | SHA_FINAL; 623 623 + ret = tegra_sha_init(req); 624 624 + if (ret) 625 625 + return ret; 624 626 627 627 + rctx->task |= SHA_UPDATE | SHA_FINAL; 625 628 return crypto_transfer_hash_request_to_engine(ctx->se->engine, req); 626 629 } 627 630

+1 -5

include/crypto/gf128mul.h

reviewed

··· 158 158 64...71 72...79 80...87 88...95 96..103 104.111 112.119 120.127 159 159 */ 160 160 161 161 - /* A slow generic version of gf_mul, implemented for lle and bbe 161 161 + /* A slow generic version of gf_mul, implemented for lle 162 162 * It multiplies a and b and puts the result in a */ 163 163 void gf128mul_lle(be128 *a, const be128 *b); 164 164 - 165 165 - void gf128mul_bbe(be128 *a, const be128 *b); 166 164 167 165 /* 168 166 * The following functions multiply a field element by x in ··· 222 224 }; 223 225 224 226 struct gf128mul_4k *gf128mul_init_4k_lle(const be128 *g); 225 225 - struct gf128mul_4k *gf128mul_init_4k_bbe(const be128 *g); 226 227 void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t); 227 227 - void gf128mul_4k_bbe(be128 *a, const struct gf128mul_4k *t); 228 228 void gf128mul_x8_ble(le128 *r, const le128 *x); 229 229 static inline void gf128mul_free_4k(struct gf128mul_4k *t) 230 230 {

-23

include/crypto/internal/hash.h

reviewed

··· 12 12 #include <crypto/hash.h> 13 13 14 14 struct ahash_request; 15 15 - struct scatterlist; 16 16 - 17 17 - struct crypto_hash_walk { 18 18 - char *data; 19 19 - 20 20 - unsigned int offset; 21 21 - unsigned int flags; 22 22 - 23 23 - struct page *pg; 24 24 - unsigned int entrylen; 25 25 - 26 26 - unsigned int total; 27 27 - struct scatterlist *sg; 28 28 - }; 29 15 30 16 struct ahash_instance { 31 17 void (*free)(struct ahash_instance *inst); ··· 42 56 struct crypto_shash_spawn { 43 57 struct crypto_spawn base; 44 58 }; 45 45 - 46 46 - int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err); 47 47 - int crypto_hash_walk_first(struct ahash_request *req, 48 48 - struct crypto_hash_walk *walk); 49 49 - 50 50 - static inline int crypto_hash_walk_last(struct crypto_hash_walk *walk) 51 51 - { 52 52 - return !(walk->entrylen | walk->total); 53 53 - } 54 59 55 60 int crypto_register_ahash(struct ahash_alg *alg); 56 61 void crypto_unregister_ahash(struct ahash_alg *alg);

+1 -13

include/crypto/internal/skcipher.h

reviewed

··· 11 11 #include <crypto/algapi.h> 12 12 #include <crypto/internal/cipher.h> 13 13 #include <crypto/skcipher.h> 14 14 - #include <linux/list.h> 15 14 #include <linux/types.h> 16 15 17 16 /* ··· 57 58 struct skcipher_walk { 58 59 union { 59 60 struct { 60 60 - struct page *page; 61 61 - unsigned long offset; 62 62 - } phys; 63 63 - 64 64 - struct { 65 65 - u8 *page; 66 61 void *addr; 67 62 } virt; 68 63 } src, dst; ··· 66 73 67 74 struct scatter_walk out; 68 75 unsigned int total; 69 69 - 70 70 - struct list_head buffers; 71 76 72 77 u8 *page; 73 78 u8 *buffer; ··· 196 205 int lskcipher_register_instance(struct crypto_template *tmpl, 197 206 struct lskcipher_instance *inst); 198 207 199 199 - int skcipher_walk_done(struct skcipher_walk *walk, int err); 208 208 + int skcipher_walk_done(struct skcipher_walk *walk, int res); 200 209 int skcipher_walk_virt(struct skcipher_walk *walk, 201 210 struct skcipher_request *req, 202 211 bool atomic); 203 203 - int skcipher_walk_async(struct skcipher_walk *walk, 204 204 - struct skcipher_request *req); 205 212 int skcipher_walk_aead_encrypt(struct skcipher_walk *walk, 206 213 struct aead_request *req, bool atomic); 207 214 int skcipher_walk_aead_decrypt(struct skcipher_walk *walk, 208 215 struct aead_request *req, bool atomic); 209 209 - void skcipher_walk_complete(struct skcipher_walk *walk, int err); 210 216 211 217 static inline void skcipher_walk_abort(struct skcipher_walk *walk) 212 218 {

+8

include/linux/hisi_acc_qm.h

reviewed

··· 97 97 /* page number for queue file region */ 98 98 #define QM_DOORBELL_PAGE_NR 1 99 99 100 100 + #define QM_DEV_ALG_MAX_LEN 256 101 101 + 100 102 /* uacce mode of the driver */ 101 103 #define UACCE_MODE_NOUACCE 0 /* don't use uacce */ 102 104 #define UACCE_MODE_SVA 1 /* use uacce sva mode */ ··· 124 122 QM_HW_V1 = 0x20, 125 123 QM_HW_V2 = 0x21, 126 124 QM_HW_V3 = 0x30, 125 125 + QM_HW_V4 = 0x50, 127 126 }; 128 127 129 128 enum qm_fun_type { ··· 159 156 QM_SUPPORT_MB_COMMAND, 160 157 QM_SUPPORT_SVA_PREFETCH, 161 158 QM_SUPPORT_RPM, 159 159 + QM_SUPPORT_DAE, 162 160 }; 163 161 164 162 struct qm_dev_alg { ··· 270 266 void (*show_last_dfx_regs)(struct hisi_qm *qm); 271 267 void (*err_info_init)(struct hisi_qm *qm); 272 268 enum acc_err_result (*get_err_result)(struct hisi_qm *qm); 269 269 + bool (*dev_is_abnormal)(struct hisi_qm *qm); 270 270 + int (*set_priv_status)(struct hisi_qm *qm); 273 271 }; 274 272 275 273 struct hisi_qm_cap_info { ··· 397 391 struct hisi_qm_poll_data *poll_data; 398 392 399 393 struct mutex mailbox_lock; 394 394 + 395 395 + struct mutex ifc_lock; 400 396 401 397 const struct hisi_qm_hw_ops *ops; 402 398

-2

include/linux/verification.h

reviewed

··· 38 38 VERIFYING_UNSPECIFIED_SIGNATURE, 39 39 NR__KEY_BEING_USED_FOR 40 40 }; 41 41 - extern const char *const key_being_used_for[NR__KEY_BEING_USED_FOR]; 42 42 - 43 41 #ifdef CONFIG_SYSTEM_DATA_VERIFICATION 44 42 45 43 struct key;

+36 -9

kernel/padata.c

reviewed

··· 47 47 static void padata_free_pd(struct parallel_data *pd); 48 48 static void __init padata_mt_helper(struct work_struct *work); 49 49 50 50 + static inline void padata_get_pd(struct parallel_data *pd) 51 51 + { 52 52 + refcount_inc(&pd->refcnt); 53 53 + } 54 54 + 55 55 + static inline void padata_put_pd_cnt(struct parallel_data *pd, int cnt) 56 56 + { 57 57 + if (refcount_sub_and_test(cnt, &pd->refcnt)) 58 58 + padata_free_pd(pd); 59 59 + } 60 60 + 61 61 + static inline void padata_put_pd(struct parallel_data *pd) 62 62 + { 63 63 + padata_put_pd_cnt(pd, 1); 64 64 + } 65 65 + 50 66 static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index) 51 67 { 52 68 int cpu, target_cpu; ··· 222 206 if ((pinst->flags & PADATA_RESET)) 223 207 goto out; 224 208 225 225 - refcount_inc(&pd->refcnt); 209 209 + padata_get_pd(pd); 226 210 padata->pd = pd; 227 211 padata->cb_cpu = *cb_cpu; 228 212 ··· 352 336 smp_mb(); 353 337 354 338 reorder = per_cpu_ptr(pd->reorder_list, pd->cpu); 355 355 - if (!list_empty(&reorder->list) && padata_find_next(pd, false)) 339 339 + if (!list_empty(&reorder->list) && padata_find_next(pd, false)) { 340 340 + /* 341 341 + * Other context(eg. the padata_serial_worker) can finish the request. 342 342 + * To avoid UAF issue, add pd ref here, and put pd ref after reorder_work finish. 343 343 + */ 344 344 + padata_get_pd(pd); 356 345 queue_work(pinst->serial_wq, &pd->reorder_work); 346 346 + } 357 347 } 358 348 359 349 static void invoke_padata_reorder(struct work_struct *work) ··· 370 348 pd = container_of(work, struct parallel_data, reorder_work); 371 349 padata_reorder(pd); 372 350 local_bh_enable(); 351 351 + /* Pairs with putting the reorder_work in the serial_wq */ 352 352 + padata_put_pd(pd); 373 353 } 374 354 375 355 static void padata_serial_worker(struct work_struct *serial_work) ··· 404 380 } 405 381 local_bh_enable(); 406 382 407 407 - if (refcount_sub_and_test(cnt, &pd->refcnt)) 408 408 - padata_free_pd(pd); 383 383 + padata_put_pd_cnt(pd, cnt); 409 384 } 410 385 411 386 /** ··· 704 681 synchronize_rcu(); 705 682 706 683 list_for_each_entry_continue_reverse(ps, &pinst->pslist, list) 707 707 - if (refcount_dec_and_test(&ps->opd->refcnt)) 708 708 - padata_free_pd(ps->opd); 684 684 + padata_put_pd(ps->opd); 709 685 710 686 pinst->flags &= ~PADATA_RESET; 711 687 ··· 992 970 993 971 pinst = kobj2pinst(kobj); 994 972 pentry = attr2pentry(attr); 995 995 - if (pentry->show) 973 973 + if (pentry->store) 996 974 ret = pentry->store(pinst, attr, buf, count); 997 975 998 976 return ret; ··· 1143 1121 if (!ps) 1144 1122 return; 1145 1123 1124 1124 + /* 1125 1125 + * Wait for all _do_serial calls to finish to avoid touching 1126 1126 + * freed pd's and ps's. 1127 1127 + */ 1128 1128 + synchronize_rcu(); 1129 1129 + 1146 1130 mutex_lock(&ps->pinst->lock); 1147 1131 list_del(&ps->list); 1148 1132 pd = rcu_dereference_protected(ps->pd, 1); 1149 1149 - if (refcount_dec_and_test(&pd->refcnt)) 1150 1150 - padata_free_pd(pd); 1133 1133 + padata_put_pd(pd); 1151 1134 mutex_unlock(&ps->pinst->lock); 1152 1135 1153 1136 kfree(ps);

+1 -1

lib/crypto/aesgcm.c

reviewed

··· 697 697 u8 tagbuf[AES_BLOCK_SIZE]; 698 698 int plen = aesgcm_tv[i].plen; 699 699 struct aesgcm_ctx ctx; 700 700 - u8 buf[sizeof(ptext12)]; 700 700 + static u8 buf[sizeof(ptext12)]; 701 701 702 702 if (aesgcm_expandkey(&ctx, aesgcm_tv[i].key, aesgcm_tv[i].klen, 703 703 aesgcm_tv[i].clen - plen)) {

-75

lib/crypto/gf128mul.c

reviewed

··· 225 225 } 226 226 EXPORT_SYMBOL(gf128mul_lle); 227 227 228 228 - void gf128mul_bbe(be128 *r, const be128 *b) 229 229 - { 230 230 - be128 p[8]; 231 231 - int i; 232 232 - 233 233 - p[0] = *r; 234 234 - for (i = 0; i < 7; ++i) 235 235 - gf128mul_x_bbe(&p[i + 1], &p[i]); 236 236 - 237 237 - memset(r, 0, sizeof(*r)); 238 238 - for (i = 0;;) { 239 239 - u8 ch = ((u8 *)b)[i]; 240 240 - 241 241 - if (ch & 0x80) 242 242 - be128_xor(r, r, &p[7]); 243 243 - if (ch & 0x40) 244 244 - be128_xor(r, r, &p[6]); 245 245 - if (ch & 0x20) 246 246 - be128_xor(r, r, &p[5]); 247 247 - if (ch & 0x10) 248 248 - be128_xor(r, r, &p[4]); 249 249 - if (ch & 0x08) 250 250 - be128_xor(r, r, &p[3]); 251 251 - if (ch & 0x04) 252 252 - be128_xor(r, r, &p[2]); 253 253 - if (ch & 0x02) 254 254 - be128_xor(r, r, &p[1]); 255 255 - if (ch & 0x01) 256 256 - be128_xor(r, r, &p[0]); 257 257 - 258 258 - if (++i >= 16) 259 259 - break; 260 260 - 261 261 - gf128mul_x8_bbe(r); 262 262 - } 263 263 - } 264 264 - EXPORT_SYMBOL(gf128mul_bbe); 265 265 - 266 228 /* This version uses 64k bytes of table space. 267 229 A 16 byte buffer has to be multiplied by a 16 byte key 268 230 value in GF(2^128). If we consider a GF(2^128) value in ··· 342 380 } 343 381 EXPORT_SYMBOL(gf128mul_init_4k_lle); 344 382 345 345 - struct gf128mul_4k *gf128mul_init_4k_bbe(const be128 *g) 346 346 - { 347 347 - struct gf128mul_4k *t; 348 348 - int j, k; 349 349 - 350 350 - t = kzalloc(sizeof(*t), GFP_KERNEL); 351 351 - if (!t) 352 352 - goto out; 353 353 - 354 354 - t->t[1] = *g; 355 355 - for (j = 1; j <= 64; j <<= 1) 356 356 - gf128mul_x_bbe(&t->t[j + j], &t->t[j]); 357 357 - 358 358 - for (j = 2; j < 256; j += j) 359 359 - for (k = 1; k < j; ++k) 360 360 - be128_xor(&t->t[j + k], &t->t[j], &t->t[k]); 361 361 - 362 362 - out: 363 363 - return t; 364 364 - } 365 365 - EXPORT_SYMBOL(gf128mul_init_4k_bbe); 366 366 - 367 383 void gf128mul_4k_lle(be128 *a, const struct gf128mul_4k *t) 368 384 { 369 385 u8 *ap = (u8 *)a; ··· 356 416 *a = *r; 357 417 } 358 418 EXPORT_SYMBOL(gf128mul_4k_lle); 359 359 - 360 360 - void gf128mul_4k_bbe(be128 *a, const struct gf128mul_4k *t) 361 361 - { 362 362 - u8 *ap = (u8 *)a; 363 363 - be128 r[1]; 364 364 - int i = 0; 365 365 - 366 366 - *r = t->t[ap[0]]; 367 367 - while (++i < 16) { 368 368 - gf128mul_x8_bbe(r); 369 369 - be128_xor(r, r, &t->t[ap[i]]); 370 370 - } 371 371 - *a = *r; 372 372 - } 373 373 - EXPORT_SYMBOL(gf128mul_4k_bbe); 374 419 375 420 MODULE_LICENSE("GPL"); 376 421 MODULE_DESCRIPTION("Functions for multiplying elements of GF(2^128)");

+8 -4

lib/rhashtable.c

reviewed

··· 584 584 */ 585 585 rht_assign_locked(bkt, obj); 586 586 587 587 - atomic_inc(&ht->nelems); 588 588 - if (rht_grow_above_75(ht, tbl)) 589 589 - schedule_work(&ht->run_work); 590 590 - 591 587 return NULL; 592 588 } 593 589 ··· 611 615 new_tbl = rht_dereference_rcu(tbl->future_tbl, ht); 612 616 data = ERR_PTR(-EAGAIN); 613 617 } else { 618 618 + bool inserted; 619 619 + 614 620 flags = rht_lock(tbl, bkt); 615 621 data = rhashtable_lookup_one(ht, bkt, tbl, 616 622 hash, key, obj); 617 623 new_tbl = rhashtable_insert_one(ht, bkt, tbl, 618 624 hash, obj, data); 625 625 + inserted = data && !new_tbl; 626 626 + if (inserted) 627 627 + atomic_inc(&ht->nelems); 619 628 if (PTR_ERR(new_tbl) != -EEXIST) 620 629 data = ERR_CAST(new_tbl); 621 630 622 631 rht_unlock(tbl, bkt, flags); 632 632 + 633 633 + if (inserted && rht_grow_above_75(ht, tbl)) 634 634 + schedule_work(&ht->run_work); 623 635 } 624 636 } while (!IS_ERR_OR_NULL(new_tbl)); 625 637