Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

crypto: starfive - Add hash and HMAC support

Adding hash/HMAC support for SHA-2 and SM3 to StarFive cryptographic
module.

Co-developed-by: Huan Feng <huan.feng@starfivetech.com>
Signed-off-by: Huan Feng <huan.feng@starfivetech.com>
Signed-off-by: Jia Jie Ho <jiajie.ho@starfivetech.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

authored by

Jia Jie Ho and committed by
Herbert Xu
7883d1b2 42ef0e94

+1003 -4
+4
drivers/crypto/starfive/Kconfig
··· 6 6 tristate "StarFive JH7110 cryptographic engine driver" 7 7 depends on SOC_STARFIVE || COMPILE_TEST 8 8 select CRYPTO_ENGINE 9 + select CRYPTO_HMAC 10 + select CRYPTO_SHA256 11 + select CRYPTO_SHA512 12 + select CRYPTO_SM3_GENERIC 9 13 select ARM_AMBA 10 14 select DMADEVICES 11 15 select AMBA_PL08X
+1 -1
drivers/crypto/starfive/Makefile
··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 2 3 3 obj-$(CONFIG_CRYPTO_DEV_JH7110) += jh7110-crypto.o 4 - jh7110-crypto-objs := jh7110-cryp.o 4 + jh7110-crypto-objs := jh7110-cryp.o jh7110-hash.o
+39
drivers/crypto/starfive/jh7110-cryp.c
··· 79 79 dma_release_channel(cryp->rx); 80 80 } 81 81 82 + static irqreturn_t starfive_cryp_irq(int irq, void *priv) 83 + { 84 + u32 status; 85 + struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)priv; 86 + 87 + status = readl(cryp->base + STARFIVE_IE_FLAG_OFFSET); 88 + if (status & STARFIVE_IE_FLAG_HASH_DONE) { 89 + writel(STARFIVE_IE_MASK_HASH_DONE, cryp->base + STARFIVE_IE_MASK_OFFSET); 90 + tasklet_schedule(&cryp->hash_done); 91 + } 92 + 93 + return IRQ_HANDLED; 94 + } 95 + 82 96 static int starfive_cryp_probe(struct platform_device *pdev) 83 97 { 84 98 struct starfive_cryp_dev *cryp; 85 99 struct resource *res; 100 + int irq; 86 101 int ret; 87 102 88 103 cryp = devm_kzalloc(&pdev->dev, sizeof(*cryp), GFP_KERNEL); ··· 111 96 if (IS_ERR(cryp->base)) 112 97 return dev_err_probe(&pdev->dev, PTR_ERR(cryp->base), 113 98 "Error remapping memory for platform device\n"); 99 + 100 + tasklet_init(&cryp->hash_done, starfive_hash_done_task, (unsigned long)cryp); 114 101 115 102 cryp->phys_base = res->start; 116 103 cryp->dma_maxburst = 32; ··· 131 114 if (IS_ERR(cryp->rst)) 132 115 return dev_err_probe(&pdev->dev, PTR_ERR(cryp->rst), 133 116 "Error getting hardware reset line\n"); 117 + 118 + irq = platform_get_irq(pdev, 0); 119 + if (irq < 0) 120 + return irq; 121 + 122 + ret = devm_request_irq(&pdev->dev, irq, starfive_cryp_irq, 0, pdev->name, 123 + (void *)cryp); 124 + if (ret) 125 + return dev_err_probe(&pdev->dev, irq, 126 + "Failed to register interrupt handler\n"); 134 127 135 128 clk_prepare_enable(cryp->hclk); 136 129 clk_prepare_enable(cryp->ahb); ··· 169 142 if (ret) 170 143 goto err_engine_start; 171 144 145 + ret = starfive_hash_register_algs(); 146 + if (ret) 147 + goto err_algs_hash; 148 + 172 149 return 0; 173 150 151 + err_algs_hash: 152 + crypto_engine_stop(cryp->engine); 174 153 err_engine_start: 175 154 crypto_engine_exit(cryp->engine); 176 155 err_engine: ··· 189 156 clk_disable_unprepare(cryp->hclk); 190 157 clk_disable_unprepare(cryp->ahb); 191 158 reset_control_assert(cryp->rst); 159 + 160 + tasklet_kill(&cryp->hash_done); 192 161 err_probe_defer: 193 162 return ret; 194 163 } ··· 198 163 static int starfive_cryp_remove(struct platform_device *pdev) 199 164 { 200 165 struct starfive_cryp_dev *cryp = platform_get_drvdata(pdev); 166 + 167 + starfive_hash_unregister_algs(); 168 + 169 + tasklet_kill(&cryp->hash_done); 201 170 202 171 crypto_engine_stop(cryp->engine); 203 172 crypto_engine_exit(cryp->engine);
+67 -3
drivers/crypto/starfive/jh7110-cryp.h
··· 7 7 #include <linux/dmaengine.h> 8 8 9 9 #include <crypto/engine.h> 10 + #include <crypto/sha2.h> 11 + #include <crypto/sm3.h> 10 12 11 13 #define STARFIVE_ALG_CR_OFFSET 0x0 12 14 #define STARFIVE_ALG_FIFO_OFFSET 0x4 ··· 17 15 #define STARFIVE_DMA_IN_LEN_OFFSET 0x10 18 16 #define STARFIVE_DMA_OUT_LEN_OFFSET 0x14 19 17 18 + #define STARFIVE_IE_MASK_HASH_DONE 0x4 19 + #define STARFIVE_IE_FLAG_HASH_DONE 0x4 20 + 20 21 #define STARFIVE_MSG_BUFFER_SIZE SZ_16K 22 + #define MAX_KEY_SIZE SHA512_BLOCK_SIZE 23 + 24 + union starfive_hash_csr { 25 + u32 v; 26 + struct { 27 + u32 start :1; 28 + u32 reset :1; 29 + u32 ie :1; 30 + u32 firstb :1; 31 + #define STARFIVE_HASH_SM3 0x0 32 + #define STARFIVE_HASH_SHA224 0x3 33 + #define STARFIVE_HASH_SHA256 0x4 34 + #define STARFIVE_HASH_SHA384 0x5 35 + #define STARFIVE_HASH_SHA512 0x6 36 + #define STARFIVE_HASH_MODE_MASK 0x7 37 + u32 mode :3; 38 + u32 rsvd_1 :1; 39 + u32 final :1; 40 + u32 rsvd_2 :2; 41 + #define STARFIVE_HASH_HMAC_FLAGS 0x800 42 + u32 hmac :1; 43 + u32 rsvd_3 :1; 44 + #define STARFIVE_HASH_KEY_DONE BIT(13) 45 + u32 key_done :1; 46 + u32 key_flag :1; 47 + u32 hmac_done :1; 48 + #define STARFIVE_HASH_BUSY BIT(16) 49 + u32 busy :1; 50 + u32 hashdone :1; 51 + u32 rsvd_4 :14; 52 + }; 53 + }; 54 + 21 55 22 56 union starfive_alg_cr { 23 57 u32 v; ··· 72 34 struct starfive_cryp_ctx { 73 35 struct crypto_engine_ctx enginectx; 74 36 struct starfive_cryp_dev *cryp; 37 + struct starfive_cryp_request_ctx *rctx; 38 + 39 + unsigned int hash_mode; 40 + u8 key[MAX_KEY_SIZE]; 41 + int keylen; 42 + bool is_hmac; 43 + struct crypto_ahash *ahash_fbk; 75 44 }; 76 45 77 46 struct starfive_cryp_dev { 78 47 struct list_head list; 79 48 struct device *dev; 80 - 81 49 struct clk *hclk; 82 50 struct clk *ahb; 83 51 struct reset_control *rst; ··· 96 52 struct dma_chan *rx; 97 53 struct dma_slave_config cfg_in; 98 54 struct dma_slave_config cfg_out; 99 - 100 55 struct crypto_engine *engine; 101 - 56 + struct tasklet_struct hash_done; 57 + int err; 102 58 union starfive_alg_cr alg_cr; 59 + union { 60 + struct ahash_request *hreq; 61 + } req; 62 + }; 63 + 64 + struct starfive_cryp_request_ctx { 65 + union { 66 + union starfive_hash_csr hash; 67 + } csr; 68 + 69 + struct scatterlist *in_sg; 70 + struct ahash_request ahash_fbk_req; 71 + size_t total; 72 + unsigned int blksize; 73 + unsigned int digsize; 74 + unsigned long in_sg_len; 103 75 }; 104 76 105 77 struct starfive_cryp_dev *starfive_cryp_find_dev(struct starfive_cryp_ctx *ctx); 106 78 79 + int starfive_hash_register_algs(void); 80 + void starfive_hash_unregister_algs(void); 81 + 82 + void starfive_hash_done_task(unsigned long param); 107 83 #endif
+892
drivers/crypto/starfive/jh7110-hash.c
··· 1 + // SPDX-License-Identifier: GPL-2.0 2 + /* 3 + * Hash function and HMAC support for StarFive driver 4 + * 5 + * Copyright (c) 2022 StarFive Technology 6 + * 7 + */ 8 + 9 + #include <linux/clk.h> 10 + #include <linux/crypto.h> 11 + #include <linux/dma-direct.h> 12 + #include <linux/interrupt.h> 13 + #include <linux/io.h> 14 + #include <linux/iopoll.h> 15 + #include <linux/kernel.h> 16 + #include <linux/module.h> 17 + #include <linux/of_device.h> 18 + #include <linux/platform_device.h> 19 + #include <linux/pm_runtime.h> 20 + #include <linux/reset.h> 21 + #include <linux/amba/pl080.h> 22 + 23 + #include <crypto/hash.h> 24 + #include <crypto/scatterwalk.h> 25 + #include <crypto/internal/hash.h> 26 + 27 + #include "jh7110-cryp.h" 28 + 29 + #define STARFIVE_HASH_REGS_OFFSET 0x300 30 + #define STARFIVE_HASH_SHACSR (STARFIVE_HASH_REGS_OFFSET + 0x0) 31 + #define STARFIVE_HASH_SHAWDR (STARFIVE_HASH_REGS_OFFSET + 0x4) 32 + #define STARFIVE_HASH_SHARDR (STARFIVE_HASH_REGS_OFFSET + 0x8) 33 + #define STARFIVE_HASH_SHAWSR (STARFIVE_HASH_REGS_OFFSET + 0xC) 34 + #define STARFIVE_HASH_SHAWLEN3 (STARFIVE_HASH_REGS_OFFSET + 0x10) 35 + #define STARFIVE_HASH_SHAWLEN2 (STARFIVE_HASH_REGS_OFFSET + 0x14) 36 + #define STARFIVE_HASH_SHAWLEN1 (STARFIVE_HASH_REGS_OFFSET + 0x18) 37 + #define STARFIVE_HASH_SHAWLEN0 (STARFIVE_HASH_REGS_OFFSET + 0x1C) 38 + #define STARFIVE_HASH_SHAWKR (STARFIVE_HASH_REGS_OFFSET + 0x20) 39 + #define STARFIVE_HASH_SHAWKLEN (STARFIVE_HASH_REGS_OFFSET + 0x24) 40 + 41 + #define STARFIVE_HASH_BUFLEN SHA512_BLOCK_SIZE 42 + 43 + static inline int starfive_hash_wait_busy(struct starfive_cryp_ctx *ctx) 44 + { 45 + struct starfive_cryp_dev *cryp = ctx->cryp; 46 + u32 status; 47 + 48 + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status, 49 + !(status & STARFIVE_HASH_BUSY), 10, 100000); 50 + } 51 + 52 + static inline int starfive_hash_wait_key_done(struct starfive_cryp_ctx *ctx) 53 + { 54 + struct starfive_cryp_dev *cryp = ctx->cryp; 55 + u32 status; 56 + 57 + return readl_relaxed_poll_timeout(cryp->base + STARFIVE_HASH_SHACSR, status, 58 + (status & STARFIVE_HASH_KEY_DONE), 10, 100000); 59 + } 60 + 61 + static int starfive_hash_hmac_key(struct starfive_cryp_ctx *ctx) 62 + { 63 + struct starfive_cryp_request_ctx *rctx = ctx->rctx; 64 + struct starfive_cryp_dev *cryp = ctx->cryp; 65 + int klen = ctx->keylen, loop; 66 + unsigned int *key = (unsigned int *)ctx->key; 67 + unsigned char *cl; 68 + 69 + writel(ctx->keylen, cryp->base + STARFIVE_HASH_SHAWKLEN); 70 + 71 + rctx->csr.hash.hmac = 1; 72 + rctx->csr.hash.key_flag = 1; 73 + 74 + writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR); 75 + 76 + for (loop = 0; loop < klen / sizeof(unsigned int); loop++, key++) 77 + writel(*key, cryp->base + STARFIVE_HASH_SHAWKR); 78 + 79 + if (klen & 0x3) { 80 + cl = (unsigned char *)key; 81 + for (loop = 0; loop < (klen & 0x3); loop++, cl++) 82 + writeb(*cl, cryp->base + STARFIVE_HASH_SHAWKR); 83 + } 84 + 85 + if (starfive_hash_wait_key_done(ctx)) 86 + return dev_err_probe(cryp->dev, -ETIMEDOUT, "starfive_hash_wait_key_done error\n"); 87 + 88 + return 0; 89 + } 90 + 91 + static void starfive_hash_start(void *param) 92 + { 93 + struct starfive_cryp_ctx *ctx = param; 94 + struct starfive_cryp_request_ctx *rctx = ctx->rctx; 95 + struct starfive_cryp_dev *cryp = ctx->cryp; 96 + union starfive_alg_cr alg_cr; 97 + union starfive_hash_csr csr; 98 + 99 + dma_unmap_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE); 100 + 101 + alg_cr.v = 0; 102 + alg_cr.clear = 1; 103 + 104 + writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); 105 + 106 + csr.v = readl(cryp->base + STARFIVE_HASH_SHACSR); 107 + csr.firstb = 0; 108 + csr.final = 1; 109 + 110 + writel(~STARFIVE_IE_MASK_HASH_DONE, cryp->base + STARFIVE_IE_MASK_OFFSET); 111 + writel(csr.v, cryp->base + STARFIVE_HASH_SHACSR); 112 + } 113 + 114 + static int starfive_hash_xmit_dma(struct starfive_cryp_ctx *ctx) 115 + { 116 + struct starfive_cryp_request_ctx *rctx = ctx->rctx; 117 + struct starfive_cryp_dev *cryp = ctx->cryp; 118 + struct dma_async_tx_descriptor *in_desc; 119 + union starfive_alg_cr alg_cr; 120 + int total_len; 121 + int ret; 122 + 123 + if (!rctx->total) { 124 + starfive_hash_start(ctx); 125 + return 0; 126 + } 127 + 128 + writel(rctx->total, cryp->base + STARFIVE_DMA_IN_LEN_OFFSET); 129 + 130 + total_len = rctx->total; 131 + total_len = (total_len & 0x3) ? (((total_len >> 2) + 1) << 2) : total_len; 132 + sg_dma_len(rctx->in_sg) = total_len; 133 + 134 + alg_cr.v = 0; 135 + alg_cr.start = 1; 136 + alg_cr.hash_dma_en = 1; 137 + 138 + writel(alg_cr.v, cryp->base + STARFIVE_ALG_CR_OFFSET); 139 + 140 + ret = dma_map_sg(cryp->dev, rctx->in_sg, rctx->in_sg_len, DMA_TO_DEVICE); 141 + if (!ret) 142 + return dev_err_probe(cryp->dev, -EINVAL, "dma_map_sg() error\n"); 143 + 144 + cryp->cfg_in.direction = DMA_MEM_TO_DEV; 145 + cryp->cfg_in.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 146 + cryp->cfg_in.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 147 + cryp->cfg_in.src_maxburst = cryp->dma_maxburst; 148 + cryp->cfg_in.dst_maxburst = cryp->dma_maxburst; 149 + cryp->cfg_in.dst_addr = cryp->phys_base + STARFIVE_ALG_FIFO_OFFSET; 150 + 151 + dmaengine_slave_config(cryp->tx, &cryp->cfg_in); 152 + 153 + in_desc = dmaengine_prep_slave_sg(cryp->tx, rctx->in_sg, 154 + ret, DMA_MEM_TO_DEV, 155 + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 156 + 157 + if (!in_desc) 158 + return -EINVAL; 159 + 160 + in_desc->callback = starfive_hash_start; 161 + in_desc->callback_param = ctx; 162 + 163 + dmaengine_submit(in_desc); 164 + dma_async_issue_pending(cryp->tx); 165 + 166 + return 0; 167 + } 168 + 169 + static int starfive_hash_xmit(struct starfive_cryp_ctx *ctx) 170 + { 171 + struct starfive_cryp_request_ctx *rctx = ctx->rctx; 172 + struct starfive_cryp_dev *cryp = ctx->cryp; 173 + int ret = 0; 174 + 175 + rctx->csr.hash.v = 0; 176 + rctx->csr.hash.reset = 1; 177 + writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR); 178 + 179 + if (starfive_hash_wait_busy(ctx)) 180 + return dev_err_probe(cryp->dev, -ETIMEDOUT, "Error resetting engine.\n"); 181 + 182 + rctx->csr.hash.v = 0; 183 + rctx->csr.hash.mode = ctx->hash_mode; 184 + rctx->csr.hash.ie = 1; 185 + 186 + if (ctx->is_hmac) { 187 + ret = starfive_hash_hmac_key(ctx); 188 + if (ret) 189 + return ret; 190 + } else { 191 + rctx->csr.hash.start = 1; 192 + rctx->csr.hash.firstb = 1; 193 + writel(rctx->csr.hash.v, cryp->base + STARFIVE_HASH_SHACSR); 194 + } 195 + 196 + return starfive_hash_xmit_dma(ctx); 197 + } 198 + 199 + static int starfive_hash_copy_hash(struct ahash_request *req) 200 + { 201 + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); 202 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); 203 + int count, *data; 204 + int mlen; 205 + 206 + if (!req->result) 207 + return 0; 208 + 209 + mlen = rctx->digsize / sizeof(u32); 210 + data = (u32 *)req->result; 211 + 212 + for (count = 0; count < mlen; count++) 213 + data[count] = readl(ctx->cryp->base + STARFIVE_HASH_SHARDR); 214 + 215 + return 0; 216 + } 217 + 218 + void starfive_hash_done_task(unsigned long param) 219 + { 220 + struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param; 221 + int err = cryp->err; 222 + 223 + if (!err) 224 + err = starfive_hash_copy_hash(cryp->req.hreq); 225 + 226 + crypto_finalize_hash_request(cryp->engine, cryp->req.hreq, err); 227 + } 228 + 229 + static int starfive_hash_check_aligned(struct scatterlist *sg, size_t total, size_t align) 230 + { 231 + int len = 0; 232 + 233 + if (!total) 234 + return 0; 235 + 236 + if (!IS_ALIGNED(total, align)) 237 + return -EINVAL; 238 + 239 + while (sg) { 240 + if (!IS_ALIGNED(sg->offset, sizeof(u32))) 241 + return -EINVAL; 242 + 243 + if (!IS_ALIGNED(sg->length, align)) 244 + return -EINVAL; 245 + 246 + len += sg->length; 247 + sg = sg_next(sg); 248 + } 249 + 250 + if (len != total) 251 + return -EINVAL; 252 + 253 + return 0; 254 + } 255 + 256 + static int starfive_hash_one_request(struct crypto_engine *engine, void *areq) 257 + { 258 + struct ahash_request *req = container_of(areq, struct ahash_request, 259 + base); 260 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); 261 + struct starfive_cryp_dev *cryp = ctx->cryp; 262 + 263 + if (!cryp) 264 + return -ENODEV; 265 + 266 + return starfive_hash_xmit(ctx); 267 + } 268 + 269 + static int starfive_hash_init(struct ahash_request *req) 270 + { 271 + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 272 + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); 273 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); 274 + 275 + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); 276 + ahash_request_set_callback(&rctx->ahash_fbk_req, 277 + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, 278 + req->base.complete, req->base.data); 279 + 280 + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, 281 + req->result, req->nbytes); 282 + 283 + return crypto_ahash_init(&rctx->ahash_fbk_req); 284 + } 285 + 286 + static int starfive_hash_update(struct ahash_request *req) 287 + { 288 + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); 289 + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 290 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); 291 + 292 + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); 293 + ahash_request_set_callback(&rctx->ahash_fbk_req, 294 + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, 295 + req->base.complete, req->base.data); 296 + 297 + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, 298 + req->result, req->nbytes); 299 + 300 + return crypto_ahash_update(&rctx->ahash_fbk_req); 301 + } 302 + 303 + static int starfive_hash_final(struct ahash_request *req) 304 + { 305 + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); 306 + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 307 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); 308 + 309 + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); 310 + ahash_request_set_callback(&rctx->ahash_fbk_req, 311 + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, 312 + req->base.complete, req->base.data); 313 + 314 + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, 315 + req->result, req->nbytes); 316 + 317 + return crypto_ahash_final(&rctx->ahash_fbk_req); 318 + } 319 + 320 + static int starfive_hash_finup(struct ahash_request *req) 321 + { 322 + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); 323 + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 324 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); 325 + 326 + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); 327 + ahash_request_set_callback(&rctx->ahash_fbk_req, 328 + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, 329 + req->base.complete, req->base.data); 330 + 331 + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, 332 + req->result, req->nbytes); 333 + 334 + return crypto_ahash_finup(&rctx->ahash_fbk_req); 335 + } 336 + 337 + static int starfive_hash_digest_fb(struct ahash_request *req) 338 + { 339 + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); 340 + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 341 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); 342 + 343 + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); 344 + ahash_request_set_callback(&rctx->ahash_fbk_req, req->base.flags, 345 + req->base.complete, req->base.data); 346 + 347 + ahash_request_set_crypt(&rctx->ahash_fbk_req, req->src, 348 + req->result, req->nbytes); 349 + 350 + return crypto_ahash_digest(&rctx->ahash_fbk_req); 351 + } 352 + 353 + static int starfive_hash_digest(struct ahash_request *req) 354 + { 355 + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 356 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); 357 + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); 358 + struct starfive_cryp_dev *cryp = ctx->cryp; 359 + 360 + memset(rctx, 0, sizeof(struct starfive_cryp_request_ctx)); 361 + 362 + cryp->req.hreq = req; 363 + rctx->total = req->nbytes; 364 + rctx->in_sg = req->src; 365 + rctx->blksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); 366 + rctx->digsize = crypto_ahash_digestsize(tfm); 367 + rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); 368 + ctx->rctx = rctx; 369 + 370 + if (starfive_hash_check_aligned(rctx->in_sg, rctx->total, rctx->blksize)) 371 + return starfive_hash_digest_fb(req); 372 + 373 + return crypto_transfer_hash_request_to_engine(cryp->engine, req); 374 + } 375 + 376 + static int starfive_hash_export(struct ahash_request *req, void *out) 377 + { 378 + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); 379 + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 380 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); 381 + 382 + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); 383 + ahash_request_set_callback(&rctx->ahash_fbk_req, 384 + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, 385 + req->base.complete, req->base.data); 386 + 387 + return crypto_ahash_export(&rctx->ahash_fbk_req, out); 388 + } 389 + 390 + static int starfive_hash_import(struct ahash_request *req, const void *in) 391 + { 392 + struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); 393 + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 394 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); 395 + 396 + ahash_request_set_tfm(&rctx->ahash_fbk_req, ctx->ahash_fbk); 397 + ahash_request_set_callback(&rctx->ahash_fbk_req, 398 + req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, 399 + req->base.complete, req->base.data); 400 + 401 + return crypto_ahash_import(&rctx->ahash_fbk_req, in); 402 + } 403 + 404 + static int starfive_hash_init_tfm(struct crypto_ahash *hash, 405 + const char *alg_name, 406 + unsigned int mode) 407 + { 408 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); 409 + 410 + ctx->cryp = starfive_cryp_find_dev(ctx); 411 + 412 + if (!ctx->cryp) 413 + return -ENODEV; 414 + 415 + ctx->ahash_fbk = crypto_alloc_ahash(alg_name, 0, 416 + CRYPTO_ALG_NEED_FALLBACK); 417 + 418 + if (IS_ERR(ctx->ahash_fbk)) 419 + return dev_err_probe(ctx->cryp->dev, PTR_ERR(ctx->ahash_fbk), 420 + "starfive_hash: Could not load fallback driver.\n"); 421 + 422 + crypto_ahash_set_statesize(hash, crypto_ahash_statesize(ctx->ahash_fbk)); 423 + crypto_ahash_set_reqsize(hash, sizeof(struct starfive_cryp_request_ctx) + 424 + crypto_ahash_reqsize(ctx->ahash_fbk)); 425 + 426 + ctx->keylen = 0; 427 + ctx->hash_mode = mode; 428 + 429 + ctx->enginectx.op.do_one_request = starfive_hash_one_request; 430 + ctx->enginectx.op.prepare_request = NULL; 431 + ctx->enginectx.op.unprepare_request = NULL; 432 + 433 + return 0; 434 + } 435 + 436 + static void starfive_hash_exit_tfm(struct crypto_ahash *hash) 437 + { 438 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); 439 + 440 + crypto_free_ahash(ctx->ahash_fbk); 441 + 442 + ctx->ahash_fbk = NULL; 443 + ctx->enginectx.op.do_one_request = NULL; 444 + ctx->enginectx.op.prepare_request = NULL; 445 + ctx->enginectx.op.unprepare_request = NULL; 446 + } 447 + 448 + static int starfive_hash_long_setkey(struct starfive_cryp_ctx *ctx, 449 + const u8 *key, unsigned int keylen, 450 + const char *alg_name) 451 + { 452 + struct crypto_wait wait; 453 + struct ahash_request *req; 454 + struct scatterlist sg; 455 + struct crypto_ahash *ahash_tfm; 456 + u8 *buf; 457 + int ret; 458 + 459 + ahash_tfm = crypto_alloc_ahash(alg_name, 0, 0); 460 + if (IS_ERR(ahash_tfm)) 461 + return PTR_ERR(ahash_tfm); 462 + 463 + req = ahash_request_alloc(ahash_tfm, GFP_KERNEL); 464 + if (!req) { 465 + ret = -ENOMEM; 466 + goto err_free_ahash; 467 + } 468 + 469 + crypto_init_wait(&wait); 470 + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, 471 + crypto_req_done, &wait); 472 + crypto_ahash_clear_flags(ahash_tfm, ~0); 473 + 474 + buf = kzalloc(keylen + STARFIVE_HASH_BUFLEN, GFP_KERNEL); 475 + if (!buf) { 476 + ret = -ENOMEM; 477 + goto err_free_req; 478 + } 479 + 480 + memcpy(buf, key, keylen); 481 + sg_init_one(&sg, buf, keylen); 482 + ahash_request_set_crypt(req, &sg, ctx->key, keylen); 483 + 484 + ret = crypto_wait_req(crypto_ahash_digest(req), &wait); 485 + 486 + kfree(buf); 487 + err_free_req: 488 + ahash_request_free(req); 489 + err_free_ahash: 490 + crypto_free_ahash(ahash_tfm); 491 + return ret; 492 + } 493 + 494 + static int starfive_hash_setkey(struct crypto_ahash *hash, 495 + const u8 *key, unsigned int keylen) 496 + { 497 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); 498 + unsigned int digestsize = crypto_ahash_digestsize(hash); 499 + unsigned int blocksize = crypto_ahash_blocksize(hash); 500 + const char *alg_name; 501 + 502 + crypto_ahash_setkey(ctx->ahash_fbk, key, keylen); 503 + 504 + if (keylen <= blocksize) { 505 + memcpy(ctx->key, key, keylen); 506 + ctx->keylen = keylen; 507 + return 0; 508 + } 509 + 510 + ctx->keylen = digestsize; 511 + 512 + switch (digestsize) { 513 + case SHA224_DIGEST_SIZE: 514 + alg_name = "sha224-starfive"; 515 + break; 516 + case SHA256_DIGEST_SIZE: 517 + if (ctx->hash_mode == STARFIVE_HASH_SM3) 518 + alg_name = "sm3-starfive"; 519 + else 520 + alg_name = "sha256-starfive"; 521 + break; 522 + case SHA384_DIGEST_SIZE: 523 + alg_name = "sha384-starfive"; 524 + break; 525 + case SHA512_DIGEST_SIZE: 526 + alg_name = "sha512-starfive"; 527 + break; 528 + default: 529 + return -EINVAL; 530 + } 531 + 532 + return starfive_hash_long_setkey(ctx, key, keylen, alg_name); 533 + } 534 + 535 + static int starfive_sha224_init_tfm(struct crypto_ahash *hash) 536 + { 537 + return starfive_hash_init_tfm(hash, "sha224-generic", 538 + STARFIVE_HASH_SHA224); 539 + } 540 + 541 + static int starfive_sha256_init_tfm(struct crypto_ahash *hash) 542 + { 543 + return starfive_hash_init_tfm(hash, "sha256-generic", 544 + STARFIVE_HASH_SHA256); 545 + } 546 + 547 + static int starfive_sha384_init_tfm(struct crypto_ahash *hash) 548 + { 549 + return starfive_hash_init_tfm(hash, "sha384-generic", 550 + STARFIVE_HASH_SHA384); 551 + } 552 + 553 + static int starfive_sha512_init_tfm(struct crypto_ahash *hash) 554 + { 555 + return starfive_hash_init_tfm(hash, "sha512-generic", 556 + STARFIVE_HASH_SHA512); 557 + } 558 + 559 + static int starfive_sm3_init_tfm(struct crypto_ahash *hash) 560 + { 561 + return starfive_hash_init_tfm(hash, "sm3-generic", 562 + STARFIVE_HASH_SM3); 563 + } 564 + 565 + static int starfive_hmac_sha224_init_tfm(struct crypto_ahash *hash) 566 + { 567 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); 568 + 569 + ctx->is_hmac = true; 570 + 571 + return starfive_hash_init_tfm(hash, "hmac(sha224-generic)", 572 + STARFIVE_HASH_SHA224); 573 + } 574 + 575 + static int starfive_hmac_sha256_init_tfm(struct crypto_ahash *hash) 576 + { 577 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); 578 + 579 + ctx->is_hmac = true; 580 + 581 + return starfive_hash_init_tfm(hash, "hmac(sha256-generic)", 582 + STARFIVE_HASH_SHA256); 583 + } 584 + 585 + static int starfive_hmac_sha384_init_tfm(struct crypto_ahash *hash) 586 + { 587 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); 588 + 589 + ctx->is_hmac = true; 590 + 591 + return starfive_hash_init_tfm(hash, "hmac(sha384-generic)", 592 + STARFIVE_HASH_SHA384); 593 + } 594 + 595 + static int starfive_hmac_sha512_init_tfm(struct crypto_ahash *hash) 596 + { 597 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); 598 + 599 + ctx->is_hmac = true; 600 + 601 + return starfive_hash_init_tfm(hash, "hmac(sha512-generic)", 602 + STARFIVE_HASH_SHA512); 603 + } 604 + 605 + static int starfive_hmac_sm3_init_tfm(struct crypto_ahash *hash) 606 + { 607 + struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(hash); 608 + 609 + ctx->is_hmac = true; 610 + 611 + return starfive_hash_init_tfm(hash, "hmac(sm3-generic)", 612 + STARFIVE_HASH_SM3); 613 + } 614 + 615 + static struct ahash_alg algs_sha2_sm3[] = { 616 + { 617 + .init = starfive_hash_init, 618 + .update = starfive_hash_update, 619 + .final = starfive_hash_final, 620 + .finup = starfive_hash_finup, 621 + .digest = starfive_hash_digest, 622 + .export = starfive_hash_export, 623 + .import = starfive_hash_import, 624 + .init_tfm = starfive_sha224_init_tfm, 625 + .exit_tfm = starfive_hash_exit_tfm, 626 + .halg = { 627 + .digestsize = SHA224_DIGEST_SIZE, 628 + .statesize = sizeof(struct sha256_state), 629 + .base = { 630 + .cra_name = "sha224", 631 + .cra_driver_name = "sha224-starfive", 632 + .cra_priority = 200, 633 + .cra_flags = CRYPTO_ALG_ASYNC | 634 + CRYPTO_ALG_TYPE_AHASH | 635 + CRYPTO_ALG_NEED_FALLBACK, 636 + .cra_blocksize = SHA224_BLOCK_SIZE, 637 + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 638 + .cra_alignmask = 3, 639 + .cra_module = THIS_MODULE, 640 + } 641 + } 642 + }, { 643 + .init = starfive_hash_init, 644 + .update = starfive_hash_update, 645 + .final = starfive_hash_final, 646 + .finup = starfive_hash_finup, 647 + .digest = starfive_hash_digest, 648 + .export = starfive_hash_export, 649 + .import = starfive_hash_import, 650 + .init_tfm = starfive_hmac_sha224_init_tfm, 651 + .exit_tfm = starfive_hash_exit_tfm, 652 + .setkey = starfive_hash_setkey, 653 + .halg = { 654 + .digestsize = SHA224_DIGEST_SIZE, 655 + .statesize = sizeof(struct sha256_state), 656 + .base = { 657 + .cra_name = "hmac(sha224)", 658 + .cra_driver_name = "sha224-hmac-starfive", 659 + .cra_priority = 200, 660 + .cra_flags = CRYPTO_ALG_ASYNC | 661 + CRYPTO_ALG_TYPE_AHASH | 662 + CRYPTO_ALG_NEED_FALLBACK, 663 + .cra_blocksize = SHA224_BLOCK_SIZE, 664 + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 665 + .cra_alignmask = 3, 666 + .cra_module = THIS_MODULE, 667 + } 668 + } 669 + }, { 670 + .init = starfive_hash_init, 671 + .update = starfive_hash_update, 672 + .final = starfive_hash_final, 673 + .finup = starfive_hash_finup, 674 + .digest = starfive_hash_digest, 675 + .export = starfive_hash_export, 676 + .import = starfive_hash_import, 677 + .init_tfm = starfive_sha256_init_tfm, 678 + .exit_tfm = starfive_hash_exit_tfm, 679 + .halg = { 680 + .digestsize = SHA256_DIGEST_SIZE, 681 + .statesize = sizeof(struct sha256_state), 682 + .base = { 683 + .cra_name = "sha256", 684 + .cra_driver_name = "sha256-starfive", 685 + .cra_priority = 200, 686 + .cra_flags = CRYPTO_ALG_ASYNC | 687 + CRYPTO_ALG_TYPE_AHASH | 688 + CRYPTO_ALG_NEED_FALLBACK, 689 + .cra_blocksize = SHA256_BLOCK_SIZE, 690 + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 691 + .cra_alignmask = 3, 692 + .cra_module = THIS_MODULE, 693 + } 694 + } 695 + }, { 696 + .init = starfive_hash_init, 697 + .update = starfive_hash_update, 698 + .final = starfive_hash_final, 699 + .finup = starfive_hash_finup, 700 + .digest = starfive_hash_digest, 701 + .export = starfive_hash_export, 702 + .import = starfive_hash_import, 703 + .init_tfm = starfive_hmac_sha256_init_tfm, 704 + .exit_tfm = starfive_hash_exit_tfm, 705 + .setkey = starfive_hash_setkey, 706 + .halg = { 707 + .digestsize = SHA256_DIGEST_SIZE, 708 + .statesize = sizeof(struct sha256_state), 709 + .base = { 710 + .cra_name = "hmac(sha256)", 711 + .cra_driver_name = "sha256-hmac-starfive", 712 + .cra_priority = 200, 713 + .cra_flags = CRYPTO_ALG_ASYNC | 714 + CRYPTO_ALG_TYPE_AHASH | 715 + CRYPTO_ALG_NEED_FALLBACK, 716 + .cra_blocksize = SHA256_BLOCK_SIZE, 717 + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 718 + .cra_alignmask = 3, 719 + .cra_module = THIS_MODULE, 720 + } 721 + } 722 + }, { 723 + .init = starfive_hash_init, 724 + .update = starfive_hash_update, 725 + .final = starfive_hash_final, 726 + .finup = starfive_hash_finup, 727 + .digest = starfive_hash_digest, 728 + .export = starfive_hash_export, 729 + .import = starfive_hash_import, 730 + .init_tfm = starfive_sha384_init_tfm, 731 + .exit_tfm = starfive_hash_exit_tfm, 732 + .halg = { 733 + .digestsize = SHA384_DIGEST_SIZE, 734 + .statesize = sizeof(struct sha512_state), 735 + .base = { 736 + .cra_name = "sha384", 737 + .cra_driver_name = "sha384-starfive", 738 + .cra_priority = 200, 739 + .cra_flags = CRYPTO_ALG_ASYNC | 740 + CRYPTO_ALG_TYPE_AHASH | 741 + CRYPTO_ALG_NEED_FALLBACK, 742 + .cra_blocksize = SHA384_BLOCK_SIZE, 743 + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 744 + .cra_alignmask = 3, 745 + .cra_module = THIS_MODULE, 746 + } 747 + } 748 + }, { 749 + .init = starfive_hash_init, 750 + .update = starfive_hash_update, 751 + .final = starfive_hash_final, 752 + .finup = starfive_hash_finup, 753 + .digest = starfive_hash_digest, 754 + .export = starfive_hash_export, 755 + .import = starfive_hash_import, 756 + .init_tfm = starfive_hmac_sha384_init_tfm, 757 + .exit_tfm = starfive_hash_exit_tfm, 758 + .setkey = starfive_hash_setkey, 759 + .halg = { 760 + .digestsize = SHA384_DIGEST_SIZE, 761 + .statesize = sizeof(struct sha512_state), 762 + .base = { 763 + .cra_name = "hmac(sha384)", 764 + .cra_driver_name = "sha384-hmac-starfive", 765 + .cra_priority = 200, 766 + .cra_flags = CRYPTO_ALG_ASYNC | 767 + CRYPTO_ALG_TYPE_AHASH | 768 + CRYPTO_ALG_NEED_FALLBACK, 769 + .cra_blocksize = SHA384_BLOCK_SIZE, 770 + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 771 + .cra_alignmask = 3, 772 + .cra_module = THIS_MODULE, 773 + } 774 + } 775 + }, { 776 + .init = starfive_hash_init, 777 + .update = starfive_hash_update, 778 + .final = starfive_hash_final, 779 + .finup = starfive_hash_finup, 780 + .digest = starfive_hash_digest, 781 + .export = starfive_hash_export, 782 + .import = starfive_hash_import, 783 + .init_tfm = starfive_sha512_init_tfm, 784 + .exit_tfm = starfive_hash_exit_tfm, 785 + .halg = { 786 + .digestsize = SHA512_DIGEST_SIZE, 787 + .statesize = sizeof(struct sha512_state), 788 + .base = { 789 + .cra_name = "sha512", 790 + .cra_driver_name = "sha512-starfive", 791 + .cra_priority = 200, 792 + .cra_flags = CRYPTO_ALG_ASYNC | 793 + CRYPTO_ALG_TYPE_AHASH | 794 + CRYPTO_ALG_NEED_FALLBACK, 795 + .cra_blocksize = SHA512_BLOCK_SIZE, 796 + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 797 + .cra_alignmask = 3, 798 + .cra_module = THIS_MODULE, 799 + } 800 + } 801 + }, { 802 + .init = starfive_hash_init, 803 + .update = starfive_hash_update, 804 + .final = starfive_hash_final, 805 + .finup = starfive_hash_finup, 806 + .digest = starfive_hash_digest, 807 + .export = starfive_hash_export, 808 + .import = starfive_hash_import, 809 + .init_tfm = starfive_hmac_sha512_init_tfm, 810 + .exit_tfm = starfive_hash_exit_tfm, 811 + .setkey = starfive_hash_setkey, 812 + .halg = { 813 + .digestsize = SHA512_DIGEST_SIZE, 814 + .statesize = sizeof(struct sha512_state), 815 + .base = { 816 + .cra_name = "hmac(sha512)", 817 + .cra_driver_name = "sha512-hmac-starfive", 818 + .cra_priority = 200, 819 + .cra_flags = CRYPTO_ALG_ASYNC | 820 + CRYPTO_ALG_TYPE_AHASH | 821 + CRYPTO_ALG_NEED_FALLBACK, 822 + .cra_blocksize = SHA512_BLOCK_SIZE, 823 + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 824 + .cra_alignmask = 3, 825 + .cra_module = THIS_MODULE, 826 + } 827 + } 828 + }, { 829 + .init = starfive_hash_init, 830 + .update = starfive_hash_update, 831 + .final = starfive_hash_final, 832 + .finup = starfive_hash_finup, 833 + .digest = starfive_hash_digest, 834 + .export = starfive_hash_export, 835 + .import = starfive_hash_import, 836 + .init_tfm = starfive_sm3_init_tfm, 837 + .exit_tfm = starfive_hash_exit_tfm, 838 + .halg = { 839 + .digestsize = SM3_DIGEST_SIZE, 840 + .statesize = sizeof(struct sm3_state), 841 + .base = { 842 + .cra_name = "sm3", 843 + .cra_driver_name = "sm3-starfive", 844 + .cra_priority = 200, 845 + .cra_flags = CRYPTO_ALG_ASYNC | 846 + CRYPTO_ALG_TYPE_AHASH | 847 + CRYPTO_ALG_NEED_FALLBACK, 848 + .cra_blocksize = SM3_BLOCK_SIZE, 849 + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 850 + .cra_alignmask = 3, 851 + .cra_module = THIS_MODULE, 852 + } 853 + } 854 + }, { 855 + .init = starfive_hash_init, 856 + .update = starfive_hash_update, 857 + .final = starfive_hash_final, 858 + .finup = starfive_hash_finup, 859 + .digest = starfive_hash_digest, 860 + .export = starfive_hash_export, 861 + .import = starfive_hash_import, 862 + .init_tfm = starfive_hmac_sm3_init_tfm, 863 + .exit_tfm = starfive_hash_exit_tfm, 864 + .setkey = starfive_hash_setkey, 865 + .halg = { 866 + .digestsize = SM3_DIGEST_SIZE, 867 + .statesize = sizeof(struct sm3_state), 868 + .base = { 869 + .cra_name = "hmac(sm3)", 870 + .cra_driver_name = "sm3-hmac-starfive", 871 + .cra_priority = 200, 872 + .cra_flags = CRYPTO_ALG_ASYNC | 873 + CRYPTO_ALG_TYPE_AHASH | 874 + CRYPTO_ALG_NEED_FALLBACK, 875 + .cra_blocksize = SM3_BLOCK_SIZE, 876 + .cra_ctxsize = sizeof(struct starfive_cryp_ctx), 877 + .cra_alignmask = 3, 878 + .cra_module = THIS_MODULE, 879 + } 880 + } 881 + }, 882 + }; 883 + 884 + int starfive_hash_register_algs(void) 885 + { 886 + return crypto_register_ahashes(algs_sha2_sm3, ARRAY_SIZE(algs_sha2_sm3)); 887 + } 888 + 889 + void starfive_hash_unregister_algs(void) 890 + { 891 + crypto_unregister_ahashes(algs_sha2_sm3, ARRAY_SIZE(algs_sha2_sm3)); 892 + }