crypto: hisilicon - Use fine grained DMA mapping direction

The following splat was triggered when booting the kernel built with
arm64's defconfig + CRYPTO_SELFTESTS + DMA_API_DEBUG.

------------[ cut here ]------------
DMA-API: hisi_sec2 0000:75:00.0: cacheline tracking EEXIST, overlapping mappings aren't supported
WARNING: CPU: 24 PID: 1273 at kernel/dma/debug.c:596 add_dma_entry+0x248/0x308

Call trace:
add_dma_entry+0x248/0x308 (P)
debug_dma_map_sg+0x208/0x3e4
__dma_map_sg_attrs+0xbc/0x118
dma_map_sg_attrs+0x10/0x24
hisi_acc_sg_buf_map_to_hw_sgl+0x80/0x218 [hisi_qm]
sec_cipher_map+0xc4/0x338 [hisi_sec2]
sec_aead_sgl_map+0x18/0x24 [hisi_sec2]
sec_process+0xb8/0x36c [hisi_sec2]
sec_aead_crypto+0xe4/0x264 [hisi_sec2]
sec_aead_encrypt+0x14/0x20 [hisi_sec2]
crypto_aead_encrypt+0x24/0x38
test_aead_vec_cfg+0x480/0x7e4
test_aead_vec+0x84/0x1b8
alg_test_aead+0xc0/0x498
alg_test.part.0+0x518/0x524
alg_test+0x20/0x64
cryptomgr_test+0x24/0x44
kthread+0x130/0x1fc
ret_from_fork+0x10/0x20
---[ end trace 0000000000000000 ]---
DMA-API: Mapped at:
debug_dma_map_sg+0x234/0x3e4
__dma_map_sg_attrs+0xbc/0x118
dma_map_sg_attrs+0x10/0x24
hisi_acc_sg_buf_map_to_hw_sgl+0x80/0x218 [hisi_qm]
sec_cipher_map+0xc4/0x338 [hisi_sec2]

This occurs in selftests where the input and the output scatterlist point
to the same underlying memory (e.g., when tested with INPLACE_TWO_SGLISTS
mode).

The problem is that the hisi_sec2 driver maps these two different
scatterlists using the DMA_BIDIRECTIONAL flag which leads to overlapped
write mappings which are not supported by the DMA layer.

Fix it by using the fine grained and correct DMA mapping directions. While
at it, switch the DMA directions used by the hisi_zip driver too.

Signed-off-by: Zenghui Yu <yuzenghui@huawei.com>
Reviewed-by: Longfang Liu <liulongfang@huawei.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

authored by

Zenghui Yu and committed by

Herbert Xu 1 year ago 2566de3e c71187c1

+30 -23

4 changed files

expand all

drivers

crypto

hisilicon

sec2

sec_crypto.c

sgl.c

zip

zip_crypto.c

include

linux

hisi_acc_qm.h

+13 -8

drivers/crypto/hisilicon/sec2/sec_crypto.c

··· 965 965 struct sec_qp_ctx *qp_ctx = req->qp_ctx; 966 966 struct sec_alg_res *res = &qp_ctx->res[req->req_id]; 967 967 struct device *dev = ctx->dev; 968 + enum dma_data_direction src_direction; 968 969 int ret; 969 970 970 971 if (req->use_pbuf) { ··· 991 990 a_req->out_mac_dma = res->out_mac_dma; 992 991 } 993 992 993 + src_direction = dst == src ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; 994 994 req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, 995 995 qp_ctx->c_in_pool, 996 996 req->req_id, 997 - &req->in_dma); 997 + &req->in_dma, src_direction); 998 998 if (IS_ERR(req->in)) { 999 999 dev_err(dev, "fail to dma map input sgl buffers!\n"); 1000 1000 return PTR_ERR(req->in); ··· 1005 1003 ret = sec_aead_mac_init(a_req); 1006 1004 if (unlikely(ret)) { 1007 1005 dev_err(dev, "fail to init mac data for ICV!\n"); 1008 - hisi_acc_sg_buf_unmap(dev, src, req->in); 1006 + hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction); 1009 1007 return ret; 1010 1008 } 1011 1009 } ··· 1017 1015 c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, 1018 1016 qp_ctx->c_out_pool, 1019 1017 req->req_id, 1020 - &c_req->c_out_dma); 1018 + &c_req->c_out_dma, 1019 + DMA_FROM_DEVICE); 1021 1020 1022 1021 if (IS_ERR(c_req->c_out)) { 1023 1022 dev_err(dev, "fail to dma map output sgl buffers!\n"); 1024 - hisi_acc_sg_buf_unmap(dev, src, req->in); 1023 + hisi_acc_sg_buf_unmap(dev, src, req->in, src_direction); 1025 1024 return PTR_ERR(c_req->c_out); 1026 1025 } 1027 1026 } ··· 1039 1036 if (req->use_pbuf) { 1040 1037 sec_cipher_pbuf_unmap(ctx, req, dst); 1041 1038 } else { 1042 - if (dst != src) 1043 - hisi_acc_sg_buf_unmap(dev, src, req->in); 1044 - 1045 - hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out); 1039 + if (dst != src) { 1040 + hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out, DMA_FROM_DEVICE); 1041 + hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_TO_DEVICE); 1042 + } else { 1043 + hisi_acc_sg_buf_unmap(dev, src, req->in, DMA_BIDIRECTIONAL); 1044 + } 1046 1045 } 1047 1046 } 1048 1047

+8 -7

drivers/crypto/hisilicon/sgl.c

··· 210 210 * @pool: Pool which hw sgl memory will be allocated in. 211 211 * @index: Index of hisi_acc_hw_sgl in pool. 212 212 * @hw_sgl_dma: The dma address of allocated hw sgl. 213 + * @dir: DMA direction. 213 214 * 214 215 * This function builds hw sgl according input sgl, user can use hw_sgl_dma 215 216 * as src/dst in its BD. Only support single hw sgl currently. 216 217 */ 217 218 struct hisi_acc_hw_sgl * 218 - hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, 219 - struct scatterlist *sgl, 220 - struct hisi_acc_sgl_pool *pool, 221 - u32 index, dma_addr_t *hw_sgl_dma) 219 + hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, struct scatterlist *sgl, 220 + struct hisi_acc_sgl_pool *pool, u32 index, 221 + dma_addr_t *hw_sgl_dma, enum dma_data_direction dir) 222 222 { 223 223 struct hisi_acc_hw_sgl *curr_hw_sgl; 224 224 unsigned int i, sg_n_mapped; ··· 232 232 233 233 sg_n = sg_nents(sgl); 234 234 235 - sg_n_mapped = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL); 235 + sg_n_mapped = dma_map_sg(dev, sgl, sg_n, dir); 236 236 if (!sg_n_mapped) { 237 237 dev_err(dev, "DMA mapping for SG error!\n"); 238 238 return ERR_PTR(-EINVAL); ··· 276 276 * @dev: The device which hw sgl belongs to. 277 277 * @sgl: Related scatterlist. 278 278 * @hw_sgl: Virtual address of hw sgl. 279 + * @dir: DMA direction. 279 280 * 280 281 * This function unmaps allocated hw sgl. 281 282 */ 282 283 void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl, 283 - struct hisi_acc_hw_sgl *hw_sgl) 284 + struct hisi_acc_hw_sgl *hw_sgl, enum dma_data_direction dir) 284 285 { 285 286 if (!dev || !sgl || !hw_sgl) 286 287 return; 287 288 288 - dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL); 289 + dma_unmap_sg(dev, sgl, sg_nents(sgl), dir); 289 290 clear_hw_sgl_sge(hw_sgl); 290 291 hw_sgl->entry_sum_in_chain = 0; 291 292 hw_sgl->entry_sum_in_sgl = 0;

+7 -6

drivers/crypto/hisilicon/zip/zip_crypto.c

··· 224 224 return -EINVAL; 225 225 226 226 req->hw_src = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->src, pool, 227 - req->req_id << 1, &req->dma_src); 227 + req->req_id << 1, &req->dma_src, 228 + DMA_TO_DEVICE); 228 229 if (IS_ERR(req->hw_src)) { 229 230 dev_err(dev, "failed to map the src buffer to hw sgl (%ld)!\n", 230 231 PTR_ERR(req->hw_src)); ··· 234 233 235 234 req->hw_dst = hisi_acc_sg_buf_map_to_hw_sgl(dev, a_req->dst, pool, 236 235 (req->req_id << 1) + 1, 237 - &req->dma_dst); 236 + &req->dma_dst, DMA_FROM_DEVICE); 238 237 if (IS_ERR(req->hw_dst)) { 239 238 ret = PTR_ERR(req->hw_dst); 240 239 dev_err(dev, "failed to map the dst buffer to hw slg (%d)!\n", ··· 259 258 return -EINPROGRESS; 260 259 261 260 err_unmap_output: 262 - hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst); 261 + hisi_acc_sg_buf_unmap(dev, a_req->dst, req->hw_dst, DMA_FROM_DEVICE); 263 262 err_unmap_input: 264 - hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src); 263 + hisi_acc_sg_buf_unmap(dev, a_req->src, req->hw_src, DMA_TO_DEVICE); 265 264 return ret; 266 265 } 267 266 ··· 304 303 err = -EIO; 305 304 } 306 305 307 - hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src); 308 - hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst); 306 + hisi_acc_sg_buf_unmap(dev, acomp_req->dst, req->hw_dst, DMA_FROM_DEVICE); 307 + hisi_acc_sg_buf_unmap(dev, acomp_req->src, req->hw_src, DMA_TO_DEVICE); 309 308 310 309 acomp_req->dlen = ops->get_dstlen(sqe); 311 310

+2 -2

include/linux/hisi_acc_qm.h

··· 556 556 struct hisi_acc_sgl_pool; 557 557 struct hisi_acc_hw_sgl *hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, 558 558 struct scatterlist *sgl, struct hisi_acc_sgl_pool *pool, 559 - u32 index, dma_addr_t *hw_sgl_dma); 559 + u32 index, dma_addr_t *hw_sgl_dma, enum dma_data_direction dir); 560 560 void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl, 561 - struct hisi_acc_hw_sgl *hw_sgl); 561 + struct hisi_acc_hw_sgl *hw_sgl, enum dma_data_direction dir); 562 562 struct hisi_acc_sgl_pool *hisi_acc_create_sgl_pool(struct device *dev, 563 563 u32 count, u32 sge_nr); 564 564 void hisi_acc_free_sgl_pool(struct device *dev,

Configure Feed

Configure Feed