tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

RDMA/core: use IOVA-based DMA mapping for bvec RDMA operations

The bvec RDMA API maps each bvec individually via dma_map_phys(),
requiring an IOTLB sync for each mapping. For large I/O operations
with many bvecs, this overhead becomes significant.

The two-step IOVA API (dma_iova_try_alloc / dma_iova_link /
dma_iova_sync) allocates a contiguous IOVA range upfront, links
all physical pages without IOTLB syncs, then performs a single
sync at the end. This reduces IOTLB flushes from O(n) to O(1).

It also requires only a single output dma_addr_t compared to extra
per-input element storage in struct scatterlist.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
Link: https://patch.msgid.link/20260128005400.25147-3-cel@kernel.org
Signed-off-by: Leon Romanovsky <leon@kernel.org>

authored by

Chuck Lever and committed by
Leon Romanovsky 853e8920 5e541553

+114
+106
drivers/infiniband/core/rw.c
··· 14 14 RDMA_RW_MULTI_WR, 15 15 RDMA_RW_MR, 16 16 RDMA_RW_SIG_MR, 17 + RDMA_RW_IOVA, 17 18 }; 18 19 19 20 static bool rdma_rw_force_mr; ··· 384 383 return -ENOMEM; 385 384 } 386 385 386 + /* 387 + * Try to use the two-step IOVA API to map bvecs into a contiguous DMA range. 388 + * This reduces IOTLB sync overhead by doing one sync at the end instead of 389 + * one per bvec, and produces a contiguous DMA address range that can be 390 + * described by a single SGE. 391 + * 392 + * Returns the number of WQEs (always 1) on success, -EOPNOTSUPP if IOVA 393 + * mapping is not available, or another negative error code on failure. 394 + */ 395 + static int rdma_rw_init_iova_wrs_bvec(struct rdma_rw_ctx *ctx, 396 + struct ib_qp *qp, const struct bio_vec *bvec, 397 + struct bvec_iter *iter, u64 remote_addr, u32 rkey, 398 + enum dma_data_direction dir) 399 + { 400 + struct ib_device *dev = qp->pd->device; 401 + struct device *dma_dev = dev->dma_device; 402 + size_t total_len = iter->bi_size; 403 + struct bio_vec first_bv; 404 + size_t mapped_len = 0; 405 + int ret; 406 + 407 + /* Virtual DMA devices cannot support IOVA allocators */ 408 + if (ib_uses_virt_dma(dev)) 409 + return -EOPNOTSUPP; 410 + 411 + /* Try to allocate contiguous IOVA space */ 412 + first_bv = mp_bvec_iter_bvec(bvec, *iter); 413 + if (!dma_iova_try_alloc(dma_dev, &ctx->iova.state, 414 + bvec_phys(&first_bv), total_len)) 415 + return -EOPNOTSUPP; 416 + 417 + /* Link all bvecs into the IOVA space */ 418 + while (iter->bi_size) { 419 + struct bio_vec bv = mp_bvec_iter_bvec(bvec, *iter); 420 + 421 + ret = dma_iova_link(dma_dev, &ctx->iova.state, bvec_phys(&bv), 422 + mapped_len, bv.bv_len, dir, 0); 423 + if (ret) 424 + goto out_destroy; 425 + 426 + mapped_len += bv.bv_len; 427 + bvec_iter_advance(bvec, iter, bv.bv_len); 428 + } 429 + 430 + /* Sync the IOTLB once for all linked pages */ 431 + ret = dma_iova_sync(dma_dev, &ctx->iova.state, 0, mapped_len); 432 + if (ret) 433 + goto out_destroy; 434 + 435 + ctx->iova.mapped_len = mapped_len; 436 + 437 + /* Single SGE covers the entire contiguous IOVA range */ 438 + ctx->iova.sge.addr = ctx->iova.state.addr; 439 + ctx->iova.sge.length = mapped_len; 440 + ctx->iova.sge.lkey = qp->pd->local_dma_lkey; 441 + 442 + /* Single WR for the whole transfer */ 443 + memset(&ctx->iova.wr, 0, sizeof(ctx->iova.wr)); 444 + if (dir == DMA_TO_DEVICE) 445 + ctx->iova.wr.wr.opcode = IB_WR_RDMA_WRITE; 446 + else 447 + ctx->iova.wr.wr.opcode = IB_WR_RDMA_READ; 448 + ctx->iova.wr.wr.num_sge = 1; 449 + ctx->iova.wr.wr.sg_list = &ctx->iova.sge; 450 + ctx->iova.wr.remote_addr = remote_addr; 451 + ctx->iova.wr.rkey = rkey; 452 + 453 + ctx->type = RDMA_RW_IOVA; 454 + ctx->nr_ops = 1; 455 + return 1; 456 + 457 + out_destroy: 458 + /* 459 + * dma_iova_destroy() expects the actual mapped length, not the 460 + * total allocation size. It unlinks only the successfully linked 461 + * range and frees the entire IOVA allocation. 462 + */ 463 + dma_iova_destroy(dma_dev, &ctx->iova.state, mapped_len, dir, 0); 464 + return ret; 465 + } 466 + 387 467 /** 388 468 * rdma_rw_ctx_init - initialize a RDMA READ/WRITE context 389 469 * @ctx: context to initialize ··· 567 485 struct bvec_iter iter, u64 remote_addr, u32 rkey, 568 486 enum dma_data_direction dir) 569 487 { 488 + int ret; 489 + 570 490 if (nr_bvec == 0 || iter.bi_size == 0) 571 491 return -EINVAL; 572 492 ··· 579 495 if (nr_bvec == 1) 580 496 return rdma_rw_init_single_wr_bvec(ctx, qp, bvecs, &iter, 581 497 remote_addr, rkey, dir); 498 + 499 + /* 500 + * Try IOVA-based mapping first for multi-bvec transfers. 501 + * This reduces IOTLB sync overhead by batching all mappings. 502 + */ 503 + ret = rdma_rw_init_iova_wrs_bvec(ctx, qp, bvecs, &iter, remote_addr, 504 + rkey, dir); 505 + if (ret != -EOPNOTSUPP) 506 + return ret; 507 + 582 508 return rdma_rw_init_map_wrs_bvec(ctx, qp, bvecs, nr_bvec, &iter, 583 509 remote_addr, rkey, dir); 584 510 } ··· 765 671 first_wr = &ctx->reg[0].reg_wr.wr; 766 672 last_wr = &ctx->reg[ctx->nr_ops - 1].wr.wr; 767 673 break; 674 + case RDMA_RW_IOVA: 675 + first_wr = &ctx->iova.wr.wr; 676 + last_wr = &ctx->iova.wr.wr; 677 + break; 768 678 case RDMA_RW_MULTI_WR: 769 679 first_wr = &ctx->map.wrs[0].wr; 770 680 last_wr = &ctx->map.wrs[ctx->nr_ops - 1].wr; ··· 843 745 break; 844 746 case RDMA_RW_SINGLE_WR: 845 747 break; 748 + case RDMA_RW_IOVA: 749 + /* IOVA contexts must use rdma_rw_ctx_destroy_bvec() */ 750 + WARN_ON_ONCE(1); 751 + return; 846 752 default: 847 753 BUG(); 848 754 break; ··· 880 778 u32 i; 881 779 882 780 switch (ctx->type) { 781 + case RDMA_RW_IOVA: 782 + dma_iova_destroy(dev->dma_device, &ctx->iova.state, 783 + ctx->iova.mapped_len, dir, 0); 784 + break; 883 785 case RDMA_RW_MULTI_WR: 884 786 for (i = 0; i < nr_bvec; i++) 885 787 ib_dma_unmap_bvec(dev, ctx->map.sges[i].addr,
+8
include/rdma/rw.h
··· 32 32 struct ib_rdma_wr *wrs; 33 33 } map; 34 34 35 + /* for IOVA-based mapping of bvecs into contiguous DMA range: */ 36 + struct { 37 + struct dma_iova_state state; 38 + struct ib_sge sge; 39 + struct ib_rdma_wr wr; 40 + size_t mapped_len; 41 + } iova; 42 + 35 43 /* for registering multiple WRs: */ 36 44 struct rdma_rw_reg_ctx { 37 45 struct ib_sge sge;