block: don't merge different kinds of P2P transfers in a single bio

To get out of the DMA mapping helpers having to check every segment for
it's P2P status, ensure that bios either contain P2P transfers or non-P2P
transfers, and that a P2P bio only contains ranges from a single device.

This means we do the page zone access in the bio add path where it should
be still page hot, and will only have do the fairly expensive P2P topology
lookup once per bio down in the DMA mapping path, and only for already
marked bios.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Keith Busch <kbusch@kernel.org>
Reviewed-by: Logan Gunthorpe <logang@deltatee.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
Link: https://lore.kernel.org/r/20250625113531.522027-2-hch@lst.de
Signed-off-by: Jens Axboe <axboe@kernel.dk>

authored by

Christoph Hellwig and committed by

Jens Axboe 1 year ago 38446014 409f9287

+18 -7

3 changed files

expand all

block

bio-integrity.c

bio.c

include

linux

blk_types.h

block/bio-integrity.c

··· 128 128 if (bip->bip_vcnt > 0) { 129 129 struct bio_vec *bv = &bip->bip_vec[bip->bip_vcnt - 1]; 130 130 131 + if (!zone_device_pages_have_same_pgmap(bv->bv_page, page)) 132 + return 0; 133 + 131 134 if (bvec_try_merge_hw_page(q, bv, page, len, offset)) { 132 135 bip->bip_iter.bi_size += len; 133 136 return len;

+13 -7

block/bio.c

··· 930 930 return false; 931 931 if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) 932 932 return false; 933 - if (!zone_device_pages_have_same_pgmap(bv->bv_page, page)) 934 - return false; 935 933 936 934 if ((vec_end_addr & PAGE_MASK) != ((page_addr + off) & PAGE_MASK)) { 937 935 if (IS_ENABLED(CONFIG_KMSAN)) ··· 980 982 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); 981 983 WARN_ON_ONCE(bio_full(bio, len)); 982 984 985 + if (is_pci_p2pdma_page(page)) 986 + bio->bi_opf |= REQ_P2PDMA | REQ_NOMERGE; 987 + 983 988 bvec_set_page(&bio->bi_io_vec[bio->bi_vcnt], page, len, off); 984 989 bio->bi_iter.bi_size += len; 985 990 bio->bi_vcnt++; ··· 1023 1022 if (bio->bi_iter.bi_size > UINT_MAX - len) 1024 1023 return 0; 1025 1024 1026 - if (bio->bi_vcnt > 0 && 1027 - bvec_try_merge_page(&bio->bi_io_vec[bio->bi_vcnt - 1], 1028 - page, len, offset)) { 1029 - bio->bi_iter.bi_size += len; 1030 - return len; 1025 + if (bio->bi_vcnt > 0) { 1026 + struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; 1027 + 1028 + if (!zone_device_pages_have_same_pgmap(bv->bv_page, page)) 1029 + return 0; 1030 + 1031 + if (bvec_try_merge_page(bv, page, len, offset)) { 1032 + bio->bi_iter.bi_size += len; 1033 + return len; 1034 + } 1031 1035 } 1032 1036 1033 1037 if (bio->bi_vcnt >= bio->bi_max_vecs)

include/linux/blk_types.h

··· 386 386 __REQ_DRV, /* for driver use */ 387 387 __REQ_FS_PRIVATE, /* for file system (submitter) use */ 388 388 __REQ_ATOMIC, /* for atomic write operations */ 389 + __REQ_P2PDMA, /* contains P2P DMA pages */ 389 390 /* 390 391 * Command specific flags, keep last: 391 392 */ ··· 419 418 #define REQ_DRV (__force blk_opf_t)(1ULL << __REQ_DRV) 420 419 #define REQ_FS_PRIVATE (__force blk_opf_t)(1ULL << __REQ_FS_PRIVATE) 421 420 #define REQ_ATOMIC (__force blk_opf_t)(1ULL << __REQ_ATOMIC) 421 + #define REQ_P2PDMA (__force blk_opf_t)(1ULL << __REQ_P2PDMA) 422 422 423 423 #define REQ_NOUNMAP (__force blk_opf_t)(1ULL << __REQ_NOUNMAP) 424 424

Configure Feed

Configure Feed