Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost

+14 -14

drivers/net/virtio_net.c

··· 962 962 if (dma->need_sync && len) { 963 963 offset = buf - (head + sizeof(*dma)); 964 964 965 - virtqueue_dma_sync_single_range_for_cpu(rq->vq, dma->addr, 965 + virtqueue_map_sync_single_range_for_cpu(rq->vq, dma->addr, 966 966 offset, len, 967 967 DMA_FROM_DEVICE); 968 968 } ··· 970 970 if (dma->ref) 971 971 return; 972 972 973 - virtqueue_dma_unmap_single_attrs(rq->vq, dma->addr, dma->len, 974 - DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 973 + virtqueue_unmap_single_attrs(rq->vq, dma->addr, dma->len, 974 + DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 975 975 put_page(page); 976 976 } 977 977 ··· 1038 1038 1039 1039 dma->len = alloc_frag->size - sizeof(*dma); 1040 1040 1041 - addr = virtqueue_dma_map_single_attrs(rq->vq, dma + 1, 1042 - dma->len, DMA_FROM_DEVICE, 0); 1043 - if (virtqueue_dma_mapping_error(rq->vq, addr)) 1041 + addr = virtqueue_map_single_attrs(rq->vq, dma + 1, 1042 + dma->len, DMA_FROM_DEVICE, 0); 1043 + if (virtqueue_map_mapping_error(rq->vq, addr)) 1044 1044 return NULL; 1045 1045 1046 1046 dma->addr = addr; 1047 - dma->need_sync = virtqueue_dma_need_sync(rq->vq, addr); 1047 + dma->need_sync = virtqueue_map_need_sync(rq->vq, addr); 1048 1048 1049 1049 /* Add a reference to dma to prevent the entire dma from 1050 1050 * being released during error handling. This reference ··· 5942 5942 if (!rq->xsk_buffs) 5943 5943 return -ENOMEM; 5944 5944 5945 - hdr_dma = virtqueue_dma_map_single_attrs(sq->vq, &xsk_hdr, vi->hdr_len, 5946 - DMA_TO_DEVICE, 0); 5947 - if (virtqueue_dma_mapping_error(sq->vq, hdr_dma)) { 5945 + hdr_dma = virtqueue_map_single_attrs(sq->vq, &xsk_hdr, vi->hdr_len, 5946 + DMA_TO_DEVICE, 0); 5947 + if (virtqueue_map_mapping_error(sq->vq, hdr_dma)) { 5948 5948 err = -ENOMEM; 5949 5949 goto err_free_buffs; 5950 5950 } ··· 5973 5973 err_rq: 5974 5974 xsk_pool_dma_unmap(pool, 0); 5975 5975 err_xsk_map: 5976 - virtqueue_dma_unmap_single_attrs(rq->vq, hdr_dma, vi->hdr_len, 5977 - DMA_TO_DEVICE, 0); 5976 + virtqueue_unmap_single_attrs(rq->vq, hdr_dma, vi->hdr_len, 5977 + DMA_TO_DEVICE, 0); 5978 5978 err_free_buffs: 5979 5979 kvfree(rq->xsk_buffs); 5980 5980 return err; ··· 6001 6001 6002 6002 xsk_pool_dma_unmap(pool, 0); 6003 6003 6004 - virtqueue_dma_unmap_single_attrs(sq->vq, sq->xsk_hdr_dma_addr, 6005 - vi->hdr_len, DMA_TO_DEVICE, 0); 6004 + virtqueue_unmap_single_attrs(sq->vq, sq->xsk_hdr_dma_addr, 6005 + vi->hdr_len, DMA_TO_DEVICE, 0); 6006 6006 kvfree(rq->xsk_buffs); 6007 6007 6008 6008 return err;

+1 -7

drivers/vdpa/Kconfig

··· 34 34 35 35 config VDPA_USER 36 36 tristate "VDUSE (vDPA Device in Userspace) support" 37 - depends on EVENTFD && MMU && HAS_DMA 38 - # 39 - # This driver incorrectly tries to override the dma_ops. It should 40 - # never have done that, but for now keep it working on architectures 41 - # that use dma ops 42 - # 43 - depends on ARCH_HAS_DMA_OPS 37 + depends on EVENTFD && MMU 44 38 select VHOST_IOTLB 45 39 select IOMMU_IOVA 46 40 help

+3 -2

drivers/vdpa/alibaba/eni_vdpa.c

··· 478 478 return ret; 479 479 480 480 eni_vdpa = vdpa_alloc_device(struct eni_vdpa, vdpa, 481 - dev, &eni_vdpa_ops, 1, 1, NULL, false); 481 + dev, &eni_vdpa_ops, NULL, 482 + 1, 1, NULL, false); 482 483 if (IS_ERR(eni_vdpa)) { 483 484 ENI_ERR(pdev, "failed to allocate vDPA structure\n"); 484 485 return PTR_ERR(eni_vdpa); ··· 497 496 pci_set_master(pdev); 498 497 pci_set_drvdata(pdev, eni_vdpa); 499 498 500 - eni_vdpa->vdpa.dma_dev = &pdev->dev; 499 + eni_vdpa->vdpa.vmap.dma_dev = &pdev->dev; 501 500 eni_vdpa->queues = eni_vdpa_get_num_queues(eni_vdpa); 502 501 503 502 eni_vdpa->vring = devm_kcalloc(&pdev->dev, eni_vdpa->queues,

+3 -2

drivers/vdpa/ifcvf/ifcvf_main.c

··· 705 705 vf = &ifcvf_mgmt_dev->vf; 706 706 pdev = vf->pdev; 707 707 adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa, 708 - &pdev->dev, &ifc_vdpa_ops, 1, 1, NULL, false); 708 + &pdev->dev, &ifc_vdpa_ops, 709 + NULL, 1, 1, NULL, false); 709 710 if (IS_ERR(adapter)) { 710 711 IFCVF_ERR(pdev, "Failed to allocate vDPA structure"); 711 712 return PTR_ERR(adapter); ··· 714 713 715 714 ifcvf_mgmt_dev->adapter = adapter; 716 715 adapter->pdev = pdev; 717 - adapter->vdpa.dma_dev = &pdev->dev; 716 + adapter->vdpa.vmap.dma_dev = &pdev->dev; 718 717 adapter->vdpa.mdev = mdev; 719 718 adapter->vf = vf; 720 719 vdpa_dev = &adapter->vdpa;

+2 -2

drivers/vdpa/mlx5/core/mr.c

··· 378 378 u64 pa, offset; 379 379 u64 paend; 380 380 struct scatterlist *sg; 381 - struct device *dma = mvdev->vdev.dma_dev; 381 + struct device *dma = mvdev->vdev.vmap.dma_dev; 382 382 383 383 for (map = vhost_iotlb_itree_first(iotlb, mr->start, mr->end - 1); 384 384 map; map = vhost_iotlb_itree_next(map, mr->start, mr->end - 1)) { ··· 432 432 433 433 static void unmap_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr) 434 434 { 435 - struct device *dma = mvdev->vdev.dma_dev; 435 + struct device *dma = mvdev->vdev.vmap.dma_dev; 436 436 437 437 destroy_direct_mr(mvdev, mr); 438 438 dma_unmap_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0);

+9 -6

drivers/vdpa/mlx5/net/mlx5_vnet.c

··· 3395 3395 return err; 3396 3396 } 3397 3397 3398 - static struct device *mlx5_get_vq_dma_dev(struct vdpa_device *vdev, u16 idx) 3398 + static union virtio_map mlx5_get_vq_map(struct vdpa_device *vdev, u16 idx) 3399 3399 { 3400 3400 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 3401 + union virtio_map map; 3401 3402 3402 3403 if (is_ctrl_vq_idx(mvdev, idx)) 3403 - return &vdev->dev; 3404 + map.dma_dev = &vdev->dev; 3405 + else 3406 + map.dma_dev = mvdev->vdev.vmap.dma_dev; 3404 3407 3405 - return mvdev->vdev.dma_dev; 3408 + return map; 3406 3409 } 3407 3410 3408 3411 static void free_irqs(struct mlx5_vdpa_net *ndev) ··· 3689 3686 .set_map = mlx5_vdpa_set_map, 3690 3687 .reset_map = mlx5_vdpa_reset_map, 3691 3688 .set_group_asid = mlx5_set_group_asid, 3692 - .get_vq_dma_dev = mlx5_get_vq_dma_dev, 3689 + .get_vq_map = mlx5_get_vq_map, 3693 3690 .free = mlx5_vdpa_free, 3694 3691 .suspend = mlx5_vdpa_suspend, 3695 3692 .resume = mlx5_vdpa_resume, /* Op disabled if not supported. */ ··· 3882 3879 } 3883 3880 3884 3881 ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mgtdev->vdpa_ops, 3885 - MLX5_VDPA_NUMVQ_GROUPS, MLX5_VDPA_NUM_AS, name, false); 3882 + NULL, MLX5_VDPA_NUMVQ_GROUPS, MLX5_VDPA_NUM_AS, name, false); 3886 3883 if (IS_ERR(ndev)) 3887 3884 return PTR_ERR(ndev); 3888 3885 ··· 3968 3965 } 3969 3966 3970 3967 ndev->mvdev.mlx_features = device_features; 3971 - mvdev->vdev.dma_dev = &mdev->pdev->dev; 3968 + mvdev->vdev.vmap.dma_dev = &mdev->pdev->dev; 3972 3969 err = mlx5_vdpa_alloc_resources(&ndev->mvdev); 3973 3970 if (err) 3974 3971 goto err_alloc;

+3 -3

drivers/vdpa/octeon_ep/octep_vdpa_main.c

··· 508 508 u64 device_features; 509 509 int ret; 510 510 511 - oct_vdpa = vdpa_alloc_device(struct octep_vdpa, vdpa, &pdev->dev, &octep_vdpa_ops, 1, 1, 512 - NULL, false); 511 + oct_vdpa = vdpa_alloc_device(struct octep_vdpa, vdpa, &pdev->dev, &octep_vdpa_ops, 512 + NULL, 1, 1, NULL, false); 513 513 if (IS_ERR(oct_vdpa)) { 514 514 dev_err(&pdev->dev, "Failed to allocate vDPA structure for octep vdpa device"); 515 515 return PTR_ERR(oct_vdpa); 516 516 } 517 517 518 518 oct_vdpa->pdev = pdev; 519 - oct_vdpa->vdpa.dma_dev = &pdev->dev; 519 + oct_vdpa->vdpa.vmap.dma_dev = &pdev->dev; 520 520 oct_vdpa->vdpa.mdev = mdev; 521 521 oct_vdpa->oct_hw = oct_hw; 522 522 vdpa_dev = &oct_vdpa->vdpa;

+3 -2

drivers/vdpa/pds/vdpa_dev.c

··· 632 632 } 633 633 634 634 pdsv = vdpa_alloc_device(struct pds_vdpa_device, vdpa_dev, 635 - dev, &pds_vdpa_ops, 1, 1, name, false); 635 + dev, &pds_vdpa_ops, NULL, 636 + 1, 1, name, false); 636 637 if (IS_ERR(pdsv)) { 637 638 dev_err(dev, "Failed to allocate vDPA structure: %pe\n", pdsv); 638 639 return PTR_ERR(pdsv); ··· 644 643 645 644 pdev = vdpa_aux->padev->vf_pdev; 646 645 dma_dev = &pdev->dev; 647 - pdsv->vdpa_dev.dma_dev = dma_dev; 646 + pdsv->vdpa_dev.vmap.dma_dev = dma_dev; 648 647 649 648 status = pds_vdpa_get_status(&pdsv->vdpa_dev); 650 649 if (status == 0xff) {

+4 -4

drivers/vdpa/solidrun/snet_main.c

··· 1008 1008 } 1009 1009 1010 1010 /* Allocate vdpa device */ 1011 - snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1, 1, NULL, 1012 - false); 1011 + snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1012 + NULL, 1, 1, NULL, false); 1013 1013 if (!snet) { 1014 1014 SNET_ERR(pdev, "Failed to allocate a vdpa device\n"); 1015 1015 ret = -ENOMEM; ··· 1052 1052 */ 1053 1053 snet_reserve_irq_idx(pf_irqs ? pdev_pf : pdev, snet); 1054 1054 1055 - /*set DMA device*/ 1056 - snet->vdpa.dma_dev = &pdev->dev; 1055 + /* set map metadata */ 1056 + snet->vdpa.vmap.dma_dev = &pdev->dev; 1057 1057 1058 1058 /* Register VDPA device */ 1059 1059 ret = vdpa_register_device(&snet->vdpa, snet->cfg->vq_num);

+4 -1

drivers/vdpa/vdpa.c

··· 142 142 * initialized but before registered. 143 143 * @parent: the parent device 144 144 * @config: the bus operations that is supported by this device 145 + * @map: the map operations that is supported by this device 145 146 * @ngroups: number of groups supported by this device 146 147 * @nas: number of address spaces supported by this device 147 148 * @size: size of the parent structure that contains private data ··· 152 151 * Driver should use vdpa_alloc_device() wrapper macro instead of 153 152 * using this directly. 154 153 * 155 - * Return: Returns an error when parent/config/dma_dev is not set or fail to get 154 + * Return: Returns an error when parent/config/map is not set or fail to get 156 155 * ida. 157 156 */ 158 157 struct vdpa_device *__vdpa_alloc_device(struct device *parent, 159 158 const struct vdpa_config_ops *config, 159 + const struct virtio_map_ops *map, 160 160 unsigned int ngroups, unsigned int nas, 161 161 size_t size, const char *name, 162 162 bool use_va) ··· 189 187 vdev->dev.release = vdpa_release_dev; 190 188 vdev->index = err; 191 189 vdev->config = config; 190 + vdev->map = map; 192 191 vdev->features_valid = false; 193 192 vdev->use_va = use_va; 194 193 vdev->ngroups = ngroups;

+2 -2

drivers/vdpa/vdpa_sim/vdpa_sim.c

··· 215 215 else 216 216 ops = &vdpasim_config_ops; 217 217 218 - vdpa = __vdpa_alloc_device(NULL, ops, 218 + vdpa = __vdpa_alloc_device(NULL, ops, NULL, 219 219 dev_attr->ngroups, dev_attr->nas, 220 220 dev_attr->alloc_size, 221 221 dev_attr->name, use_va); ··· 272 272 vringh_set_iotlb(&vdpasim->vqs[i].vring, &vdpasim->iommu[0], 273 273 &vdpasim->iommu_lock); 274 274 275 - vdpasim->vdpa.dma_dev = dev; 275 + vdpasim->vdpa.vmap.dma_dev = dev; 276 276 277 277 return vdpasim; 278 278

+90 -42

drivers/vdpa/vdpa_user/iova_domain.c

··· 103 103 static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain, 104 104 u64 iova, u64 size, u64 paddr) 105 105 { 106 - struct vduse_bounce_map *map; 106 + struct vduse_bounce_map *map, *head_map; 107 + struct page *tmp_page; 107 108 u64 last = iova + size - 1; 108 109 109 110 while (iova <= last) { 110 - map = &domain->bounce_maps[iova >> PAGE_SHIFT]; 111 + /* 112 + * When PAGE_SIZE is larger than 4KB, multiple adjacent bounce_maps will 113 + * point to the same memory page of PAGE_SIZE. Since bounce_maps originate 114 + * from IO requests, we may not be able to guarantee that the orig_phys 115 + * values of all IO requests within the same 64KB memory page are contiguous. 116 + * Therefore, we need to store them separately. 117 + * 118 + * Bounce pages are allocated on demand. As a result, it may occur that 119 + * multiple bounce pages corresponding to the same 64KB memory page attempt 120 + * to allocate memory simultaneously, so we use cmpxchg to handle this 121 + * concurrency. 122 + */ 123 + map = &domain->bounce_maps[iova >> BOUNCE_MAP_SHIFT]; 111 124 if (!map->bounce_page) { 112 - map->bounce_page = alloc_page(GFP_ATOMIC); 113 - if (!map->bounce_page) 114 - return -ENOMEM; 125 + head_map = &domain->bounce_maps[(iova & PAGE_MASK) >> BOUNCE_MAP_SHIFT]; 126 + if (!head_map->bounce_page) { 127 + tmp_page = alloc_page(GFP_ATOMIC); 128 + if (!tmp_page) 129 + return -ENOMEM; 130 + if (cmpxchg(&head_map->bounce_page, NULL, tmp_page)) 131 + __free_page(tmp_page); 132 + } 133 + map->bounce_page = head_map->bounce_page; 115 134 } 116 135 map->orig_phys = paddr; 117 - paddr += PAGE_SIZE; 118 - iova += PAGE_SIZE; 136 + paddr += BOUNCE_MAP_SIZE; 137 + iova += BOUNCE_MAP_SIZE; 119 138 } 120 139 return 0; 121 140 } ··· 146 127 u64 last = iova + size - 1; 147 128 148 129 while (iova <= last) { 149 - map = &domain->bounce_maps[iova >> PAGE_SHIFT]; 130 + map = &domain->bounce_maps[iova >> BOUNCE_MAP_SHIFT]; 150 131 map->orig_phys = INVALID_PHYS_ADDR; 151 - iova += PAGE_SIZE; 132 + iova += BOUNCE_MAP_SIZE; 152 133 } 134 + } 135 + 136 + static unsigned int offset_in_bounce_page(dma_addr_t addr) 137 + { 138 + return (addr & ~BOUNCE_MAP_MASK); 153 139 } 154 140 155 141 static void do_bounce(phys_addr_t orig, void *addr, size_t size, ··· 187 163 { 188 164 struct vduse_bounce_map *map; 189 165 struct page *page; 190 - unsigned int offset; 166 + unsigned int offset, head_offset; 191 167 void *addr; 192 168 size_t sz; 193 169 ··· 195 171 return; 196 172 197 173 while (size) { 198 - map = &domain->bounce_maps[iova >> PAGE_SHIFT]; 199 - offset = offset_in_page(iova); 200 - sz = min_t(size_t, PAGE_SIZE - offset, size); 174 + map = &domain->bounce_maps[iova >> BOUNCE_MAP_SHIFT]; 175 + head_offset = offset_in_page(iova); 176 + offset = offset_in_bounce_page(iova); 177 + sz = min_t(size_t, BOUNCE_MAP_SIZE - offset, size); 201 178 202 179 if (WARN_ON(!map->bounce_page || 203 180 map->orig_phys == INVALID_PHYS_ADDR)) ··· 208 183 map->user_bounce_page : map->bounce_page; 209 184 210 185 addr = kmap_local_page(page); 211 - do_bounce(map->orig_phys + offset, addr + offset, sz, dir); 186 + do_bounce(map->orig_phys + offset, addr + head_offset, sz, dir); 212 187 kunmap_local(addr); 213 188 size -= sz; 214 189 iova += sz; ··· 243 218 struct page *page = NULL; 244 219 245 220 read_lock(&domain->bounce_lock); 246 - map = &domain->bounce_maps[iova >> PAGE_SHIFT]; 221 + map = &domain->bounce_maps[iova >> BOUNCE_MAP_SHIFT]; 247 222 if (domain->user_bounce_pages || !map->bounce_page) 248 223 goto out; 249 224 ··· 261 236 struct vduse_bounce_map *map; 262 237 unsigned long pfn, bounce_pfns; 263 238 264 - bounce_pfns = domain->bounce_size >> PAGE_SHIFT; 239 + bounce_pfns = domain->bounce_size >> BOUNCE_MAP_SHIFT; 265 240 266 241 for (pfn = 0; pfn < bounce_pfns; pfn++) { 267 242 map = &domain->bounce_maps[pfn]; ··· 271 246 if (!map->bounce_page) 272 247 continue; 273 248 274 - __free_page(map->bounce_page); 249 + if (!((pfn << BOUNCE_MAP_SHIFT) & ~PAGE_MASK)) 250 + __free_page(map->bounce_page); 275 251 map->bounce_page = NULL; 276 252 } 277 253 } ··· 280 254 int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain, 281 255 struct page **pages, int count) 282 256 { 283 - struct vduse_bounce_map *map; 284 - int i, ret; 257 + struct vduse_bounce_map *map, *head_map; 258 + int i, j, ret; 259 + int inner_pages = PAGE_SIZE / BOUNCE_MAP_SIZE; 260 + int bounce_pfns = domain->bounce_size >> BOUNCE_MAP_SHIFT; 261 + struct page *head_page = NULL; 262 + bool need_copy; 285 263 286 264 /* Now we don't support partial mapping */ 287 265 if (count != (domain->bounce_size >> PAGE_SHIFT)) ··· 297 267 goto out; 298 268 299 269 for (i = 0; i < count; i++) { 300 - map = &domain->bounce_maps[i]; 301 - if (map->bounce_page) { 270 + need_copy = false; 271 + head_map = &domain->bounce_maps[(i * inner_pages)]; 272 + head_page = head_map->bounce_page; 273 + for (j = 0; j < inner_pages; j++) { 274 + if ((i * inner_pages + j) >= bounce_pfns) 275 + break; 276 + map = &domain->bounce_maps[(i * inner_pages + j)]; 302 277 /* Copy kernel page to user page if it's in use */ 303 - if (map->orig_phys != INVALID_PHYS_ADDR) 304 - memcpy_to_page(pages[i], 0, 305 - page_address(map->bounce_page), 306 - PAGE_SIZE); 278 + if ((head_page) && (map->orig_phys != INVALID_PHYS_ADDR)) 279 + need_copy = true; 280 + map->user_bounce_page = pages[i]; 307 281 } 308 - map->user_bounce_page = pages[i]; 309 282 get_page(pages[i]); 283 + if ((head_page) && (need_copy)) 284 + memcpy_to_page(pages[i], 0, 285 + page_address(head_page), 286 + PAGE_SIZE); 310 287 } 311 288 domain->user_bounce_pages = true; 312 289 ret = 0; ··· 325 288 326 289 void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain) 327 290 { 328 - struct vduse_bounce_map *map; 329 - unsigned long i, count; 291 + struct vduse_bounce_map *map, *head_map; 292 + unsigned long i, j, count; 293 + int inner_pages = PAGE_SIZE / BOUNCE_MAP_SIZE; 294 + int bounce_pfns = domain->bounce_size >> BOUNCE_MAP_SHIFT; 295 + struct page *head_page = NULL; 296 + bool need_copy; 330 297 331 298 write_lock(&domain->bounce_lock); 332 299 if (!domain->user_bounce_pages) ··· 338 297 339 298 count = domain->bounce_size >> PAGE_SHIFT; 340 299 for (i = 0; i < count; i++) { 341 - struct page *page = NULL; 342 - 343 - map = &domain->bounce_maps[i]; 344 - if (WARN_ON(!map->user_bounce_page)) 300 + need_copy = false; 301 + head_map = &domain->bounce_maps[(i * inner_pages)]; 302 + if (WARN_ON(!head_map->user_bounce_page)) 345 303 continue; 304 + head_page = head_map->user_bounce_page; 346 305 347 - /* Copy user page to kernel page if it's in use */ 348 - if (map->orig_phys != INVALID_PHYS_ADDR) { 349 - page = map->bounce_page; 350 - memcpy_from_page(page_address(page), 351 - map->user_bounce_page, 0, PAGE_SIZE); 306 + for (j = 0; j < inner_pages; j++) { 307 + if ((i * inner_pages + j) >= bounce_pfns) 308 + break; 309 + map = &domain->bounce_maps[(i * inner_pages + j)]; 310 + if (WARN_ON(!map->user_bounce_page)) 311 + continue; 312 + /* Copy user page to kernel page if it's in use */ 313 + if ((map->orig_phys != INVALID_PHYS_ADDR) && (head_map->bounce_page)) 314 + need_copy = true; 315 + map->user_bounce_page = NULL; 352 316 } 353 - put_page(map->user_bounce_page); 354 - map->user_bounce_page = NULL; 317 + if (need_copy) 318 + memcpy_from_page(page_address(head_map->bounce_page), 319 + head_page, 0, PAGE_SIZE); 320 + put_page(head_page); 355 321 } 356 322 domain->user_bounce_pages = false; 357 323 out: ··· 495 447 496 448 void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain, 497 449 size_t size, dma_addr_t *dma_addr, 498 - gfp_t flag, unsigned long attrs) 450 + gfp_t flag) 499 451 { 500 452 struct iova_domain *iovad = &domain->consistent_iovad; 501 453 unsigned long limit = domain->iova_limit; ··· 629 581 unsigned long pfn, bounce_pfns; 630 582 int ret; 631 583 632 - bounce_pfns = PAGE_ALIGN(bounce_size) >> PAGE_SHIFT; 584 + bounce_pfns = PAGE_ALIGN(bounce_size) >> BOUNCE_MAP_SHIFT; 633 585 if (iova_limit <= bounce_size) 634 586 return NULL; 635 587 ··· 661 613 rwlock_init(&domain->bounce_lock); 662 614 spin_lock_init(&domain->iotlb_lock); 663 615 init_iova_domain(&domain->stream_iovad, 664 - PAGE_SIZE, IOVA_START_PFN); 616 + BOUNCE_MAP_SIZE, IOVA_START_PFN); 665 617 ret = iova_domain_init_rcaches(&domain->stream_iovad); 666 618 if (ret) 667 619 goto err_iovad_stream;

+6 -1

drivers/vdpa/vdpa_user/iova_domain.h

··· 19 19 20 20 #define INVALID_PHYS_ADDR (~(phys_addr_t)0) 21 21 22 + #define BOUNCE_MAP_SHIFT 12 23 + #define BOUNCE_MAP_SIZE (1 << BOUNCE_MAP_SHIFT) 24 + #define BOUNCE_MAP_MASK (~(BOUNCE_MAP_SIZE - 1)) 25 + #define BOUNCE_MAP_ALIGN(addr) (((addr) + BOUNCE_MAP_SIZE - 1) & ~(BOUNCE_MAP_SIZE - 1)) 26 + 22 27 struct vduse_bounce_map { 23 28 struct page *bounce_page; 24 29 struct page *user_bounce_page; ··· 69 64 70 65 void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain, 71 66 size_t size, dma_addr_t *dma_addr, 72 - gfp_t flag, unsigned long attrs); 67 + gfp_t flag); 73 68 74 69 void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size, 75 70 void *vaddr, dma_addr_t dma_addr,

+40 -39

drivers/vdpa/vdpa_user/vduse_dev.c

··· 814 814 .free = vduse_vdpa_free, 815 815 }; 816 816 817 - static void vduse_dev_sync_single_for_device(struct device *dev, 817 + static void vduse_dev_sync_single_for_device(union virtio_map token, 818 818 dma_addr_t dma_addr, size_t size, 819 819 enum dma_data_direction dir) 820 820 { 821 - struct vduse_dev *vdev = dev_to_vduse(dev); 822 - struct vduse_iova_domain *domain = vdev->domain; 821 + struct vduse_iova_domain *domain = token.iova_domain; 823 822 824 823 vduse_domain_sync_single_for_device(domain, dma_addr, size, dir); 825 824 } 826 825 827 - static void vduse_dev_sync_single_for_cpu(struct device *dev, 826 + static void vduse_dev_sync_single_for_cpu(union virtio_map token, 828 827 dma_addr_t dma_addr, size_t size, 829 828 enum dma_data_direction dir) 830 829 { 831 - struct vduse_dev *vdev = dev_to_vduse(dev); 832 - struct vduse_iova_domain *domain = vdev->domain; 830 + struct vduse_iova_domain *domain = token.iova_domain; 833 831 834 832 vduse_domain_sync_single_for_cpu(domain, dma_addr, size, dir); 835 833 } 836 834 837 - static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page, 835 + static dma_addr_t vduse_dev_map_page(union virtio_map token, struct page *page, 838 836 unsigned long offset, size_t size, 839 837 enum dma_data_direction dir, 840 838 unsigned long attrs) 841 839 { 842 - struct vduse_dev *vdev = dev_to_vduse(dev); 843 - struct vduse_iova_domain *domain = vdev->domain; 840 + struct vduse_iova_domain *domain = token.iova_domain; 844 841 845 842 return vduse_domain_map_page(domain, page, offset, size, dir, attrs); 846 843 } 847 844 848 - static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr, 849 - size_t size, enum dma_data_direction dir, 850 - unsigned long attrs) 845 + static void vduse_dev_unmap_page(union virtio_map token, dma_addr_t dma_addr, 846 + size_t size, enum dma_data_direction dir, 847 + unsigned long attrs) 851 848 { 852 - struct vduse_dev *vdev = dev_to_vduse(dev); 853 - struct vduse_iova_domain *domain = vdev->domain; 849 + struct vduse_iova_domain *domain = token.iova_domain; 854 850 855 851 return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs); 856 852 } 857 853 858 - static void *vduse_dev_alloc_coherent(struct device *dev, size_t size, 859 - dma_addr_t *dma_addr, gfp_t flag, 860 - unsigned long attrs) 854 + static void *vduse_dev_alloc_coherent(union virtio_map token, size_t size, 855 + dma_addr_t *dma_addr, gfp_t flag) 861 856 { 862 - struct vduse_dev *vdev = dev_to_vduse(dev); 863 - struct vduse_iova_domain *domain = vdev->domain; 857 + struct vduse_iova_domain *domain = token.iova_domain; 864 858 unsigned long iova; 865 859 void *addr; 866 860 867 861 *dma_addr = DMA_MAPPING_ERROR; 868 862 addr = vduse_domain_alloc_coherent(domain, size, 869 - (dma_addr_t *)&iova, flag, attrs); 863 + (dma_addr_t *)&iova, flag); 870 864 if (!addr) 871 865 return NULL; 872 866 ··· 869 875 return addr; 870 876 } 871 877 872 - static void vduse_dev_free_coherent(struct device *dev, size_t size, 873 - void *vaddr, dma_addr_t dma_addr, 874 - unsigned long attrs) 878 + static void vduse_dev_free_coherent(union virtio_map token, size_t size, 879 + void *vaddr, dma_addr_t dma_addr, 880 + unsigned long attrs) 875 881 { 876 - struct vduse_dev *vdev = dev_to_vduse(dev); 877 - struct vduse_iova_domain *domain = vdev->domain; 882 + struct vduse_iova_domain *domain = token.iova_domain; 878 883 879 884 vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs); 880 885 } 881 886 882 - static size_t vduse_dev_max_mapping_size(struct device *dev) 887 + static bool vduse_dev_need_sync(union virtio_map token, dma_addr_t dma_addr) 883 888 { 884 - struct vduse_dev *vdev = dev_to_vduse(dev); 885 - struct vduse_iova_domain *domain = vdev->domain; 889 + struct vduse_iova_domain *domain = token.iova_domain; 890 + 891 + return dma_addr < domain->bounce_size; 892 + } 893 + 894 + static int vduse_dev_mapping_error(union virtio_map token, dma_addr_t dma_addr) 895 + { 896 + if (unlikely(dma_addr == DMA_MAPPING_ERROR)) 897 + return -ENOMEM; 898 + return 0; 899 + } 900 + 901 + static size_t vduse_dev_max_mapping_size(union virtio_map token) 902 + { 903 + struct vduse_iova_domain *domain = token.iova_domain; 886 904 887 905 return domain->bounce_size; 888 906 } 889 907 890 - static const struct dma_map_ops vduse_dev_dma_ops = { 908 + static const struct virtio_map_ops vduse_map_ops = { 891 909 .sync_single_for_device = vduse_dev_sync_single_for_device, 892 910 .sync_single_for_cpu = vduse_dev_sync_single_for_cpu, 893 911 .map_page = vduse_dev_map_page, 894 912 .unmap_page = vduse_dev_unmap_page, 895 913 .alloc = vduse_dev_alloc_coherent, 896 914 .free = vduse_dev_free_coherent, 915 + .need_sync = vduse_dev_need_sync, 916 + .mapping_error = vduse_dev_mapping_error, 897 917 .max_mapping_size = vduse_dev_max_mapping_size, 898 918 }; 899 919 ··· 2011 2003 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name) 2012 2004 { 2013 2005 struct vduse_vdpa *vdev; 2014 - int ret; 2015 2006 2016 2007 if (dev->vdev) 2017 2008 return -EEXIST; 2018 2009 2019 2010 vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev, 2020 - &vduse_vdpa_config_ops, 1, 1, name, true); 2011 + &vduse_vdpa_config_ops, &vduse_map_ops, 2012 + 1, 1, name, true); 2021 2013 if (IS_ERR(vdev)) 2022 2014 return PTR_ERR(vdev); 2023 2015 2024 2016 dev->vdev = vdev; 2025 2017 vdev->dev = dev; 2026 - vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask; 2027 - ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64)); 2028 - if (ret) { 2029 - put_device(&vdev->vdpa.dev); 2030 - return ret; 2031 - } 2032 - set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops); 2033 - vdev->vdpa.dma_dev = &vdev->vdpa.dev; 2034 2018 vdev->vdpa.mdev = &vduse_mgmt->mgmt_dev; 2035 2019 2036 2020 return 0; ··· 2055 2055 return -ENOMEM; 2056 2056 } 2057 2057 2058 + dev->vdev->vdpa.vmap.iova_domain = dev->domain; 2058 2059 ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num); 2059 2060 if (ret) { 2060 2061 put_device(&dev->vdev->vdpa.dev);

+3 -2

drivers/vdpa/virtio_pci/vp_vdpa.c

··· 511 511 int ret, i; 512 512 513 513 vp_vdpa = vdpa_alloc_device(struct vp_vdpa, vdpa, 514 - dev, &vp_vdpa_ops, 1, 1, name, false); 514 + dev, &vp_vdpa_ops, NULL, 515 + 1, 1, name, false); 515 516 516 517 if (IS_ERR(vp_vdpa)) { 517 518 dev_err(dev, "vp_vdpa: Failed to allocate vDPA structure\n"); ··· 521 520 522 521 vp_vdpa_mgtdev->vp_vdpa = vp_vdpa; 523 522 524 - vp_vdpa->vdpa.dma_dev = &pdev->dev; 523 + vp_vdpa->vdpa.vmap.dma_dev = &pdev->dev; 525 524 vp_vdpa->queues = vp_modern_get_num_queues(mdev); 526 525 vp_vdpa->mdev = mdev; 527 526

+4 -2

drivers/vhost/vdpa.c

··· 1318 1318 { 1319 1319 struct vdpa_device *vdpa = v->vdpa; 1320 1320 const struct vdpa_config_ops *ops = vdpa->config; 1321 - struct device *dma_dev = vdpa_get_dma_dev(vdpa); 1321 + union virtio_map map = vdpa_get_map(vdpa); 1322 + struct device *dma_dev = map.dma_dev; 1322 1323 int ret; 1323 1324 1324 1325 /* Device want to do DMA by itself */ ··· 1354 1353 static void vhost_vdpa_free_domain(struct vhost_vdpa *v) 1355 1354 { 1356 1355 struct vdpa_device *vdpa = v->vdpa; 1357 - struct device *dma_dev = vdpa_get_dma_dev(vdpa); 1356 + union virtio_map map = vdpa_get_map(vdpa); 1357 + struct device *dma_dev = map.dma_dev; 1358 1358 1359 1359 if (v->domain) { 1360 1360 iommu_detach_device(v->domain, dma_dev);

+4 -3

drivers/vhost/vringh.c

··· 1115 1115 struct iov_iter iter; 1116 1116 u64 translated; 1117 1117 int ret; 1118 + size_t size; 1118 1119 1119 1120 ret = iotlb_translate(vrh, (u64)(uintptr_t)src, 1120 1121 len - total_translated, &translated, ··· 1133 1132 translated); 1134 1133 } 1135 1134 1136 - ret = copy_from_iter(dst, translated, &iter); 1137 - if (ret < 0) 1138 - return ret; 1135 + size = copy_from_iter(dst, translated, &iter); 1136 + if (size != translated) 1137 + return -EFAULT; 1139 1138 1140 1139 src += translated; 1141 1140 dst += translated;

+1 -1

drivers/virtio/virtio_balloon.c

··· 205 205 unsigned int unused, err; 206 206 207 207 /* We should always be able to add these buffers to an empty queue. */ 208 - err = virtqueue_add_inbuf(vq, sg, nents, vb, GFP_NOWAIT | __GFP_NOWARN); 208 + err = virtqueue_add_inbuf(vq, sg, nents, vb, GFP_NOWAIT); 209 209 210 210 /* 211 211 * In the extremely unlikely case that something has occurred and we

+290 -169

drivers/virtio/virtio_ring.c

··· 166 166 bool packed_ring; 167 167 168 168 /* Is DMA API used? */ 169 - bool use_dma_api; 169 + bool use_map_api; 170 170 171 171 /* Can we use weak barriers? */ 172 172 bool weak_barriers; ··· 210 210 /* DMA, allocation, and size information */ 211 211 bool we_own_ring; 212 212 213 - /* Device used for doing DMA */ 214 - struct device *dma_dev; 213 + union virtio_map map; 215 214 216 215 #ifdef DEBUG 217 216 /* They're supposed to lock for us. */ ··· 267 268 * unconditionally on data path. 268 269 */ 269 270 270 - static bool vring_use_dma_api(const struct virtio_device *vdev) 271 + static bool vring_use_map_api(const struct virtio_device *vdev) 271 272 { 272 273 if (!virtio_has_dma_quirk(vdev)) 273 274 return true; ··· 290 291 static bool vring_need_unmap_buffer(const struct vring_virtqueue *vring, 291 292 const struct vring_desc_extra *extra) 292 293 { 293 - return vring->use_dma_api && (extra->addr != DMA_MAPPING_ERROR); 294 + return vring->use_map_api && (extra->addr != DMA_MAPPING_ERROR); 294 295 } 295 296 296 297 size_t virtio_max_dma_size(const struct virtio_device *vdev) 297 298 { 298 299 size_t max_segment_size = SIZE_MAX; 299 300 300 - if (vring_use_dma_api(vdev)) 301 - max_segment_size = dma_max_mapping_size(vdev->dev.parent); 301 + if (vring_use_map_api(vdev)) { 302 + if (vdev->map) { 303 + max_segment_size = 304 + vdev->map->max_mapping_size(vdev->vmap); 305 + } else 306 + max_segment_size = 307 + dma_max_mapping_size(vdev->dev.parent); 308 + } 302 309 303 310 return max_segment_size; 304 311 } 305 312 EXPORT_SYMBOL_GPL(virtio_max_dma_size); 306 313 307 314 static void *vring_alloc_queue(struct virtio_device *vdev, size_t size, 308 - dma_addr_t *dma_handle, gfp_t flag, 309 - struct device *dma_dev) 315 + dma_addr_t *map_handle, gfp_t flag, 316 + union virtio_map map) 310 317 { 311 - if (vring_use_dma_api(vdev)) { 312 - return dma_alloc_coherent(dma_dev, size, 313 - dma_handle, flag); 318 + if (vring_use_map_api(vdev)) { 319 + return virtqueue_map_alloc_coherent(vdev, map, size, 320 + map_handle, flag); 314 321 } else { 315 322 void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag); 316 323 317 324 if (queue) { 318 325 phys_addr_t phys_addr = virt_to_phys(queue); 319 - *dma_handle = (dma_addr_t)phys_addr; 326 + *map_handle = (dma_addr_t)phys_addr; 320 327 321 328 /* 322 329 * Sanity check: make sure we dind't truncate ··· 335 330 * warning and abort if we end up with an 336 331 * unrepresentable address. 337 332 */ 338 - if (WARN_ON_ONCE(*dma_handle != phys_addr)) { 333 + if (WARN_ON_ONCE(*map_handle != phys_addr)) { 339 334 free_pages_exact(queue, PAGE_ALIGN(size)); 340 335 return NULL; 341 336 } ··· 345 340 } 346 341 347 342 static void vring_free_queue(struct virtio_device *vdev, size_t size, 348 - void *queue, dma_addr_t dma_handle, 349 - struct device *dma_dev) 343 + void *queue, dma_addr_t map_handle, 344 + union virtio_map map) 350 345 { 351 - if (vring_use_dma_api(vdev)) 352 - dma_free_coherent(dma_dev, size, queue, dma_handle); 346 + if (vring_use_map_api(vdev)) 347 + virtqueue_map_free_coherent(vdev, map, size, 348 + queue, map_handle); 353 349 else 354 350 free_pages_exact(queue, PAGE_ALIGN(size)); 355 351 } ··· 362 356 */ 363 357 static struct device *vring_dma_dev(const struct vring_virtqueue *vq) 364 358 { 365 - return vq->dma_dev; 359 + return vq->map.dma_dev; 360 + } 361 + 362 + static int vring_mapping_error(const struct vring_virtqueue *vq, 363 + dma_addr_t addr) 364 + { 365 + struct virtio_device *vdev = vq->vq.vdev; 366 + 367 + if (!vq->use_map_api) 368 + return 0; 369 + 370 + if (vdev->map) 371 + return vdev->map->mapping_error(vq->map, addr); 372 + else 373 + return dma_mapping_error(vring_dma_dev(vq), addr); 366 374 } 367 375 368 376 /* Map one sg entry. */ ··· 392 372 393 373 *len = sg->length; 394 374 395 - if (!vq->use_dma_api) { 375 + if (!vq->use_map_api) { 396 376 /* 397 377 * If DMA is not used, KMSAN doesn't know that the scatterlist 398 378 * is initialized by the hardware. Explicitly check/unpoison it ··· 408 388 * the way it expects (we don't guarantee that the scatterlist 409 389 * will exist for the lifetime of the mapping). 410 390 */ 411 - *addr = dma_map_page(vring_dma_dev(vq), 412 - sg_page(sg), sg->offset, sg->length, 413 - direction); 391 + *addr = virtqueue_map_page_attrs(&vq->vq, sg_page(sg), 392 + sg->offset, sg->length, 393 + direction, 0); 414 394 415 - if (dma_mapping_error(vring_dma_dev(vq), *addr)) 395 + if (vring_mapping_error(vq, *addr)) 416 396 return -ENOMEM; 417 397 418 398 return 0; ··· 422 402 void *cpu_addr, size_t size, 423 403 enum dma_data_direction direction) 424 404 { 425 - if (!vq->use_dma_api) 405 + if (!vq->use_map_api) 426 406 return (dma_addr_t)virt_to_phys(cpu_addr); 427 407 428 - return dma_map_single(vring_dma_dev(vq), 429 - cpu_addr, size, direction); 430 - } 431 - 432 - static int vring_mapping_error(const struct vring_virtqueue *vq, 433 - dma_addr_t addr) 434 - { 435 - if (!vq->use_dma_api) 436 - return 0; 437 - 438 - return dma_mapping_error(vring_dma_dev(vq), addr); 408 + return virtqueue_map_single_attrs(&vq->vq, cpu_addr, 409 + size, direction, 0); 439 410 } 440 411 441 412 static void virtqueue_init(struct vring_virtqueue *vq, u32 num) ··· 460 449 flags = extra->flags; 461 450 462 451 if (flags & VRING_DESC_F_INDIRECT) { 463 - if (!vq->use_dma_api) 452 + if (!vq->use_map_api) 464 453 goto out; 454 + } else if (!vring_need_unmap_buffer(vq, extra)) 455 + goto out; 465 456 466 - dma_unmap_single(vring_dma_dev(vq), 467 - extra->addr, 468 - extra->len, 469 - (flags & VRING_DESC_F_WRITE) ? 470 - DMA_FROM_DEVICE : DMA_TO_DEVICE); 471 - } else { 472 - if (!vring_need_unmap_buffer(vq, extra)) 473 - goto out; 474 - 475 - dma_unmap_page(vring_dma_dev(vq), 476 - extra->addr, 477 - extra->len, 478 - (flags & VRING_DESC_F_WRITE) ? 479 - DMA_FROM_DEVICE : DMA_TO_DEVICE); 480 - } 457 + virtqueue_unmap_page_attrs(&vq->vq, 458 + extra->addr, 459 + extra->len, 460 + (flags & VRING_DESC_F_WRITE) ? 461 + DMA_FROM_DEVICE : DMA_TO_DEVICE, 462 + 0); 481 463 482 464 out: 483 465 return extra->next; ··· 794 790 795 791 extra = (struct vring_desc_extra *)&indir_desc[num]; 796 792 797 - if (vq->use_dma_api) { 793 + if (vq->use_map_api) { 798 794 for (j = 0; j < num; j++) 799 795 vring_unmap_one_split(vq, &extra[j]); 800 796 } ··· 1068 1064 } 1069 1065 1070 1066 static void vring_free_split(struct vring_virtqueue_split *vring_split, 1071 - struct virtio_device *vdev, struct device *dma_dev) 1067 + struct virtio_device *vdev, 1068 + union virtio_map map) 1072 1069 { 1073 1070 vring_free_queue(vdev, vring_split->queue_size_in_bytes, 1074 1071 vring_split->vring.desc, 1075 1072 vring_split->queue_dma_addr, 1076 - dma_dev); 1073 + map); 1077 1074 1078 1075 kfree(vring_split->desc_state); 1079 1076 kfree(vring_split->desc_extra); ··· 1085 1080 u32 num, 1086 1081 unsigned int vring_align, 1087 1082 bool may_reduce_num, 1088 - struct device *dma_dev) 1083 + union virtio_map map) 1089 1084 { 1090 1085 void *queue = NULL; 1091 1086 dma_addr_t dma_addr; ··· 1101 1096 queue = vring_alloc_queue(vdev, vring_size(num, vring_align), 1102 1097 &dma_addr, 1103 1098 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 1104 - dma_dev); 1099 + map); 1105 1100 if (queue) 1106 1101 break; 1107 1102 if (!may_reduce_num) ··· 1115 1110 /* Try to get a single page. You are my only hope! */ 1116 1111 queue = vring_alloc_queue(vdev, vring_size(num, vring_align), 1117 1112 &dma_addr, GFP_KERNEL | __GFP_ZERO, 1118 - dma_dev); 1113 + map); 1119 1114 } 1120 1115 if (!queue) 1121 1116 return -ENOMEM; ··· 1139 1134 bool (*notify)(struct virtqueue *), 1140 1135 void (*callback)(struct virtqueue *), 1141 1136 const char *name, 1142 - struct device *dma_dev) 1137 + union virtio_map map) 1143 1138 { 1144 1139 struct vring_virtqueue *vq; 1145 1140 int err; ··· 1162 1157 #else 1163 1158 vq->broken = false; 1164 1159 #endif 1165 - vq->dma_dev = dma_dev; 1166 - vq->use_dma_api = vring_use_dma_api(vdev); 1160 + vq->map = map; 1161 + vq->use_map_api = vring_use_map_api(vdev); 1167 1162 1168 1163 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && 1169 1164 !context; ··· 1200 1195 bool (*notify)(struct virtqueue *), 1201 1196 void (*callback)(struct virtqueue *), 1202 1197 const char *name, 1203 - struct device *dma_dev) 1198 + union virtio_map map) 1204 1199 { 1205 1200 struct vring_virtqueue_split vring_split = {}; 1206 1201 struct virtqueue *vq; 1207 1202 int err; 1208 1203 1209 1204 err = vring_alloc_queue_split(&vring_split, vdev, num, vring_align, 1210 - may_reduce_num, dma_dev); 1205 + may_reduce_num, map); 1211 1206 if (err) 1212 1207 return NULL; 1213 1208 1214 1209 vq = __vring_new_virtqueue_split(index, &vring_split, vdev, weak_barriers, 1215 - context, notify, callback, name, dma_dev); 1210 + context, notify, callback, name, map); 1216 1211 if (!vq) { 1217 - vring_free_split(&vring_split, vdev, dma_dev); 1212 + vring_free_split(&vring_split, vdev, map); 1218 1213 return NULL; 1219 1214 } 1220 1215 ··· 1233 1228 err = vring_alloc_queue_split(&vring_split, vdev, num, 1234 1229 vq->split.vring_align, 1235 1230 vq->split.may_reduce_num, 1236 - vring_dma_dev(vq)); 1231 + vq->map); 1237 1232 if (err) 1238 1233 goto err; 1239 1234 ··· 1251 1246 return 0; 1252 1247 1253 1248 err_state_extra: 1254 - vring_free_split(&vring_split, vdev, vring_dma_dev(vq)); 1249 + vring_free_split(&vring_split, vdev, vq->map); 1255 1250 err: 1256 1251 virtqueue_reinit_split(vq); 1257 1252 return -ENOMEM; ··· 1279 1274 flags = extra->flags; 1280 1275 1281 1276 if (flags & VRING_DESC_F_INDIRECT) { 1282 - if (!vq->use_dma_api) 1277 + if (!vq->use_map_api) 1283 1278 return; 1279 + } else if (!vring_need_unmap_buffer(vq, extra)) 1280 + return; 1284 1281 1285 - dma_unmap_single(vring_dma_dev(vq), 1286 - extra->addr, extra->len, 1287 - (flags & VRING_DESC_F_WRITE) ? 1288 - DMA_FROM_DEVICE : DMA_TO_DEVICE); 1289 - } else { 1290 - if (!vring_need_unmap_buffer(vq, extra)) 1291 - return; 1292 - 1293 - dma_unmap_page(vring_dma_dev(vq), 1294 - extra->addr, extra->len, 1295 - (flags & VRING_DESC_F_WRITE) ? 1296 - DMA_FROM_DEVICE : DMA_TO_DEVICE); 1297 - } 1282 + virtqueue_unmap_page_attrs(&vq->vq, 1283 + extra->addr, extra->len, 1284 + (flags & VRING_DESC_F_WRITE) ? 1285 + DMA_FROM_DEVICE : DMA_TO_DEVICE, 1286 + 0); 1298 1287 } 1299 1288 1300 1289 static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg, ··· 1365 1366 desc[i].addr = cpu_to_le64(addr); 1366 1367 desc[i].len = cpu_to_le32(len); 1367 1368 1368 - if (unlikely(vq->use_dma_api)) { 1369 + if (unlikely(vq->use_map_api)) { 1369 1370 extra[i].addr = premapped ? DMA_MAPPING_ERROR : addr; 1370 1371 extra[i].len = len; 1371 1372 extra[i].flags = n < out_sgs ? 0 : VRING_DESC_F_WRITE; ··· 1387 1388 sizeof(struct vring_packed_desc)); 1388 1389 vq->packed.vring.desc[head].id = cpu_to_le16(id); 1389 1390 1390 - if (vq->use_dma_api) { 1391 + if (vq->use_map_api) { 1391 1392 vq->packed.desc_extra[id].addr = addr; 1392 1393 vq->packed.desc_extra[id].len = total_sg * 1393 1394 sizeof(struct vring_packed_desc); ··· 1529 1530 desc[i].len = cpu_to_le32(len); 1530 1531 desc[i].id = cpu_to_le16(id); 1531 1532 1532 - if (unlikely(vq->use_dma_api)) { 1533 + if (unlikely(vq->use_map_api)) { 1533 1534 vq->packed.desc_extra[curr].addr = premapped ? 1534 1535 DMA_MAPPING_ERROR : addr; 1535 1536 vq->packed.desc_extra[curr].len = len; ··· 1664 1665 vq->free_head = id; 1665 1666 vq->vq.num_free += state->num; 1666 1667 1667 - if (unlikely(vq->use_dma_api)) { 1668 + if (unlikely(vq->use_map_api)) { 1668 1669 curr = id; 1669 1670 for (i = 0; i < state->num; i++) { 1670 1671 vring_unmap_extra_packed(vq, ··· 1682 1683 if (!desc) 1683 1684 return; 1684 1685 1685 - if (vq->use_dma_api) { 1686 + if (vq->use_map_api) { 1686 1687 len = vq->packed.desc_extra[id].len; 1687 1688 num = len / sizeof(struct vring_packed_desc); 1688 1689 ··· 1961 1962 1962 1963 static void vring_free_packed(struct vring_virtqueue_packed *vring_packed, 1963 1964 struct virtio_device *vdev, 1964 - struct device *dma_dev) 1965 + union virtio_map map) 1965 1966 { 1966 1967 if (vring_packed->vring.desc) 1967 1968 vring_free_queue(vdev, vring_packed->ring_size_in_bytes, 1968 1969 vring_packed->vring.desc, 1969 1970 vring_packed->ring_dma_addr, 1970 - dma_dev); 1971 + map); 1971 1972 1972 1973 if (vring_packed->vring.driver) 1973 1974 vring_free_queue(vdev, vring_packed->event_size_in_bytes, 1974 1975 vring_packed->vring.driver, 1975 1976 vring_packed->driver_event_dma_addr, 1976 - dma_dev); 1977 + map); 1977 1978 1978 1979 if (vring_packed->vring.device) 1979 1980 vring_free_queue(vdev, vring_packed->event_size_in_bytes, 1980 1981 vring_packed->vring.device, 1981 1982 vring_packed->device_event_dma_addr, 1982 - dma_dev); 1983 + map); 1983 1984 1984 1985 kfree(vring_packed->desc_state); 1985 1986 kfree(vring_packed->desc_extra); ··· 1987 1988 1988 1989 static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed, 1989 1990 struct virtio_device *vdev, 1990 - u32 num, struct device *dma_dev) 1991 + u32 num, union virtio_map map) 1991 1992 { 1992 1993 struct vring_packed_desc *ring; 1993 1994 struct vring_packed_desc_event *driver, *device; ··· 1999 2000 ring = vring_alloc_queue(vdev, ring_size_in_bytes, 2000 2001 &ring_dma_addr, 2001 2002 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 2002 - dma_dev); 2003 + map); 2003 2004 if (!ring) 2004 2005 goto err; 2005 2006 ··· 2012 2013 driver = vring_alloc_queue(vdev, event_size_in_bytes, 2013 2014 &driver_event_dma_addr, 2014 2015 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 2015 - dma_dev); 2016 + map); 2016 2017 if (!driver) 2017 2018 goto err; 2018 2019 ··· 2023 2024 device = vring_alloc_queue(vdev, event_size_in_bytes, 2024 2025 &device_event_dma_addr, 2025 2026 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 2026 - dma_dev); 2027 + map); 2027 2028 if (!device) 2028 2029 goto err; 2029 2030 ··· 2035 2036 return 0; 2036 2037 2037 2038 err: 2038 - vring_free_packed(vring_packed, vdev, dma_dev); 2039 + vring_free_packed(vring_packed, vdev, map); 2039 2040 return -ENOMEM; 2040 2041 } 2041 2042 ··· 2111 2112 bool (*notify)(struct virtqueue *), 2112 2113 void (*callback)(struct virtqueue *), 2113 2114 const char *name, 2114 - struct device *dma_dev) 2115 + union virtio_map map) 2115 2116 { 2116 2117 struct vring_virtqueue *vq; 2117 2118 int err; ··· 2134 2135 vq->broken = false; 2135 2136 #endif 2136 2137 vq->packed_ring = true; 2137 - vq->dma_dev = dma_dev; 2138 - vq->use_dma_api = vring_use_dma_api(vdev); 2138 + vq->map = map; 2139 + vq->use_map_api = vring_use_map_api(vdev); 2139 2140 2140 2141 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && 2141 2142 !context; ··· 2172 2173 bool (*notify)(struct virtqueue *), 2173 2174 void (*callback)(struct virtqueue *), 2174 2175 const char *name, 2175 - struct device *dma_dev) 2176 + union virtio_map map) 2176 2177 { 2177 2178 struct vring_virtqueue_packed vring_packed = {}; 2178 2179 struct virtqueue *vq; 2179 2180 2180 - if (vring_alloc_queue_packed(&vring_packed, vdev, num, dma_dev)) 2181 + if (vring_alloc_queue_packed(&vring_packed, vdev, num, map)) 2181 2182 return NULL; 2182 2183 2183 2184 vq = __vring_new_virtqueue_packed(index, &vring_packed, vdev, weak_barriers, 2184 - context, notify, callback, name, dma_dev); 2185 + context, notify, callback, name, map); 2185 2186 if (!vq) { 2186 - vring_free_packed(&vring_packed, vdev, dma_dev); 2187 + vring_free_packed(&vring_packed, vdev, map); 2187 2188 return NULL; 2188 2189 } 2189 2190 ··· 2199 2200 struct virtio_device *vdev = _vq->vdev; 2200 2201 int err; 2201 2202 2202 - if (vring_alloc_queue_packed(&vring_packed, vdev, num, vring_dma_dev(vq))) 2203 + if (vring_alloc_queue_packed(&vring_packed, vdev, num, vq->map)) 2203 2204 goto err_ring; 2204 2205 2205 2206 err = vring_alloc_state_extra_packed(&vring_packed); ··· 2216 2217 return 0; 2217 2218 2218 2219 err_state_extra: 2219 - vring_free_packed(&vring_packed, vdev, vring_dma_dev(vq)); 2220 + vring_free_packed(&vring_packed, vdev, vq->map); 2220 2221 err_ring: 2221 2222 virtqueue_reinit_packed(vq); 2222 2223 return -ENOMEM; ··· 2447 2448 { 2448 2449 struct vring_virtqueue *vq = to_vvq(_vq); 2449 2450 2450 - if (vq->use_dma_api) 2451 - return vring_dma_dev(vq); 2451 + if (vq->use_map_api && !_vq->vdev->map) 2452 + return vq->map.dma_dev; 2452 2453 else 2453 2454 return NULL; 2454 2455 } ··· 2733 2734 void (*callback)(struct virtqueue *), 2734 2735 const char *name) 2735 2736 { 2737 + union virtio_map map = {.dma_dev = vdev->dev.parent}; 2736 2738 2737 2739 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) 2738 2740 return vring_create_virtqueue_packed(index, num, vring_align, 2739 2741 vdev, weak_barriers, may_reduce_num, 2740 - context, notify, callback, name, vdev->dev.parent); 2742 + context, notify, callback, name, map); 2741 2743 2742 2744 return vring_create_virtqueue_split(index, num, vring_align, 2743 2745 vdev, weak_barriers, may_reduce_num, 2744 - context, notify, callback, name, vdev->dev.parent); 2746 + context, notify, callback, name, map); 2745 2747 } 2746 2748 EXPORT_SYMBOL_GPL(vring_create_virtqueue); 2747 2749 2748 - struct virtqueue *vring_create_virtqueue_dma( 2750 + struct virtqueue *vring_create_virtqueue_map( 2749 2751 unsigned int index, 2750 2752 unsigned int num, 2751 2753 unsigned int vring_align, ··· 2757 2757 bool (*notify)(struct virtqueue *), 2758 2758 void (*callback)(struct virtqueue *), 2759 2759 const char *name, 2760 - struct device *dma_dev) 2760 + union virtio_map map) 2761 2761 { 2762 2762 2763 2763 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) 2764 2764 return vring_create_virtqueue_packed(index, num, vring_align, 2765 2765 vdev, weak_barriers, may_reduce_num, 2766 - context, notify, callback, name, dma_dev); 2766 + context, notify, callback, name, map); 2767 2767 2768 2768 return vring_create_virtqueue_split(index, num, vring_align, 2769 2769 vdev, weak_barriers, may_reduce_num, 2770 - context, notify, callback, name, dma_dev); 2770 + context, notify, callback, name, map); 2771 2771 } 2772 - EXPORT_SYMBOL_GPL(vring_create_virtqueue_dma); 2772 + EXPORT_SYMBOL_GPL(vring_create_virtqueue_map); 2773 2773 2774 2774 /** 2775 2775 * virtqueue_resize - resize the vring of vq ··· 2880 2880 const char *name) 2881 2881 { 2882 2882 struct vring_virtqueue_split vring_split = {}; 2883 + union virtio_map map = {.dma_dev = vdev->dev.parent}; 2883 2884 2884 2885 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) { 2885 2886 struct vring_virtqueue_packed vring_packed = {}; ··· 2890 2889 return __vring_new_virtqueue_packed(index, &vring_packed, 2891 2890 vdev, weak_barriers, 2892 2891 context, notify, callback, 2893 - name, vdev->dev.parent); 2892 + name, map); 2894 2893 } 2895 2894 2896 2895 vring_init(&vring_split.vring, num, pages, vring_align); 2897 2896 return __vring_new_virtqueue_split(index, &vring_split, vdev, weak_barriers, 2898 2897 context, notify, callback, name, 2899 - vdev->dev.parent); 2898 + map); 2900 2899 } 2901 2900 EXPORT_SYMBOL_GPL(vring_new_virtqueue); 2902 2901 ··· 2910 2909 vq->packed.ring_size_in_bytes, 2911 2910 vq->packed.vring.desc, 2912 2911 vq->packed.ring_dma_addr, 2913 - vring_dma_dev(vq)); 2912 + vq->map); 2914 2913 2915 2914 vring_free_queue(vq->vq.vdev, 2916 2915 vq->packed.event_size_in_bytes, 2917 2916 vq->packed.vring.driver, 2918 2917 vq->packed.driver_event_dma_addr, 2919 - vring_dma_dev(vq)); 2918 + vq->map); 2920 2919 2921 2920 vring_free_queue(vq->vq.vdev, 2922 2921 vq->packed.event_size_in_bytes, 2923 2922 vq->packed.vring.device, 2924 2923 vq->packed.device_event_dma_addr, 2925 - vring_dma_dev(vq)); 2924 + vq->map); 2926 2925 2927 2926 kfree(vq->packed.desc_state); 2928 2927 kfree(vq->packed.desc_extra); ··· 2931 2930 vq->split.queue_size_in_bytes, 2932 2931 vq->split.vring.desc, 2933 2932 vq->split.queue_dma_addr, 2934 - vring_dma_dev(vq)); 2933 + vq->map); 2935 2934 } 2936 2935 } 2937 2936 if (!vq->packed_ring) { ··· 3138 3137 EXPORT_SYMBOL_GPL(virtqueue_get_vring); 3139 3138 3140 3139 /** 3141 - * virtqueue_dma_map_single_attrs - map DMA for _vq 3140 + * virtqueue_map_alloc_coherent - alloc coherent mapping 3141 + * @vdev: the virtio device we are talking to 3142 + * @map: metadata for performing mapping 3143 + * @size: the size of the buffer 3144 + * @map_handle: the pointer to the mapped address 3145 + * @gfp: allocation flag (GFP_XXX) 3146 + * 3147 + * return virtual address or NULL on error 3148 + */ 3149 + void *virtqueue_map_alloc_coherent(struct virtio_device *vdev, 3150 + union virtio_map map, 3151 + size_t size, dma_addr_t *map_handle, 3152 + gfp_t gfp) 3153 + { 3154 + if (vdev->map) 3155 + return vdev->map->alloc(map, size, 3156 + map_handle, gfp); 3157 + else 3158 + return dma_alloc_coherent(map.dma_dev, size, 3159 + map_handle, gfp); 3160 + } 3161 + EXPORT_SYMBOL_GPL(virtqueue_map_alloc_coherent); 3162 + 3163 + /** 3164 + * virtqueue_map_free_coherent - free coherent mapping 3165 + * @vdev: the virtio device we are talking to 3166 + * @map: metadata for performing mapping 3167 + * @size: the size of the buffer 3168 + * @map_handle: the mapped address that needs to be freed 3169 + * 3170 + */ 3171 + void virtqueue_map_free_coherent(struct virtio_device *vdev, 3172 + union virtio_map map, size_t size, void *vaddr, 3173 + dma_addr_t map_handle) 3174 + { 3175 + if (vdev->map) 3176 + vdev->map->free(map, size, vaddr, 3177 + map_handle, 0); 3178 + else 3179 + dma_free_coherent(map.dma_dev, size, vaddr, map_handle); 3180 + } 3181 + EXPORT_SYMBOL_GPL(virtqueue_map_free_coherent); 3182 + 3183 + /** 3184 + * virtqueue_map_page_attrs - map a page to the device 3185 + * @_vq: the virtqueue we are talking to 3186 + * @page: the page that will be mapped by the device 3187 + * @offset: the offset in the page for a buffer 3188 + * @size: the buffer size 3189 + * @dir: mapping direction 3190 + * @attrs: mapping attributes 3191 + * 3192 + * Returns mapped address. Caller should check that by virtqueue_mapping_error(). 3193 + */ 3194 + dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq, 3195 + struct page *page, 3196 + unsigned long offset, 3197 + size_t size, 3198 + enum dma_data_direction dir, 3199 + unsigned long attrs) 3200 + { 3201 + const struct vring_virtqueue *vq = to_vvq(_vq); 3202 + struct virtio_device *vdev = _vq->vdev; 3203 + 3204 + if (vdev->map) 3205 + return vdev->map->map_page(vq->map, 3206 + page, offset, size, 3207 + dir, attrs); 3208 + 3209 + return dma_map_page_attrs(vring_dma_dev(vq), 3210 + page, offset, size, 3211 + dir, attrs); 3212 + } 3213 + EXPORT_SYMBOL_GPL(virtqueue_map_page_attrs); 3214 + 3215 + /** 3216 + * virtqueue_unmap_page_attrs - map a page to the device 3217 + * @_vq: the virtqueue we are talking to 3218 + * @map_handle: the mapped address 3219 + * @size: the buffer size 3220 + * @dir: mapping direction 3221 + * @attrs: unmapping attributes 3222 + */ 3223 + void virtqueue_unmap_page_attrs(const struct virtqueue *_vq, 3224 + dma_addr_t map_handle, 3225 + size_t size, enum dma_data_direction dir, 3226 + unsigned long attrs) 3227 + { 3228 + const struct vring_virtqueue *vq = to_vvq(_vq); 3229 + struct virtio_device *vdev = _vq->vdev; 3230 + 3231 + if (vdev->map) 3232 + vdev->map->unmap_page(vq->map, 3233 + map_handle, size, dir, attrs); 3234 + else 3235 + dma_unmap_page_attrs(vring_dma_dev(vq), map_handle, 3236 + size, dir, attrs); 3237 + } 3238 + EXPORT_SYMBOL_GPL(virtqueue_unmap_page_attrs); 3239 + 3240 + /** 3241 + * virtqueue_map_single_attrs - map DMA for _vq 3142 3242 * @_vq: the struct virtqueue we're talking about. 3143 3243 * @ptr: the pointer of the buffer to do dma 3144 3244 * @size: the size of the buffer to do dma ··· 3249 3147 * The caller calls this to do dma mapping in advance. The DMA address can be 3250 3148 * passed to this _vq when it is in pre-mapped mode. 3251 3149 * 3252 - * return DMA address. Caller should check that by virtqueue_dma_mapping_error(). 3150 + * return mapped address. Caller should check that by virtqueue_mapping_error(). 3253 3151 */ 3254 - dma_addr_t virtqueue_dma_map_single_attrs(struct virtqueue *_vq, void *ptr, 3255 - size_t size, 3256 - enum dma_data_direction dir, 3257 - unsigned long attrs) 3152 + dma_addr_t virtqueue_map_single_attrs(const struct virtqueue *_vq, void *ptr, 3153 + size_t size, 3154 + enum dma_data_direction dir, 3155 + unsigned long attrs) 3258 3156 { 3259 - struct vring_virtqueue *vq = to_vvq(_vq); 3157 + const struct vring_virtqueue *vq = to_vvq(_vq); 3260 3158 3261 - if (!vq->use_dma_api) { 3159 + if (!vq->use_map_api) { 3262 3160 kmsan_handle_dma(virt_to_phys(ptr), size, dir); 3263 3161 return (dma_addr_t)virt_to_phys(ptr); 3264 3162 } 3265 3163 3266 - return dma_map_single_attrs(vring_dma_dev(vq), ptr, size, dir, attrs); 3164 + /* DMA must never operate on areas that might be remapped. */ 3165 + if (dev_WARN_ONCE(&_vq->vdev->dev, is_vmalloc_addr(ptr), 3166 + "rejecting DMA map of vmalloc memory\n")) 3167 + return DMA_MAPPING_ERROR; 3168 + 3169 + return virtqueue_map_page_attrs(&vq->vq, virt_to_page(ptr), 3170 + offset_in_page(ptr), size, dir, attrs); 3267 3171 } 3268 - EXPORT_SYMBOL_GPL(virtqueue_dma_map_single_attrs); 3172 + EXPORT_SYMBOL_GPL(virtqueue_map_single_attrs); 3269 3173 3270 3174 /** 3271 - * virtqueue_dma_unmap_single_attrs - unmap DMA for _vq 3175 + * virtqueue_unmap_single_attrs - unmap map for _vq 3272 3176 * @_vq: the struct virtqueue we're talking about. 3273 3177 * @addr: the dma address to unmap 3274 3178 * @size: the size of the buffer 3275 3179 * @dir: DMA direction 3276 3180 * @attrs: DMA Attrs 3277 3181 * 3278 - * Unmap the address that is mapped by the virtqueue_dma_map_* APIs. 3182 + * Unmap the address that is mapped by the virtqueue_map_* APIs. 3279 3183 * 3280 3184 */ 3281 - void virtqueue_dma_unmap_single_attrs(struct virtqueue *_vq, dma_addr_t addr, 3282 - size_t size, enum dma_data_direction dir, 3283 - unsigned long attrs) 3185 + void virtqueue_unmap_single_attrs(const struct virtqueue *_vq, 3186 + dma_addr_t addr, 3187 + size_t size, enum dma_data_direction dir, 3188 + unsigned long attrs) 3284 3189 { 3285 - struct vring_virtqueue *vq = to_vvq(_vq); 3190 + const struct vring_virtqueue *vq = to_vvq(_vq); 3286 3191 3287 - if (!vq->use_dma_api) 3192 + if (!vq->use_map_api) 3288 3193 return; 3289 3194 3290 - dma_unmap_single_attrs(vring_dma_dev(vq), addr, size, dir, attrs); 3195 + virtqueue_unmap_page_attrs(_vq, addr, size, dir, attrs); 3291 3196 } 3292 - EXPORT_SYMBOL_GPL(virtqueue_dma_unmap_single_attrs); 3197 + EXPORT_SYMBOL_GPL(virtqueue_unmap_single_attrs); 3293 3198 3294 3199 /** 3295 - * virtqueue_dma_mapping_error - check dma address 3200 + * virtqueue_mapping_error - check dma address 3296 3201 * @_vq: the struct virtqueue we're talking about. 3297 3202 * @addr: DMA address 3298 3203 * 3299 3204 * Returns 0 means dma valid. Other means invalid dma address. 3300 3205 */ 3301 - int virtqueue_dma_mapping_error(struct virtqueue *_vq, dma_addr_t addr) 3206 + int virtqueue_map_mapping_error(const struct virtqueue *_vq, dma_addr_t addr) 3302 3207 { 3303 - struct vring_virtqueue *vq = to_vvq(_vq); 3208 + const struct vring_virtqueue *vq = to_vvq(_vq); 3304 3209 3305 - if (!vq->use_dma_api) 3306 - return 0; 3307 - 3308 - return dma_mapping_error(vring_dma_dev(vq), addr); 3210 + return vring_mapping_error(vq, addr); 3309 3211 } 3310 - EXPORT_SYMBOL_GPL(virtqueue_dma_mapping_error); 3212 + EXPORT_SYMBOL_GPL(virtqueue_map_mapping_error); 3311 3213 3312 3214 /** 3313 - * virtqueue_dma_need_sync - check a dma address needs sync 3215 + * virtqueue_map_need_sync - check a dma address needs sync 3314 3216 * @_vq: the struct virtqueue we're talking about. 3315 3217 * @addr: DMA address 3316 3218 * 3317 - * Check if the dma address mapped by the virtqueue_dma_map_* APIs needs to be 3219 + * Check if the dma address mapped by the virtqueue_map_* APIs needs to be 3318 3220 * synchronized 3319 3221 * 3320 3222 * return bool 3321 3223 */ 3322 - bool virtqueue_dma_need_sync(struct virtqueue *_vq, dma_addr_t addr) 3224 + bool virtqueue_map_need_sync(const struct virtqueue *_vq, dma_addr_t addr) 3323 3225 { 3324 - struct vring_virtqueue *vq = to_vvq(_vq); 3226 + const struct vring_virtqueue *vq = to_vvq(_vq); 3227 + struct virtio_device *vdev = _vq->vdev; 3325 3228 3326 - if (!vq->use_dma_api) 3229 + if (!vq->use_map_api) 3327 3230 return false; 3328 3231 3329 - return dma_need_sync(vring_dma_dev(vq), addr); 3232 + if (vdev->map) 3233 + return vdev->map->need_sync(vq->map, addr); 3234 + else 3235 + return dma_need_sync(vring_dma_dev(vq), addr); 3330 3236 } 3331 - EXPORT_SYMBOL_GPL(virtqueue_dma_need_sync); 3237 + EXPORT_SYMBOL_GPL(virtqueue_map_need_sync); 3332 3238 3333 3239 /** 3334 - * virtqueue_dma_sync_single_range_for_cpu - dma sync for cpu 3240 + * virtqueue_map_sync_single_range_for_cpu - map sync for cpu 3335 3241 * @_vq: the struct virtqueue we're talking about. 3336 3242 * @addr: DMA address 3337 3243 * @offset: DMA address offset 3338 3244 * @size: buf size for sync 3339 3245 * @dir: DMA direction 3340 3246 * 3341 - * Before calling this function, use virtqueue_dma_need_sync() to confirm that 3247 + * Before calling this function, use virtqueue_map_need_sync() to confirm that 3342 3248 * the DMA address really needs to be synchronized 3343 3249 * 3344 3250 */ 3345 - void virtqueue_dma_sync_single_range_for_cpu(struct virtqueue *_vq, 3251 + void virtqueue_map_sync_single_range_for_cpu(const struct virtqueue *_vq, 3346 3252 dma_addr_t addr, 3347 3253 unsigned long offset, size_t size, 3348 3254 enum dma_data_direction dir) 3349 3255 { 3350 - struct vring_virtqueue *vq = to_vvq(_vq); 3351 - struct device *dev = vring_dma_dev(vq); 3256 + const struct vring_virtqueue *vq = to_vvq(_vq); 3257 + struct virtio_device *vdev = _vq->vdev; 3352 3258 3353 - if (!vq->use_dma_api) 3259 + if (!vq->use_map_api) 3354 3260 return; 3355 3261 3356 - dma_sync_single_range_for_cpu(dev, addr, offset, size, dir); 3262 + if (vdev->map) 3263 + vdev->map->sync_single_for_cpu(vq->map, 3264 + addr + offset, size, dir); 3265 + else 3266 + dma_sync_single_range_for_cpu(vring_dma_dev(vq), 3267 + addr, offset, size, dir); 3357 3268 } 3358 - EXPORT_SYMBOL_GPL(virtqueue_dma_sync_single_range_for_cpu); 3269 + EXPORT_SYMBOL_GPL(virtqueue_map_sync_single_range_for_cpu); 3359 3270 3360 3271 /** 3361 - * virtqueue_dma_sync_single_range_for_device - dma sync for device 3272 + * virtqueue_map_sync_single_range_for_device - map sync for device 3362 3273 * @_vq: the struct virtqueue we're talking about. 3363 3274 * @addr: DMA address 3364 3275 * @offset: DMA address offset 3365 3276 * @size: buf size for sync 3366 3277 * @dir: DMA direction 3367 3278 * 3368 - * Before calling this function, use virtqueue_dma_need_sync() to confirm that 3279 + * Before calling this function, use virtqueue_map_need_sync() to confirm that 3369 3280 * the DMA address really needs to be synchronized 3370 3281 */ 3371 - void virtqueue_dma_sync_single_range_for_device(struct virtqueue *_vq, 3282 + void virtqueue_map_sync_single_range_for_device(const struct virtqueue *_vq, 3372 3283 dma_addr_t addr, 3373 3284 unsigned long offset, size_t size, 3374 3285 enum dma_data_direction dir) 3375 3286 { 3376 - struct vring_virtqueue *vq = to_vvq(_vq); 3377 - struct device *dev = vring_dma_dev(vq); 3287 + const struct vring_virtqueue *vq = to_vvq(_vq); 3288 + struct virtio_device *vdev = _vq->vdev; 3378 3289 3379 - if (!vq->use_dma_api) 3290 + if (!vq->use_map_api) 3380 3291 return; 3381 3292 3382 - dma_sync_single_range_for_device(dev, addr, offset, size, dir); 3293 + if (vdev->map) 3294 + vdev->map->sync_single_for_device(vq->map, 3295 + addr + offset, 3296 + size, dir); 3297 + else 3298 + dma_sync_single_range_for_device(vring_dma_dev(vq), addr, 3299 + offset, size, dir); 3383 3300 } 3384 - EXPORT_SYMBOL_GPL(virtqueue_dma_sync_single_range_for_device); 3301 + EXPORT_SYMBOL_GPL(virtqueue_map_sync_single_range_for_device); 3385 3302 3386 3303 MODULE_DESCRIPTION("Virtio ring implementation"); 3387 3304 MODULE_LICENSE("GPL");

+14 -8

drivers/virtio/virtio_vdpa.c

··· 133 133 const char *name, bool ctx) 134 134 { 135 135 struct vdpa_device *vdpa = vd_get_vdpa(vdev); 136 - struct device *dma_dev; 137 136 const struct vdpa_config_ops *ops = vdpa->config; 138 137 bool (*notify)(struct virtqueue *vq) = virtio_vdpa_notify; 139 138 struct vdpa_callback cb; 140 139 struct virtqueue *vq; 141 140 u64 desc_addr, driver_addr, device_addr; 141 + union virtio_map map = {0}; 142 142 /* Assume split virtqueue, switch to packed if necessary */ 143 143 struct vdpa_vq_state state = {0}; 144 144 u32 align, max_num, min_num = 1; ··· 176 176 if (ops->get_vq_num_min) 177 177 min_num = ops->get_vq_num_min(vdpa); 178 178 179 - may_reduce_num = (max_num == min_num) ? false : true; 179 + may_reduce_num = (max_num != min_num); 180 180 181 181 /* Create the vring */ 182 182 align = ops->get_vq_align(vdpa); 183 183 184 - if (ops->get_vq_dma_dev) 185 - dma_dev = ops->get_vq_dma_dev(vdpa, index); 184 + if (ops->get_vq_map) 185 + map = ops->get_vq_map(vdpa, index); 186 186 else 187 - dma_dev = vdpa_get_dma_dev(vdpa); 188 - vq = vring_create_virtqueue_dma(index, max_num, align, vdev, 187 + map = vdpa_get_map(vdpa); 188 + 189 + vq = vring_create_virtqueue_map(index, max_num, align, vdev, 189 190 true, may_reduce_num, ctx, 190 - notify, callback, name, dma_dev); 191 + notify, callback, name, map); 191 192 if (!vq) { 192 193 err = -ENOMEM; 193 194 goto error_new_virtqueue; 194 195 } 196 + 197 + if (index == 0) 198 + vdev->vmap = map; 195 199 196 200 vq->num_max = max_num; 197 201 ··· 466 462 if (!vd_dev) 467 463 return -ENOMEM; 468 464 469 - vd_dev->vdev.dev.parent = vdpa_get_dma_dev(vdpa); 465 + vd_dev->vdev.dev.parent = vdpa->map ? &vdpa->dev : 466 + vdpa_get_map(vdpa).dma_dev; 470 467 vd_dev->vdev.dev.release = virtio_vdpa_release_dev; 471 468 vd_dev->vdev.config = &virtio_vdpa_config_ops; 469 + vd_dev->vdev.map = vdpa->map; 472 470 vd_dev->vdpa = vdpa; 473 471 474 472 vd_dev->vdev.id.device = ops->get_device_id(vdpa);

+15 -10

include/linux/vdpa.h

··· 5 5 #include <linux/kernel.h> 6 6 #include <linux/device.h> 7 7 #include <linux/interrupt.h> 8 + #include <linux/virtio.h> 8 9 #include <linux/vhost_iotlb.h> 9 10 #include <linux/virtio_net.h> 10 11 #include <linux/virtio_blk.h> ··· 71 70 /** 72 71 * struct vdpa_device - representation of a vDPA device 73 72 * @dev: underlying device 74 - * @dma_dev: the actual device that is performing DMA 73 + * @vmap: the metadata passed to upper layer to be used for mapping 75 74 * @driver_override: driver name to force a match; do not set directly, 76 75 * because core frees it; use driver_set_override() to 77 76 * set or clear it. 78 77 * @config: the configuration ops for this device. 78 + * @map: the map ops for this device 79 79 * @cf_lock: Protects get and set access to configuration layout. 80 80 * @index: device index 81 81 * @features_valid: were features initialized? for legacy guests ··· 89 87 */ 90 88 struct vdpa_device { 91 89 struct device dev; 92 - struct device *dma_dev; 90 + union virtio_map vmap; 93 91 const char *driver_override; 94 92 const struct vdpa_config_ops *config; 93 + const struct virtio_map_ops *map; 95 94 struct rw_semaphore cf_lock; /* Protects get/set config */ 96 95 unsigned int index; 97 96 bool features_valid; ··· 355 352 * @vdev: vdpa device 356 353 * @asid: address space identifier 357 354 * Returns integer: success (0) or error (< 0) 358 - * @get_vq_dma_dev: Get the dma device for a specific 355 + * @get_vq_map: Get the map metadata for a specific 359 356 * virtqueue (optional) 360 357 * @vdev: vdpa device 361 358 * @idx: virtqueue index 362 - * Returns pointer to structure device or error (NULL) 359 + * Returns map token union error (NULL) 363 360 * @bind_mm: Bind the device to a specific address space 364 361 * so the vDPA framework can use VA when this 365 362 * callback is implemented. (optional) ··· 439 436 int (*reset_map)(struct vdpa_device *vdev, unsigned int asid); 440 437 int (*set_group_asid)(struct vdpa_device *vdev, unsigned int group, 441 438 unsigned int asid); 442 - struct device *(*get_vq_dma_dev)(struct vdpa_device *vdev, u16 idx); 439 + union virtio_map (*get_vq_map)(struct vdpa_device *vdev, u16 idx); 443 440 int (*bind_mm)(struct vdpa_device *vdev, struct mm_struct *mm); 444 441 void (*unbind_mm)(struct vdpa_device *vdev); 445 442 ··· 449 446 450 447 struct vdpa_device *__vdpa_alloc_device(struct device *parent, 451 448 const struct vdpa_config_ops *config, 449 + const struct virtio_map_ops *map, 452 450 unsigned int ngroups, unsigned int nas, 453 451 size_t size, const char *name, 454 452 bool use_va); ··· 461 457 * @member: the name of struct vdpa_device within the @dev_struct 462 458 * @parent: the parent device 463 459 * @config: the bus operations that is supported by this device 460 + * @map: the map operations that is supported by this device 464 461 * @ngroups: the number of virtqueue groups supported by this device 465 462 * @nas: the number of address spaces 466 463 * @name: name of the vdpa device ··· 469 464 * 470 465 * Return allocated data structure or ERR_PTR upon error 471 466 */ 472 - #define vdpa_alloc_device(dev_struct, member, parent, config, ngroups, nas, \ 473 - name, use_va) \ 467 + #define vdpa_alloc_device(dev_struct, member, parent, config, map, \ 468 + ngroups, nas, name, use_va) \ 474 469 container_of((__vdpa_alloc_device( \ 475 - parent, config, ngroups, nas, \ 470 + parent, config, map, ngroups, nas, \ 476 471 (sizeof(dev_struct) + \ 477 472 BUILD_BUG_ON_ZERO(offsetof( \ 478 473 dev_struct, member))), name, use_va)), \ ··· 525 520 dev_set_drvdata(&vdev->dev, data); 526 521 } 527 522 528 - static inline struct device *vdpa_get_dma_dev(struct vdpa_device *vdev) 523 + static inline union virtio_map vdpa_get_map(struct vdpa_device *vdev) 529 524 { 530 - return vdev->dma_dev; 525 + return vdev->vmap; 531 526 } 532 527 533 528 static inline int vdpa_reset(struct vdpa_device *vdev, u32 flags)

+40 -6

include/linux/virtio.h

··· 41 41 void *priv; 42 42 }; 43 43 44 + struct vduse_iova_domain; 45 + 46 + union virtio_map { 47 + /* Device that performs DMA */ 48 + struct device *dma_dev; 49 + /* VDUSE specific mapping data */ 50 + struct vduse_iova_domain *iova_domain; 51 + }; 52 + 44 53 int virtqueue_add_outbuf(struct virtqueue *vq, 45 54 struct scatterlist sg[], unsigned int num, 46 55 void *data, ··· 170 161 struct virtio_device_id id; 171 162 const struct virtio_config_ops *config; 172 163 const struct vringh_config_ops *vringh_config; 164 + const struct virtio_map_ops *map; 173 165 struct list_head vqs; 174 166 VIRTIO_DECLARE_FEATURES(features); 175 167 void *priv; 168 + union virtio_map vmap; 176 169 #ifdef CONFIG_VIRTIO_DEBUG 177 170 struct dentry *debugfs_dir; 178 171 u64 debugfs_filter_features[VIRTIO_FEATURES_DWORDS]; ··· 273 262 module_driver(__virtio_driver, register_virtio_driver, \ 274 263 unregister_virtio_driver) 275 264 276 - dma_addr_t virtqueue_dma_map_single_attrs(struct virtqueue *_vq, void *ptr, size_t size, 265 + 266 + void *virtqueue_map_alloc_coherent(struct virtio_device *vdev, 267 + union virtio_map mapping_token, 268 + size_t size, dma_addr_t *dma_handle, 269 + gfp_t gfp); 270 + 271 + void virtqueue_map_free_coherent(struct virtio_device *vdev, 272 + union virtio_map mapping_token, 273 + size_t size, void *vaddr, 274 + dma_addr_t dma_handle); 275 + 276 + dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq, 277 + struct page *page, 278 + unsigned long offset, 279 + size_t size, 280 + enum dma_data_direction dir, 281 + unsigned long attrs); 282 + 283 + void virtqueue_unmap_page_attrs(const struct virtqueue *_vq, 284 + dma_addr_t dma_handle, 285 + size_t size, enum dma_data_direction dir, 286 + unsigned long attrs); 287 + 288 + dma_addr_t virtqueue_map_single_attrs(const struct virtqueue *_vq, void *ptr, size_t size, 277 289 enum dma_data_direction dir, unsigned long attrs); 278 - void virtqueue_dma_unmap_single_attrs(struct virtqueue *_vq, dma_addr_t addr, 290 + void virtqueue_unmap_single_attrs(const struct virtqueue *_vq, dma_addr_t addr, 279 291 size_t size, enum dma_data_direction dir, 280 292 unsigned long attrs); 281 - int virtqueue_dma_mapping_error(struct virtqueue *_vq, dma_addr_t addr); 293 + int virtqueue_map_mapping_error(const struct virtqueue *_vq, dma_addr_t addr); 282 294 283 - bool virtqueue_dma_need_sync(struct virtqueue *_vq, dma_addr_t addr); 284 - void virtqueue_dma_sync_single_range_for_cpu(struct virtqueue *_vq, dma_addr_t addr, 295 + bool virtqueue_map_need_sync(const struct virtqueue *_vq, dma_addr_t addr); 296 + void virtqueue_map_sync_single_range_for_cpu(const struct virtqueue *_vq, dma_addr_t addr, 285 297 unsigned long offset, size_t size, 286 298 enum dma_data_direction dir); 287 - void virtqueue_dma_sync_single_range_for_device(struct virtqueue *_vq, dma_addr_t addr, 299 + void virtqueue_map_sync_single_range_for_device(const struct virtqueue *_vq, dma_addr_t addr, 288 300 unsigned long offset, size_t size, 289 301 enum dma_data_direction dir); 290 302

+72

include/linux/virtio_config.h

··· 139 139 int (*enable_vq_after_reset)(struct virtqueue *vq); 140 140 }; 141 141 142 + /** 143 + * struct virtio_map_ops - operations for mapping buffer for a virtio device 144 + * Note: For transport that has its own mapping logic it must 145 + * implements all of the operations 146 + * @map_page: map a buffer to the device 147 + * map: metadata for performing mapping 148 + * page: the page that will be mapped by the device 149 + * offset: the offset in the page for a buffer 150 + * size: the buffer size 151 + * dir: mapping direction 152 + * attrs: mapping attributes 153 + * Returns: the mapped address 154 + * @unmap_page: unmap a buffer from the device 155 + * map: device specific mapping map 156 + * map_handle: the mapped address 157 + * size: the buffer size 158 + * dir: mapping direction 159 + * attrs: unmapping attributes 160 + * @sync_single_for_cpu: sync a single buffer from device to cpu 161 + * map: metadata for performing mapping 162 + * map_handle: the mapping address to sync 163 + * size: the size of the buffer 164 + * dir: synchronization direction 165 + * @sync_single_for_device: sync a single buffer from cpu to device 166 + * map: metadata for performing mapping 167 + * map_handle: the mapping address to sync 168 + * size: the size of the buffer 169 + * dir: synchronization direction 170 + * @alloc: alloc a coherent buffer mapping 171 + * map: metadata for performing mapping 172 + * size: the size of the buffer 173 + * map_handle: the mapping address to sync 174 + * gfp: allocation flag (GFP_XXX) 175 + * Returns: virtual address of the allocated buffer 176 + * @free: free a coherent buffer mapping 177 + * map: metadata for performing mapping 178 + * size: the size of the buffer 179 + * vaddr: virtual address of the buffer 180 + * map_handle: the mapping address to sync 181 + * attrs: unmapping attributes 182 + * @need_sync: if the buffer needs synchronization 183 + * map: metadata for performing mapping 184 + * map_handle: the mapped address 185 + * Returns: whether the buffer needs synchronization 186 + * @mapping_error: if the mapping address is error 187 + * map: metadata for performing mapping 188 + * map_handle: the mapped address 189 + * @max_mapping_size: get the maximum buffer size that can be mapped 190 + * map: metadata for performing mapping 191 + * Returns: the maximum buffer size that can be mapped 192 + */ 193 + struct virtio_map_ops { 194 + dma_addr_t (*map_page)(union virtio_map map, struct page *page, 195 + unsigned long offset, size_t size, 196 + enum dma_data_direction dir, unsigned long attrs); 197 + void (*unmap_page)(union virtio_map map, dma_addr_t map_handle, 198 + size_t size, enum dma_data_direction dir, 199 + unsigned long attrs); 200 + void (*sync_single_for_cpu)(union virtio_map map, dma_addr_t map_handle, 201 + size_t size, enum dma_data_direction dir); 202 + void (*sync_single_for_device)(union virtio_map map, 203 + dma_addr_t map_handle, size_t size, 204 + enum dma_data_direction dir); 205 + void *(*alloc)(union virtio_map map, size_t size, 206 + dma_addr_t *map_handle, gfp_t gfp); 207 + void (*free)(union virtio_map map, size_t size, void *vaddr, 208 + dma_addr_t map_handle, unsigned long attrs); 209 + bool (*need_sync)(union virtio_map map, dma_addr_t map_handle); 210 + int (*mapping_error)(union virtio_map map, dma_addr_t map_handle); 211 + size_t (*max_mapping_size)(union virtio_map map); 212 + }; 213 + 142 214 /* If driver didn't advertise the feature, it will never appear. */ 143 215 void virtio_check_driver_offered_feature(const struct virtio_device *vdev, 144 216 unsigned int fbit);

+4 -3

include/linux/virtio_ring.h

··· 3 3 #define _LINUX_VIRTIO_RING_H 4 4 5 5 #include <asm/barrier.h> 6 + #include <linux/virtio.h> 6 7 #include <linux/irqreturn.h> 7 8 #include <uapi/linux/virtio_ring.h> 8 9 ··· 80 79 81 80 /* 82 81 * Creates a virtqueue and allocates the descriptor ring with per 83 - * virtqueue DMA device. 82 + * virtqueue mapping operations. 84 83 */ 85 - struct virtqueue *vring_create_virtqueue_dma(unsigned int index, 84 + struct virtqueue *vring_create_virtqueue_map(unsigned int index, 86 85 unsigned int num, 87 86 unsigned int vring_align, 88 87 struct virtio_device *vdev, ··· 92 91 bool (*notify)(struct virtqueue *vq), 93 92 void (*callback)(struct virtqueue *vq), 94 93 const char *name, 95 - struct device *dma_dev); 94 + union virtio_map map); 96 95 97 96 /* 98 97 * Creates a virtqueue with a standard layout but a caller-allocated

Configure Feed

Configure Feed