virtio: introduce map ops in virtio core

+169 -42

drivers/virtio/virtio_ring.c

··· 297 297 { 298 298 size_t max_segment_size = SIZE_MAX; 299 299 300 - if (vring_use_map_api(vdev)) 301 - max_segment_size = dma_max_mapping_size(vdev->dev.parent); 300 + if (vring_use_map_api(vdev)) { 301 + if (vdev->map) { 302 + max_segment_size = 303 + vdev->map->max_mapping_size(vdev->vmap); 304 + } else 305 + max_segment_size = 306 + dma_max_mapping_size(vdev->dev.parent); 307 + } 302 308 303 309 return max_segment_size; 304 310 } ··· 315 309 union virtio_map map) 316 310 { 317 311 if (vring_use_map_api(vdev)) { 318 - return dma_alloc_coherent(map.dma_dev, size, 319 - map_handle, flag); 312 + return virtqueue_map_alloc_coherent(vdev, map, size, 313 + map_handle, flag); 320 314 } else { 321 315 void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag); 322 316 ··· 349 343 union virtio_map map) 350 344 { 351 345 if (vring_use_map_api(vdev)) 352 - dma_free_coherent(map.dma_dev, size, queue, map_handle); 346 + virtqueue_map_free_coherent(vdev, map, size, 347 + queue, map_handle); 353 348 else 354 349 free_pages_exact(queue, PAGE_ALIGN(size)); 355 350 } ··· 363 356 static struct device *vring_dma_dev(const struct vring_virtqueue *vq) 364 357 { 365 358 return vq->map.dma_dev; 359 + } 360 + 361 + static int vring_mapping_error(const struct vring_virtqueue *vq, 362 + dma_addr_t addr) 363 + { 364 + struct virtio_device *vdev = vq->vq.vdev; 365 + 366 + if (!vq->use_map_api) 367 + return 0; 368 + 369 + if (vdev->map) 370 + return vdev->map->mapping_error(vq->map, addr); 371 + else 372 + return dma_mapping_error(vring_dma_dev(vq), addr); 366 373 } 367 374 368 375 /* Map one sg entry. */ ··· 408 387 * the way it expects (we don't guarantee that the scatterlist 409 388 * will exist for the lifetime of the mapping). 410 389 */ 411 - *addr = dma_map_page(vring_dma_dev(vq), 412 - sg_page(sg), sg->offset, sg->length, 413 - direction); 390 + *addr = virtqueue_map_page_attrs(&vq->vq, sg_page(sg), 391 + sg->offset, sg->length, 392 + direction, 0); 414 393 415 - if (dma_mapping_error(vring_dma_dev(vq), *addr)) 394 + if (vring_mapping_error(vq, *addr)) 416 395 return -ENOMEM; 417 396 418 397 return 0; ··· 427 406 428 407 return virtqueue_map_single_attrs(&vq->vq, cpu_addr, 429 408 size, direction, 0); 430 - } 431 - 432 - static int vring_mapping_error(const struct vring_virtqueue *vq, 433 - dma_addr_t addr) 434 - { 435 - if (!vq->use_map_api) 436 - return 0; 437 - 438 - return dma_mapping_error(vring_dma_dev(vq), addr); 439 409 } 440 410 441 411 static void virtqueue_init(struct vring_virtqueue *vq, u32 num) ··· 465 453 } else if (!vring_need_unmap_buffer(vq, extra)) 466 454 goto out; 467 455 468 - dma_unmap_page(vring_dma_dev(vq), 469 - extra->addr, 470 - extra->len, 471 - (flags & VRING_DESC_F_WRITE) ? 472 - DMA_FROM_DEVICE : DMA_TO_DEVICE); 456 + virtqueue_unmap_page_attrs(&vq->vq, 457 + extra->addr, 458 + extra->len, 459 + (flags & VRING_DESC_F_WRITE) ? 460 + DMA_FROM_DEVICE : DMA_TO_DEVICE, 461 + 0); 473 462 474 463 out: 475 464 return extra->next; ··· 1284 1271 } else if (!vring_need_unmap_buffer(vq, extra)) 1285 1272 return; 1286 1273 1287 - dma_unmap_page(vring_dma_dev(vq), 1288 - extra->addr, extra->len, 1289 - (flags & VRING_DESC_F_WRITE) ? 1290 - DMA_FROM_DEVICE : DMA_TO_DEVICE); 1274 + virtqueue_unmap_page_attrs(&vq->vq, 1275 + extra->addr, extra->len, 1276 + (flags & VRING_DESC_F_WRITE) ? 1277 + DMA_FROM_DEVICE : DMA_TO_DEVICE, 1278 + 0); 1291 1279 } 1292 1280 1293 1281 static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg, ··· 2447 2433 { 2448 2434 struct vring_virtqueue *vq = to_vvq(_vq); 2449 2435 2450 - if (vq->use_map_api) 2436 + if (vq->use_map_api && !_vq->vdev->map) 2451 2437 return vq->map.dma_dev; 2452 2438 else 2453 2439 return NULL; ··· 3138 3124 EXPORT_SYMBOL_GPL(virtqueue_get_vring); 3139 3125 3140 3126 /** 3127 + * virtqueue_map_alloc_coherent - alloc coherent mapping 3128 + * @vdev: the virtio device we are talking to 3129 + * @map: metadata for performing mapping 3130 + * @size: the size of the buffer 3131 + * @map_handle: the pointer to the mapped address 3132 + * @gfp: allocation flag (GFP_XXX) 3133 + * 3134 + * return virtual address or NULL on error 3135 + */ 3136 + void *virtqueue_map_alloc_coherent(struct virtio_device *vdev, 3137 + union virtio_map map, 3138 + size_t size, dma_addr_t *map_handle, 3139 + gfp_t gfp) 3140 + { 3141 + if (vdev->map) 3142 + return vdev->map->alloc(map, size, 3143 + map_handle, gfp); 3144 + else 3145 + return dma_alloc_coherent(map.dma_dev, size, 3146 + map_handle, gfp); 3147 + } 3148 + EXPORT_SYMBOL_GPL(virtqueue_map_alloc_coherent); 3149 + 3150 + /** 3151 + * virtqueue_map_free_coherent - free coherent mapping 3152 + * @vdev: the virtio device we are talking to 3153 + * @map: metadata for performing mapping 3154 + * @size: the size of the buffer 3155 + * @map_handle: the mapped address that needs to be freed 3156 + * 3157 + */ 3158 + void virtqueue_map_free_coherent(struct virtio_device *vdev, 3159 + union virtio_map map, size_t size, void *vaddr, 3160 + dma_addr_t map_handle) 3161 + { 3162 + if (vdev->map) 3163 + vdev->map->free(map, size, vaddr, 3164 + map_handle, 0); 3165 + else 3166 + dma_free_coherent(map.dma_dev, size, vaddr, map_handle); 3167 + } 3168 + EXPORT_SYMBOL_GPL(virtqueue_map_free_coherent); 3169 + 3170 + /** 3171 + * virtqueue_map_page_attrs - map a page to the device 3172 + * @_vq: the virtqueue we are talking to 3173 + * @page: the page that will be mapped by the device 3174 + * @offset: the offset in the page for a buffer 3175 + * @size: the buffer size 3176 + * @dir: mapping direction 3177 + * @attrs: mapping attributes 3178 + * 3179 + * Returns mapped address. Caller should check that by virtqueue_mapping_error(). 3180 + */ 3181 + dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq, 3182 + struct page *page, 3183 + unsigned long offset, 3184 + size_t size, 3185 + enum dma_data_direction dir, 3186 + unsigned long attrs) 3187 + { 3188 + const struct vring_virtqueue *vq = to_vvq(_vq); 3189 + struct virtio_device *vdev = _vq->vdev; 3190 + 3191 + if (vdev->map) 3192 + return vdev->map->map_page(vq->map, 3193 + page, offset, size, 3194 + dir, attrs); 3195 + 3196 + return dma_map_page_attrs(vring_dma_dev(vq), 3197 + page, offset, size, 3198 + dir, attrs); 3199 + } 3200 + EXPORT_SYMBOL_GPL(virtqueue_map_page_attrs); 3201 + 3202 + /** 3203 + * virtqueue_unmap_page_attrs - map a page to the device 3204 + * @_vq: the virtqueue we are talking to 3205 + * @map_handle: the mapped address 3206 + * @size: the buffer size 3207 + * @dir: mapping direction 3208 + * @attrs: unmapping attributes 3209 + */ 3210 + void virtqueue_unmap_page_attrs(const struct virtqueue *_vq, 3211 + dma_addr_t map_handle, 3212 + size_t size, enum dma_data_direction dir, 3213 + unsigned long attrs) 3214 + { 3215 + const struct vring_virtqueue *vq = to_vvq(_vq); 3216 + struct virtio_device *vdev = _vq->vdev; 3217 + 3218 + if (vdev->map) 3219 + vdev->map->unmap_page(vq->map, 3220 + map_handle, size, dir, attrs); 3221 + else 3222 + dma_unmap_page_attrs(vring_dma_dev(vq), map_handle, 3223 + size, dir, attrs); 3224 + } 3225 + EXPORT_SYMBOL_GPL(virtqueue_unmap_page_attrs); 3226 + 3227 + /** 3141 3228 * virtqueue_map_single_attrs - map DMA for _vq 3142 3229 * @_vq: the struct virtqueue we're talking about. 3143 3230 * @ptr: the pointer of the buffer to do dma ··· 3249 3134 * The caller calls this to do dma mapping in advance. The DMA address can be 3250 3135 * passed to this _vq when it is in pre-mapped mode. 3251 3136 * 3252 - * return DMA address. Caller should check that by virtqueue_mapping_error(). 3137 + * return mapped address. Caller should check that by virtqueue_mapping_error(). 3253 3138 */ 3254 3139 dma_addr_t virtqueue_map_single_attrs(const struct virtqueue *_vq, void *ptr, 3255 3140 size_t size, ··· 3268 3153 "rejecting DMA map of vmalloc memory\n")) 3269 3154 return DMA_MAPPING_ERROR; 3270 3155 3271 - return dma_map_page_attrs(vring_dma_dev(vq), virt_to_page(ptr), 3272 - offset_in_page(ptr), size, dir, attrs); 3156 + return virtqueue_map_page_attrs(&vq->vq, virt_to_page(ptr), 3157 + offset_in_page(ptr), size, dir, attrs); 3273 3158 } 3274 3159 EXPORT_SYMBOL_GPL(virtqueue_map_single_attrs); 3275 3160 ··· 3294 3179 if (!vq->use_map_api) 3295 3180 return; 3296 3181 3297 - dma_unmap_page_attrs(vring_dma_dev(vq), addr, size, dir, attrs); 3182 + virtqueue_unmap_page_attrs(_vq, addr, size, dir, attrs); 3298 3183 } 3299 3184 EXPORT_SYMBOL_GPL(virtqueue_unmap_single_attrs); 3300 3185 3301 3186 /** 3302 - * virtqueue_map_mapping_error - check dma address 3187 + * virtqueue_mapping_error - check dma address 3303 3188 * @_vq: the struct virtqueue we're talking about. 3304 3189 * @addr: DMA address 3305 3190 * ··· 3309 3194 { 3310 3195 const struct vring_virtqueue *vq = to_vvq(_vq); 3311 3196 3312 - if (!vq->use_map_api) 3313 - return 0; 3314 - 3315 - return dma_mapping_error(vring_dma_dev(vq), addr); 3197 + return vring_mapping_error(vq, addr); 3316 3198 } 3317 3199 EXPORT_SYMBOL_GPL(virtqueue_map_mapping_error); 3318 3200 ··· 3326 3214 bool virtqueue_map_need_sync(const struct virtqueue *_vq, dma_addr_t addr) 3327 3215 { 3328 3216 const struct vring_virtqueue *vq = to_vvq(_vq); 3217 + struct virtio_device *vdev = _vq->vdev; 3329 3218 3330 3219 if (!vq->use_map_api) 3331 3220 return false; 3332 3221 3333 - return dma_need_sync(vring_dma_dev(vq), addr); 3222 + if (vdev->map) 3223 + return vdev->map->need_sync(vq->map, addr); 3224 + else 3225 + return dma_need_sync(vring_dma_dev(vq), addr); 3334 3226 } 3335 3227 EXPORT_SYMBOL_GPL(virtqueue_map_need_sync); 3336 3228 ··· 3356 3240 enum dma_data_direction dir) 3357 3241 { 3358 3242 const struct vring_virtqueue *vq = to_vvq(_vq); 3359 - struct device *dev = vring_dma_dev(vq); 3243 + struct virtio_device *vdev = _vq->vdev; 3360 3244 3361 3245 if (!vq->use_map_api) 3362 3246 return; 3363 3247 3364 - dma_sync_single_range_for_cpu(dev, addr, offset, size, dir); 3248 + if (vdev->map) 3249 + vdev->map->sync_single_for_cpu(vq->map, 3250 + addr + offset, size, dir); 3251 + else 3252 + dma_sync_single_range_for_cpu(vring_dma_dev(vq), 3253 + addr, offset, size, dir); 3365 3254 } 3366 3255 EXPORT_SYMBOL_GPL(virtqueue_map_sync_single_range_for_cpu); 3367 3256 ··· 3387 3266 enum dma_data_direction dir) 3388 3267 { 3389 3268 const struct vring_virtqueue *vq = to_vvq(_vq); 3390 - struct device *dev = vring_dma_dev(vq); 3269 + struct virtio_device *vdev = _vq->vdev; 3391 3270 3392 3271 if (!vq->use_map_api) 3393 3272 return; 3394 3273 3395 - dma_sync_single_range_for_device(dev, addr, offset, size, dir); 3274 + if (vdev->map) 3275 + vdev->map->sync_single_for_device(vq->map, 3276 + addr + offset, 3277 + size, dir); 3278 + else 3279 + dma_sync_single_range_for_device(vring_dma_dev(vq), addr, 3280 + offset, size, dir); 3396 3281 } 3397 3282 EXPORT_SYMBOL_GPL(virtqueue_map_sync_single_range_for_device); 3398 3283

+3

drivers/virtio/virtio_vdpa.c

··· 195 195 goto error_new_virtqueue; 196 196 } 197 197 198 + if (index == 0) 199 + vdev->vmap = map; 200 + 198 201 vq->num_max = max_num; 199 202 200 203 /* Setup virtqueue callback */

+25

include/linux/virtio.h

··· 166 166 struct virtio_device_id id; 167 167 const struct virtio_config_ops *config; 168 168 const struct vringh_config_ops *vringh_config; 169 + const struct virtio_map_ops *map; 169 170 struct list_head vqs; 170 171 VIRTIO_DECLARE_FEATURES(features); 171 172 void *priv; 173 + union virtio_map vmap; 172 174 #ifdef CONFIG_VIRTIO_DEBUG 173 175 struct dentry *debugfs_dir; 174 176 u64 debugfs_filter_features[VIRTIO_FEATURES_DWORDS]; ··· 268 266 #define module_virtio_driver(__virtio_driver) \ 269 267 module_driver(__virtio_driver, register_virtio_driver, \ 270 268 unregister_virtio_driver) 269 + 270 + 271 + void *virtqueue_map_alloc_coherent(struct virtio_device *vdev, 272 + union virtio_map mapping_token, 273 + size_t size, dma_addr_t *dma_handle, 274 + gfp_t gfp); 275 + 276 + void virtqueue_map_free_coherent(struct virtio_device *vdev, 277 + union virtio_map mapping_token, 278 + size_t size, void *vaddr, 279 + dma_addr_t dma_handle); 280 + 281 + dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq, 282 + struct page *page, 283 + unsigned long offset, 284 + size_t size, 285 + enum dma_data_direction dir, 286 + unsigned long attrs); 287 + 288 + void virtqueue_unmap_page_attrs(const struct virtqueue *_vq, 289 + dma_addr_t dma_handle, 290 + size_t size, enum dma_data_direction dir, 291 + unsigned long attrs); 271 292 272 293 dma_addr_t virtqueue_map_single_attrs(const struct virtqueue *_vq, void *ptr, size_t size, 273 294 enum dma_data_direction dir, unsigned long attrs);

+72

include/linux/virtio_config.h

··· 139 139 int (*enable_vq_after_reset)(struct virtqueue *vq); 140 140 }; 141 141 142 + /** 143 + * struct virtio_map_ops - operations for mapping buffer for a virtio device 144 + * Note: For transport that has its own mapping logic it must 145 + * implements all of the operations 146 + * @map_page: map a buffer to the device 147 + * map: metadata for performing mapping 148 + * page: the page that will be mapped by the device 149 + * offset: the offset in the page for a buffer 150 + * size: the buffer size 151 + * dir: mapping direction 152 + * attrs: mapping attributes 153 + * Returns: the mapped address 154 + * @unmap_page: unmap a buffer from the device 155 + * map: device specific mapping map 156 + * map_handle: the mapped address 157 + * size: the buffer size 158 + * dir: mapping direction 159 + * attrs: unmapping attributes 160 + * @sync_single_for_cpu: sync a single buffer from device to cpu 161 + * map: metadata for performing mapping 162 + * map_handle: the mapping address to sync 163 + * size: the size of the buffer 164 + * dir: synchronization direction 165 + * @sync_single_for_device: sync a single buffer from cpu to device 166 + * map: metadata for performing mapping 167 + * map_handle: the mapping address to sync 168 + * size: the size of the buffer 169 + * dir: synchronization direction 170 + * @alloc: alloc a coherent buffer mapping 171 + * map: metadata for performing mapping 172 + * size: the size of the buffer 173 + * map_handle: the mapping address to sync 174 + * gfp: allocation flag (GFP_XXX) 175 + * Returns: virtual address of the allocated buffer 176 + * @free: free a coherent buffer mapping 177 + * map: metadata for performing mapping 178 + * size: the size of the buffer 179 + * vaddr: virtual address of the buffer 180 + * map_handle: the mapping address to sync 181 + * attrs: unmapping attributes 182 + * @need_sync: if the buffer needs synchronization 183 + * map: metadata for performing mapping 184 + * map_handle: the mapped address 185 + * Returns: whether the buffer needs synchronization 186 + * @mapping_error: if the mapping address is error 187 + * map: metadata for performing mapping 188 + * map_handle: the mapped address 189 + * @max_mapping_size: get the maximum buffer size that can be mapped 190 + * map: metadata for performing mapping 191 + * Returns: the maximum buffer size that can be mapped 192 + */ 193 + struct virtio_map_ops { 194 + dma_addr_t (*map_page)(union virtio_map map, struct page *page, 195 + unsigned long offset, size_t size, 196 + enum dma_data_direction dir, unsigned long attrs); 197 + void (*unmap_page)(union virtio_map map, dma_addr_t map_handle, 198 + size_t size, enum dma_data_direction dir, 199 + unsigned long attrs); 200 + void (*sync_single_for_cpu)(union virtio_map map, dma_addr_t map_handle, 201 + size_t size, enum dma_data_direction dir); 202 + void (*sync_single_for_device)(union virtio_map map, 203 + dma_addr_t map_handle, size_t size, 204 + enum dma_data_direction dir); 205 + void *(*alloc)(union virtio_map map, size_t size, 206 + dma_addr_t *map_handle, gfp_t gfp); 207 + void (*free)(union virtio_map map, size_t size, void *vaddr, 208 + dma_addr_t map_handle, unsigned long attrs); 209 + bool (*need_sync)(union virtio_map map, dma_addr_t map_handle); 210 + int (*mapping_error)(union virtio_map map, dma_addr_t map_handle); 211 + size_t (*max_mapping_size)(union virtio_map map); 212 + }; 213 + 142 214 /* If driver didn't advertise the feature, it will never appear. */ 143 215 void virtio_check_driver_offered_feature(const struct virtio_device *vdev, 144 216 unsigned int fbit);

Configure Feed

Configure Feed