Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost

+85

Documentation/networking/device_drivers/ethernet/amd/pds_vdpa.rst

··· 1 + .. SPDX-License-Identifier: GPL-2.0+ 2 + .. note: can be edited and viewed with /usr/bin/formiko-vim 3 + 4 + ========================================================== 5 + PCI vDPA driver for the AMD/Pensando(R) DSC adapter family 6 + ========================================================== 7 + 8 + AMD/Pensando vDPA VF Device Driver 9 + 10 + Copyright(c) 2023 Advanced Micro Devices, Inc 11 + 12 + Overview 13 + ======== 14 + 15 + The ``pds_vdpa`` driver is an auxiliary bus driver that supplies 16 + a vDPA device for use by the virtio network stack. It is used with 17 + the Pensando Virtual Function devices that offer vDPA and virtio queue 18 + services. It depends on the ``pds_core`` driver and hardware for the PF 19 + and VF PCI handling as well as for device configuration services. 20 + 21 + Using the device 22 + ================ 23 + 24 + The ``pds_vdpa`` device is enabled via multiple configuration steps and 25 + depends on the ``pds_core`` driver to create and enable SR-IOV Virtual 26 + Function devices. After the VFs are enabled, we enable the vDPA service 27 + in the ``pds_core`` device to create the auxiliary devices used by pds_vdpa. 28 + 29 + Example steps: 30 + 31 + .. code-block:: bash 32 + 33 + #!/bin/bash 34 + 35 + modprobe pds_core 36 + modprobe vdpa 37 + modprobe pds_vdpa 38 + 39 + PF_BDF=`ls /sys/module/pds_core/drivers/pci\:pds_core/*/sriov_numvfs | awk -F / '{print $7}'` 40 + 41 + # Enable vDPA VF auxiliary device(s) in the PF 42 + devlink dev param set pci/$PF_BDF name enable_vnet cmode runtime value true 43 + 44 + # Create a VF for vDPA use 45 + echo 1 > /sys/bus/pci/drivers/pds_core/$PF_BDF/sriov_numvfs 46 + 47 + # Find the vDPA services/devices available 48 + PDS_VDPA_MGMT=`vdpa mgmtdev show | grep vDPA | head -1 | cut -d: -f1` 49 + 50 + # Create a vDPA device for use in virtio network configurations 51 + vdpa dev add name vdpa1 mgmtdev $PDS_VDPA_MGMT mac 00:11:22:33:44:55 52 + 53 + # Set up an ethernet interface on the vdpa device 54 + modprobe virtio_vdpa 55 + 56 + 57 + 58 + Enabling the driver 59 + =================== 60 + 61 + The driver is enabled via the standard kernel configuration system, 62 + using the make command:: 63 + 64 + make oldconfig/menuconfig/etc. 65 + 66 + The driver is located in the menu structure at: 67 + 68 + -> Device Drivers 69 + -> Network device support (NETDEVICES [=y]) 70 + -> Ethernet driver support 71 + -> Pensando devices 72 + -> Pensando Ethernet PDS_VDPA Support 73 + 74 + Support 75 + ======= 76 + 77 + For general Linux networking support, please use the netdev mailing 78 + list, which is monitored by Pensando personnel:: 79 + 80 + netdev@vger.kernel.org 81 + 82 + For more specific support needs, please use the Pensando driver support 83 + email:: 84 + 85 + drivers@pensando.io

+1

Documentation/networking/device_drivers/ethernet/index.rst

··· 15 15 amazon/ena 16 16 altera/altera_tse 17 17 amd/pds_core 18 + amd/pds_vdpa 18 19 aquantia/atlantic 19 20 chelsio/cxgb 20 21 cirrus/cs89x0

+4

MAINTAINERS

··· 22535 22535 F: include/uapi/linux/virtio_*.h 22536 22536 F: tools/virtio/ 22537 22537 22538 + PDS DSC VIRTIO DATA PATH ACCELERATOR 22539 + R: Shannon Nelson <shannon.nelson@amd.com> 22540 + F: drivers/vdpa/pds/ 22541 + 22538 22542 VIRTIO CRYPTO DRIVER 22539 22543 M: Gonglei <arei.gonglei@huawei.com> 22540 22544 L: virtualization@lists.linux-foundation.org

+1

drivers/bluetooth/virtio_bt.c

··· 79 79 80 80 while ((skb = virtqueue_detach_unused_buf(vq))) 81 81 kfree_skb(skb); 82 + cond_resched(); 82 83 } 83 84 84 85 return 0;

+1

drivers/char/virtio_console.c

··· 1936 1936 flush_bufs(vq, true); 1937 1937 while ((buf = virtqueue_detach_unused_buf(vq))) 1938 1938 free_buf(buf, true); 1939 + cond_resched(); 1939 1940 } 1940 1941 portdev->vdev->config->del_vqs(portdev->vdev); 1941 1942 kfree(portdev->in_vqs);

+1

drivers/crypto/virtio/virtio_crypto_core.c

··· 480 480 kfree(vc_req->req_data); 481 481 kfree(vc_req->sgs); 482 482 } 483 + cond_resched(); 483 484 } 484 485 } 485 486

+10

drivers/vdpa/Kconfig

··· 116 116 This driver includes a HW monitor device that 117 117 reads health values from the DPU. 118 118 119 + config PDS_VDPA 120 + tristate "vDPA driver for AMD/Pensando DSC devices" 121 + select VIRTIO_PCI_LIB 122 + depends on PCI_MSI 123 + depends on PDS_CORE 124 + help 125 + vDPA network driver for AMD/Pensando's PDS Core devices. 126 + With this driver, the VirtIO dataplane can be 127 + offloaded to an AMD/Pensando DSC device. 128 + 119 129 endif # VDPA

+1

drivers/vdpa/Makefile

··· 7 7 obj-$(CONFIG_VP_VDPA) += virtio_pci/ 8 8 obj-$(CONFIG_ALIBABA_ENI_VDPA) += alibaba/ 9 9 obj-$(CONFIG_SNET_VDPA) += solidrun/ 10 + obj-$(CONFIG_PDS_VDPA) += pds/

+127 -88

drivers/vdpa/ifcvf/ifcvf_base.c

··· 69 69 return 0; 70 70 } 71 71 72 + static u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid) 73 + { 74 + u16 queue_size; 75 + 76 + vp_iowrite16(qid, &hw->common_cfg->queue_select); 77 + queue_size = vp_ioread16(&hw->common_cfg->queue_size); 78 + 79 + return queue_size; 80 + } 81 + 82 + /* This function returns the max allowed safe size for 83 + * all virtqueues. It is the minimal size that can be 84 + * suppprted by all virtqueues. 85 + */ 86 + u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw) 87 + { 88 + u16 queue_size, max_size, qid; 89 + 90 + max_size = ifcvf_get_vq_size(hw, 0); 91 + for (qid = 1; qid < hw->nr_vring; qid++) { 92 + queue_size = ifcvf_get_vq_size(hw, qid); 93 + /* 0 means the queue is unavailable */ 94 + if (!queue_size) 95 + continue; 96 + 97 + max_size = min(queue_size, max_size); 98 + } 99 + 100 + return max_size; 101 + } 102 + 72 103 int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev) 73 104 { 74 105 struct virtio_pci_cap cap; ··· 165 134 } 166 135 167 136 hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues); 137 + hw->vring = kzalloc(sizeof(struct vring_info) * hw->nr_vring, GFP_KERNEL); 138 + if (!hw->vring) 139 + return -ENOMEM; 168 140 169 141 for (i = 0; i < hw->nr_vring; i++) { 170 142 vp_iowrite16(i, &hw->common_cfg->queue_select); ··· 204 170 205 171 void ifcvf_reset(struct ifcvf_hw *hw) 206 172 { 207 - hw->config_cb.callback = NULL; 208 - hw->config_cb.private = NULL; 209 - 210 173 ifcvf_set_status(hw, 0); 211 - /* flush set_status, make sure VF is stopped, reset */ 212 - ifcvf_get_status(hw); 213 - } 214 - 215 - static void ifcvf_add_status(struct ifcvf_hw *hw, u8 status) 216 - { 217 - if (status != 0) 218 - status |= ifcvf_get_status(hw); 219 - 220 - ifcvf_set_status(hw, status); 221 - ifcvf_get_status(hw); 174 + while (ifcvf_get_status(hw)) 175 + msleep(1); 222 176 } 223 177 224 178 u64 ifcvf_get_hw_features(struct ifcvf_hw *hw) ··· 226 204 return features; 227 205 } 228 206 229 - u64 ifcvf_get_features(struct ifcvf_hw *hw) 207 + /* return provisioned vDPA dev features */ 208 + u64 ifcvf_get_dev_features(struct ifcvf_hw *hw) 230 209 { 231 210 return hw->dev_features; 211 + } 212 + 213 + u64 ifcvf_get_driver_features(struct ifcvf_hw *hw) 214 + { 215 + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; 216 + u32 features_lo, features_hi; 217 + u64 features; 218 + 219 + vp_iowrite32(0, &cfg->device_feature_select); 220 + features_lo = vp_ioread32(&cfg->guest_feature); 221 + 222 + vp_iowrite32(1, &cfg->device_feature_select); 223 + features_hi = vp_ioread32(&cfg->guest_feature); 224 + 225 + features = ((u64)features_hi << 32) | features_lo; 226 + 227 + return features; 232 228 } 233 229 234 230 int ifcvf_verify_min_features(struct ifcvf_hw *hw, u64 features) ··· 315 275 vp_iowrite8(*p++, hw->dev_cfg + offset + i); 316 276 } 317 277 318 - static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features) 278 + void ifcvf_set_driver_features(struct ifcvf_hw *hw, u64 features) 319 279 { 320 280 struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; 321 281 ··· 326 286 vp_iowrite32(features >> 32, &cfg->guest_feature); 327 287 } 328 288 329 - static int ifcvf_config_features(struct ifcvf_hw *hw) 330 - { 331 - ifcvf_set_features(hw, hw->req_features); 332 - ifcvf_add_status(hw, VIRTIO_CONFIG_S_FEATURES_OK); 333 - 334 - if (!(ifcvf_get_status(hw) & VIRTIO_CONFIG_S_FEATURES_OK)) { 335 - IFCVF_ERR(hw->pdev, "Failed to set FEATURES_OK status\n"); 336 - return -EIO; 337 - } 338 - 339 - return 0; 340 - } 341 - 342 289 u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid) 343 290 { 344 - struct ifcvf_lm_cfg __iomem *ifcvf_lm; 345 - void __iomem *avail_idx_addr; 291 + struct ifcvf_lm_cfg __iomem *lm_cfg = hw->lm_cfg; 346 292 u16 last_avail_idx; 347 - u32 q_pair_id; 348 293 349 - ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg; 350 - q_pair_id = qid / 2; 351 - avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2]; 352 - last_avail_idx = vp_ioread16(avail_idx_addr); 294 + last_avail_idx = vp_ioread16(&lm_cfg->vq_state_region + qid * 2); 353 295 354 296 return last_avail_idx; 355 297 } 356 298 357 299 int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num) 358 300 { 359 - struct ifcvf_lm_cfg __iomem *ifcvf_lm; 360 - void __iomem *avail_idx_addr; 361 - u32 q_pair_id; 301 + struct ifcvf_lm_cfg __iomem *lm_cfg = hw->lm_cfg; 362 302 363 - ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg; 364 - q_pair_id = qid / 2; 365 - avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2]; 366 - hw->vring[qid].last_avail_idx = num; 367 - vp_iowrite16(num, avail_idx_addr); 303 + vp_iowrite16(num, &lm_cfg->vq_state_region + qid * 2); 368 304 369 305 return 0; 370 306 } 371 307 372 - static int ifcvf_hw_enable(struct ifcvf_hw *hw) 308 + void ifcvf_set_vq_num(struct ifcvf_hw *hw, u16 qid, u32 num) 373 309 { 374 - struct virtio_pci_common_cfg __iomem *cfg; 375 - u32 i; 310 + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; 376 311 377 - cfg = hw->common_cfg; 378 - for (i = 0; i < hw->nr_vring; i++) { 379 - if (!hw->vring[i].ready) 380 - break; 312 + vp_iowrite16(qid, &cfg->queue_select); 313 + vp_iowrite16(num, &cfg->queue_size); 314 + } 381 315 382 - vp_iowrite16(i, &cfg->queue_select); 383 - vp_iowrite64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo, 384 - &cfg->queue_desc_hi); 385 - vp_iowrite64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo, 386 - &cfg->queue_avail_hi); 387 - vp_iowrite64_twopart(hw->vring[i].used, &cfg->queue_used_lo, 388 - &cfg->queue_used_hi); 389 - vp_iowrite16(hw->vring[i].size, &cfg->queue_size); 390 - ifcvf_set_vq_state(hw, i, hw->vring[i].last_avail_idx); 391 - vp_iowrite16(1, &cfg->queue_enable); 316 + int ifcvf_set_vq_address(struct ifcvf_hw *hw, u16 qid, u64 desc_area, 317 + u64 driver_area, u64 device_area) 318 + { 319 + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; 320 + 321 + vp_iowrite16(qid, &cfg->queue_select); 322 + vp_iowrite64_twopart(desc_area, &cfg->queue_desc_lo, 323 + &cfg->queue_desc_hi); 324 + vp_iowrite64_twopart(driver_area, &cfg->queue_avail_lo, 325 + &cfg->queue_avail_hi); 326 + vp_iowrite64_twopart(device_area, &cfg->queue_used_lo, 327 + &cfg->queue_used_hi); 328 + 329 + return 0; 330 + } 331 + 332 + bool ifcvf_get_vq_ready(struct ifcvf_hw *hw, u16 qid) 333 + { 334 + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; 335 + u16 queue_enable; 336 + 337 + vp_iowrite16(qid, &cfg->queue_select); 338 + queue_enable = vp_ioread16(&cfg->queue_enable); 339 + 340 + return (bool)queue_enable; 341 + } 342 + 343 + void ifcvf_set_vq_ready(struct ifcvf_hw *hw, u16 qid, bool ready) 344 + { 345 + struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; 346 + 347 + vp_iowrite16(qid, &cfg->queue_select); 348 + vp_iowrite16(ready, &cfg->queue_enable); 349 + } 350 + 351 + static void ifcvf_reset_vring(struct ifcvf_hw *hw) 352 + { 353 + u16 qid; 354 + 355 + for (qid = 0; qid < hw->nr_vring; qid++) { 356 + hw->vring[qid].cb.callback = NULL; 357 + hw->vring[qid].cb.private = NULL; 358 + ifcvf_set_vq_vector(hw, qid, VIRTIO_MSI_NO_VECTOR); 392 359 } 393 - 394 - return 0; 395 360 } 396 361 397 - static void ifcvf_hw_disable(struct ifcvf_hw *hw) 362 + static void ifcvf_reset_config_handler(struct ifcvf_hw *hw) 398 363 { 399 - u32 i; 400 - 364 + hw->config_cb.callback = NULL; 365 + hw->config_cb.private = NULL; 401 366 ifcvf_set_config_vector(hw, VIRTIO_MSI_NO_VECTOR); 402 - for (i = 0; i < hw->nr_vring; i++) { 403 - ifcvf_set_vq_vector(hw, i, VIRTIO_MSI_NO_VECTOR); 367 + } 368 + 369 + static void ifcvf_synchronize_irq(struct ifcvf_hw *hw) 370 + { 371 + u32 nvectors = hw->num_msix_vectors; 372 + struct pci_dev *pdev = hw->pdev; 373 + int i, irq; 374 + 375 + for (i = 0; i < nvectors; i++) { 376 + irq = pci_irq_vector(pdev, i); 377 + if (irq >= 0) 378 + synchronize_irq(irq); 404 379 } 405 380 } 406 381 407 - int ifcvf_start_hw(struct ifcvf_hw *hw) 382 + void ifcvf_stop(struct ifcvf_hw *hw) 408 383 { 409 - ifcvf_reset(hw); 410 - ifcvf_add_status(hw, VIRTIO_CONFIG_S_ACKNOWLEDGE); 411 - ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER); 412 - 413 - if (ifcvf_config_features(hw) < 0) 414 - return -EINVAL; 415 - 416 - if (ifcvf_hw_enable(hw) < 0) 417 - return -EINVAL; 418 - 419 - ifcvf_add_status(hw, VIRTIO_CONFIG_S_DRIVER_OK); 420 - 421 - return 0; 422 - } 423 - 424 - void ifcvf_stop_hw(struct ifcvf_hw *hw) 425 - { 426 - ifcvf_hw_disable(hw); 427 - ifcvf_reset(hw); 384 + ifcvf_synchronize_irq(hw); 385 + ifcvf_reset_vring(hw); 386 + ifcvf_reset_config_handler(hw); 428 387 } 429 388 430 389 void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)

+21 -27

drivers/vdpa/ifcvf/ifcvf_base.h

··· 24 24 #define N3000_DEVICE_ID 0x1041 25 25 #define N3000_SUBSYS_DEVICE_ID 0x001A 26 26 27 - /* Max 8 data queue pairs(16 queues) and one control vq for now. */ 28 - #define IFCVF_MAX_QUEUES 17 29 - 30 27 #define IFCVF_QUEUE_ALIGNMENT PAGE_SIZE 31 - #define IFCVF_QUEUE_MAX 32768 32 28 #define IFCVF_PCI_MAX_RESOURCE 6 33 29 34 - #define IFCVF_LM_CFG_SIZE 0x40 35 - #define IFCVF_LM_RING_STATE_OFFSET 0x20 36 30 #define IFCVF_LM_BAR 4 37 31 38 32 #define IFCVF_ERR(pdev, fmt, ...) dev_err(&pdev->dev, fmt, ##__VA_ARGS__) ··· 41 47 #define MSIX_VECTOR_DEV_SHARED 3 42 48 43 49 struct vring_info { 44 - u64 desc; 45 - u64 avail; 46 - u64 used; 47 - u16 size; 48 50 u16 last_avail_idx; 49 - bool ready; 50 51 void __iomem *notify_addr; 51 52 phys_addr_t notify_pa; 52 53 u32 irq; ··· 49 60 char msix_name[256]; 50 61 }; 51 62 63 + struct ifcvf_lm_cfg { 64 + __le64 control; 65 + __le64 status; 66 + __le64 lm_mem_log_start_addr; 67 + __le64 lm_mem_log_end_addr; 68 + __le16 vq_state_region; 69 + }; 70 + 52 71 struct ifcvf_hw { 53 72 u8 __iomem *isr; 54 73 /* Live migration */ 55 - u8 __iomem *lm_cfg; 74 + struct ifcvf_lm_cfg __iomem *lm_cfg; 56 75 /* Notification bar number */ 57 76 u8 notify_bar; 58 77 u8 msix_vector_status; ··· 71 74 phys_addr_t notify_base_pa; 72 75 u32 notify_off_multiplier; 73 76 u32 dev_type; 74 - u64 req_features; 75 77 u64 hw_features; 76 78 /* provisioned device features */ 77 79 u64 dev_features; 78 80 struct virtio_pci_common_cfg __iomem *common_cfg; 79 81 void __iomem *dev_cfg; 80 - struct vring_info vring[IFCVF_MAX_QUEUES]; 82 + struct vring_info *vring; 81 83 void __iomem * const *base; 82 84 char config_msix_name[256]; 83 85 struct vdpa_callback config_cb; ··· 84 88 int vqs_reused_irq; 85 89 u16 nr_vring; 86 90 /* VIRTIO_PCI_CAP_DEVICE_CFG size */ 91 + u32 num_msix_vectors; 87 92 u32 cap_dev_config_size; 88 93 struct pci_dev *pdev; 89 94 }; ··· 95 98 struct ifcvf_hw *vf; 96 99 }; 97 100 98 - struct ifcvf_vring_lm_cfg { 99 - u32 idx_addr[2]; 100 - u8 reserved[IFCVF_LM_CFG_SIZE - 8]; 101 - }; 102 - 103 - struct ifcvf_lm_cfg { 104 - u8 reserved[IFCVF_LM_RING_STATE_OFFSET]; 105 - struct ifcvf_vring_lm_cfg vring_lm_cfg[IFCVF_MAX_QUEUES]; 106 - }; 107 - 108 101 struct ifcvf_vdpa_mgmt_dev { 109 102 struct vdpa_mgmt_dev mdev; 110 103 struct ifcvf_hw vf; ··· 103 116 }; 104 117 105 118 int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *dev); 106 - int ifcvf_start_hw(struct ifcvf_hw *hw); 107 - void ifcvf_stop_hw(struct ifcvf_hw *hw); 119 + void ifcvf_stop(struct ifcvf_hw *hw); 108 120 void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid); 109 121 void ifcvf_read_dev_config(struct ifcvf_hw *hw, u64 offset, 110 122 void *dst, int length); ··· 113 127 void ifcvf_set_status(struct ifcvf_hw *hw, u8 status); 114 128 void io_write64_twopart(u64 val, u32 *lo, u32 *hi); 115 129 void ifcvf_reset(struct ifcvf_hw *hw); 116 - u64 ifcvf_get_features(struct ifcvf_hw *hw); 130 + u64 ifcvf_get_dev_features(struct ifcvf_hw *hw); 117 131 u64 ifcvf_get_hw_features(struct ifcvf_hw *hw); 118 132 int ifcvf_verify_min_features(struct ifcvf_hw *hw, u64 features); 119 133 u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid); ··· 123 137 u32 ifcvf_get_config_size(struct ifcvf_hw *hw); 124 138 u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector); 125 139 u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector); 140 + void ifcvf_set_vq_num(struct ifcvf_hw *hw, u16 qid, u32 num); 141 + int ifcvf_set_vq_address(struct ifcvf_hw *hw, u16 qid, u64 desc_area, 142 + u64 driver_area, u64 device_area); 143 + bool ifcvf_get_vq_ready(struct ifcvf_hw *hw, u16 qid); 144 + void ifcvf_set_vq_ready(struct ifcvf_hw *hw, u16 qid, bool ready); 145 + void ifcvf_set_driver_features(struct ifcvf_hw *hw, u64 features); 146 + u64 ifcvf_get_driver_features(struct ifcvf_hw *hw); 147 + u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw); 126 148 #endif /* _IFCVF_H_ */

+26 -82

drivers/vdpa/ifcvf/ifcvf_main.c

··· 125 125 ifcvf_free_vq_irq(vf); 126 126 ifcvf_free_config_irq(vf); 127 127 ifcvf_free_irq_vectors(pdev); 128 + vf->num_msix_vectors = 0; 128 129 } 129 130 130 131 /* ifcvf MSIX vectors allocator, this helper tries to allocate ··· 344 343 if (ret) 345 344 return ret; 346 345 347 - return 0; 348 - } 349 - 350 - static int ifcvf_start_datapath(struct ifcvf_adapter *adapter) 351 - { 352 - struct ifcvf_hw *vf = adapter->vf; 353 - u8 status; 354 - int ret; 355 - 356 - ret = ifcvf_start_hw(vf); 357 - if (ret < 0) { 358 - status = ifcvf_get_status(vf); 359 - status |= VIRTIO_CONFIG_S_FAILED; 360 - ifcvf_set_status(vf, status); 361 - } 362 - 363 - return ret; 364 - } 365 - 366 - static int ifcvf_stop_datapath(struct ifcvf_adapter *adapter) 367 - { 368 - struct ifcvf_hw *vf = adapter->vf; 369 - int i; 370 - 371 - for (i = 0; i < vf->nr_vring; i++) 372 - vf->vring[i].cb.callback = NULL; 373 - 374 - ifcvf_stop_hw(vf); 346 + vf->num_msix_vectors = nvectors; 375 347 376 348 return 0; 377 - } 378 - 379 - static void ifcvf_reset_vring(struct ifcvf_adapter *adapter) 380 - { 381 - struct ifcvf_hw *vf = adapter->vf; 382 - int i; 383 - 384 - for (i = 0; i < vf->nr_vring; i++) { 385 - vf->vring[i].last_avail_idx = 0; 386 - vf->vring[i].desc = 0; 387 - vf->vring[i].avail = 0; 388 - vf->vring[i].used = 0; 389 - vf->vring[i].ready = 0; 390 - vf->vring[i].cb.callback = NULL; 391 - vf->vring[i].cb.private = NULL; 392 - } 393 - 394 - ifcvf_reset(vf); 395 349 } 396 350 397 351 static struct ifcvf_adapter *vdpa_to_adapter(struct vdpa_device *vdpa_dev) ··· 370 414 u64 features; 371 415 372 416 if (type == VIRTIO_ID_NET || type == VIRTIO_ID_BLOCK) 373 - features = ifcvf_get_features(vf); 417 + features = ifcvf_get_dev_features(vf); 374 418 else { 375 419 features = 0; 376 420 IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", vf->dev_type); ··· 388 432 if (ret) 389 433 return ret; 390 434 391 - vf->req_features = features; 435 + ifcvf_set_driver_features(vf, features); 392 436 393 437 return 0; 394 438 } ··· 396 440 static u64 ifcvf_vdpa_get_driver_features(struct vdpa_device *vdpa_dev) 397 441 { 398 442 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 443 + u64 features; 399 444 400 - return vf->req_features; 445 + features = ifcvf_get_driver_features(vf); 446 + 447 + return features; 401 448 } 402 449 403 450 static u8 ifcvf_vdpa_get_status(struct vdpa_device *vdpa_dev) ··· 412 453 413 454 static void ifcvf_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) 414 455 { 415 - struct ifcvf_adapter *adapter; 416 456 struct ifcvf_hw *vf; 417 457 u8 status_old; 418 458 int ret; 419 459 420 460 vf = vdpa_to_vf(vdpa_dev); 421 - adapter = vdpa_to_adapter(vdpa_dev); 422 461 status_old = ifcvf_get_status(vf); 423 462 424 463 if (status_old == status) ··· 426 469 !(status_old & VIRTIO_CONFIG_S_DRIVER_OK)) { 427 470 ret = ifcvf_request_irq(vf); 428 471 if (ret) { 429 - status = ifcvf_get_status(vf); 430 - status |= VIRTIO_CONFIG_S_FAILED; 431 - ifcvf_set_status(vf, status); 472 + IFCVF_ERR(vf->pdev, "failed to request irq with error %d\n", ret); 432 473 return; 433 474 } 434 - 435 - if (ifcvf_start_datapath(adapter) < 0) 436 - IFCVF_ERR(adapter->pdev, 437 - "Failed to set ifcvf vdpa status %u\n", 438 - status); 439 475 } 440 476 441 477 ifcvf_set_status(vf, status); ··· 436 486 437 487 static int ifcvf_vdpa_reset(struct vdpa_device *vdpa_dev) 438 488 { 439 - struct ifcvf_adapter *adapter; 440 - struct ifcvf_hw *vf; 441 - u8 status_old; 489 + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 490 + u8 status = ifcvf_get_status(vf); 442 491 443 - vf = vdpa_to_vf(vdpa_dev); 444 - adapter = vdpa_to_adapter(vdpa_dev); 445 - status_old = ifcvf_get_status(vf); 492 + ifcvf_stop(vf); 446 493 447 - if (status_old == 0) 448 - return 0; 449 - 450 - if (status_old & VIRTIO_CONFIG_S_DRIVER_OK) { 451 - ifcvf_stop_datapath(adapter); 494 + if (status & VIRTIO_CONFIG_S_DRIVER_OK) 452 495 ifcvf_free_irq(vf); 453 - } 454 496 455 - ifcvf_reset_vring(adapter); 497 + ifcvf_reset(vf); 456 498 457 499 return 0; 458 500 } 459 501 460 502 static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev) 461 503 { 462 - return IFCVF_QUEUE_MAX; 504 + struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 505 + 506 + return ifcvf_get_max_vq_size(vf); 463 507 } 464 508 465 509 static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid, ··· 486 542 { 487 543 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 488 544 489 - vf->vring[qid].ready = ready; 545 + ifcvf_set_vq_ready(vf, qid, ready); 490 546 } 491 547 492 548 static bool ifcvf_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid) 493 549 { 494 550 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 495 551 496 - return vf->vring[qid].ready; 552 + return ifcvf_get_vq_ready(vf, qid); 497 553 } 498 554 499 555 static void ifcvf_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, ··· 501 557 { 502 558 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 503 559 504 - vf->vring[qid].size = num; 560 + ifcvf_set_vq_num(vf, qid, num); 505 561 } 506 562 507 563 static int ifcvf_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid, ··· 510 566 { 511 567 struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev); 512 568 513 - vf->vring[qid].desc = desc_area; 514 - vf->vring[qid].avail = driver_area; 515 - vf->vring[qid].used = device_area; 516 - 517 - return 0; 569 + return ifcvf_set_vq_address(vf, qid, desc_area, driver_area, device_area); 518 570 } 519 571 520 572 static void ifcvf_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid) ··· 832 892 return 0; 833 893 834 894 err: 895 + kfree(ifcvf_mgmt_dev->vf.vring); 835 896 kfree(ifcvf_mgmt_dev); 836 897 return ret; 837 898 } ··· 843 902 844 903 ifcvf_mgmt_dev = pci_get_drvdata(pdev); 845 904 vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev); 905 + kfree(ifcvf_mgmt_dev->vf.vring); 846 906 kfree(ifcvf_mgmt_dev); 847 907 } 848 908 ··· 853 911 N3000_DEVICE_ID, 854 912 PCI_VENDOR_ID_INTEL, 855 913 N3000_SUBSYS_DEVICE_ID) }, 856 - /* C5000X-PL network device */ 914 + /* C5000X-PL network device 915 + * F2000X-PL network device 916 + */ 857 917 { PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET, 858 918 VIRTIO_TRANS_ID_NET, 859 919 PCI_VENDOR_ID_INTEL,

+156 -9

drivers/vdpa/mlx5/net/mlx5_vnet.c

··· 83 83 u64 driver_addr; 84 84 u16 avail_index; 85 85 u16 used_index; 86 + struct msi_map map; 86 87 bool ready; 87 88 bool restore; 88 89 }; ··· 119 118 u16 avail_idx; 120 119 u16 used_idx; 121 120 int fw_state; 121 + struct msi_map map; 122 122 123 123 /* keep last in the struct */ 124 124 struct mlx5_vq_restore_info ri; ··· 810 808 BIT_ULL(MLX5_OBJ_TYPE_VIRTIO_Q_COUNTERS); 811 809 } 812 810 811 + static bool msix_mode_supported(struct mlx5_vdpa_dev *mvdev) 812 + { 813 + return MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, event_mode) & 814 + (1 << MLX5_VIRTIO_Q_EVENT_MODE_MSIX_MODE) && 815 + pci_msix_can_alloc_dyn(mvdev->mdev->pdev); 816 + } 817 + 813 818 static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 814 819 { 815 820 int inlen = MLX5_ST_SZ_BYTES(create_virtio_net_q_in); ··· 858 849 if (vq_is_tx(mvq->index)) 859 850 MLX5_SET(virtio_net_q_object, obj_context, tisn_or_qpn, ndev->res.tisn); 860 851 861 - MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); 852 + if (mvq->map.virq) { 853 + MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_MSIX_MODE); 854 + MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->map.index); 855 + } else { 856 + MLX5_SET(virtio_q, vq_ctx, event_mode, MLX5_VIRTIO_Q_EVENT_MODE_QP_MODE); 857 + MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); 858 + } 859 + 862 860 MLX5_SET(virtio_q, vq_ctx, queue_index, mvq->index); 863 - MLX5_SET(virtio_q, vq_ctx, event_qpn_or_msix, mvq->fwqp.mqp.qpn); 864 861 MLX5_SET(virtio_q, vq_ctx, queue_size, mvq->num_ent); 865 862 MLX5_SET(virtio_q, vq_ctx, virtio_version_1_0, 866 863 !!(ndev->mvdev.actual_features & BIT_ULL(VIRTIO_F_VERSION_1))); ··· 1209 1194 mlx5_vdpa_warn(&ndev->mvdev, "dealloc counter set 0x%x\n", mvq->counter_set_id); 1210 1195 } 1211 1196 1197 + static irqreturn_t mlx5_vdpa_int_handler(int irq, void *priv) 1198 + { 1199 + struct vdpa_callback *cb = priv; 1200 + 1201 + if (cb->callback) 1202 + return cb->callback(cb->private); 1203 + 1204 + return IRQ_HANDLED; 1205 + } 1206 + 1207 + static void alloc_vector(struct mlx5_vdpa_net *ndev, 1208 + struct mlx5_vdpa_virtqueue *mvq) 1209 + { 1210 + struct mlx5_vdpa_irq_pool *irqp = &ndev->irqp; 1211 + struct mlx5_vdpa_irq_pool_entry *ent; 1212 + int err; 1213 + int i; 1214 + 1215 + for (i = 0; i < irqp->num_ent; i++) { 1216 + ent = &irqp->entries[i]; 1217 + if (!ent->used) { 1218 + snprintf(ent->name, MLX5_VDPA_IRQ_NAME_LEN, "%s-vq-%d", 1219 + dev_name(&ndev->mvdev.vdev.dev), mvq->index); 1220 + ent->dev_id = &ndev->event_cbs[mvq->index]; 1221 + err = request_irq(ent->map.virq, mlx5_vdpa_int_handler, 0, 1222 + ent->name, ent->dev_id); 1223 + if (err) 1224 + return; 1225 + 1226 + ent->used = true; 1227 + mvq->map = ent->map; 1228 + return; 1229 + } 1230 + } 1231 + } 1232 + 1233 + static void dealloc_vector(struct mlx5_vdpa_net *ndev, 1234 + struct mlx5_vdpa_virtqueue *mvq) 1235 + { 1236 + struct mlx5_vdpa_irq_pool *irqp = &ndev->irqp; 1237 + int i; 1238 + 1239 + for (i = 0; i < irqp->num_ent; i++) 1240 + if (mvq->map.virq == irqp->entries[i].map.virq) { 1241 + free_irq(mvq->map.virq, irqp->entries[i].dev_id); 1242 + irqp->entries[i].used = false; 1243 + return; 1244 + } 1245 + } 1246 + 1212 1247 static int setup_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq) 1213 1248 { 1214 1249 u16 idx = mvq->index; ··· 1288 1223 1289 1224 err = counter_set_alloc(ndev, mvq); 1290 1225 if (err) 1291 - goto err_counter; 1226 + goto err_connect; 1292 1227 1228 + alloc_vector(ndev, mvq); 1293 1229 err = create_virtqueue(ndev, mvq); 1294 1230 if (err) 1295 - goto err_connect; 1231 + goto err_vq; 1296 1232 1297 1233 if (mvq->ready) { 1298 1234 err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); 1299 1235 if (err) { 1300 1236 mlx5_vdpa_warn(&ndev->mvdev, "failed to modify to ready vq idx %d(%d)\n", 1301 1237 idx, err); 1302 - goto err_connect; 1238 + goto err_modify; 1303 1239 } 1304 1240 } 1305 1241 1306 1242 mvq->initialized = true; 1307 1243 return 0; 1308 1244 1309 - err_connect: 1245 + err_modify: 1246 + destroy_virtqueue(ndev, mvq); 1247 + err_vq: 1248 + dealloc_vector(ndev, mvq); 1310 1249 counter_set_dealloc(ndev, mvq); 1311 - err_counter: 1250 + err_connect: 1312 1251 qp_destroy(ndev, &mvq->vqqp); 1313 1252 err_vqqp: 1314 1253 qp_destroy(ndev, &mvq->fwqp); ··· 1357 1288 1358 1289 suspend_vq(ndev, mvq); 1359 1290 destroy_virtqueue(ndev, mvq); 1291 + dealloc_vector(ndev, mvq); 1360 1292 counter_set_dealloc(ndev, mvq); 1361 1293 qp_destroy(ndev, &mvq->vqqp); 1362 1294 qp_destroy(ndev, &mvq->fwqp); ··· 2575 2505 ri->desc_addr = mvq->desc_addr; 2576 2506 ri->device_addr = mvq->device_addr; 2577 2507 ri->driver_addr = mvq->driver_addr; 2508 + ri->map = mvq->map; 2578 2509 ri->restore = true; 2579 2510 return 0; 2580 2511 } ··· 2620 2549 mvq->desc_addr = ri->desc_addr; 2621 2550 mvq->device_addr = ri->device_addr; 2622 2551 mvq->driver_addr = ri->driver_addr; 2552 + mvq->map = ri->map; 2623 2553 } 2624 2554 } 2625 2555 ··· 2905 2833 return mvdev->vdev.dma_dev; 2906 2834 } 2907 2835 2836 + static void free_irqs(struct mlx5_vdpa_net *ndev) 2837 + { 2838 + struct mlx5_vdpa_irq_pool_entry *ent; 2839 + int i; 2840 + 2841 + if (!msix_mode_supported(&ndev->mvdev)) 2842 + return; 2843 + 2844 + if (!ndev->irqp.entries) 2845 + return; 2846 + 2847 + for (i = ndev->irqp.num_ent - 1; i >= 0; i--) { 2848 + ent = ndev->irqp.entries + i; 2849 + if (ent->map.virq) 2850 + pci_msix_free_irq(ndev->mvdev.mdev->pdev, ent->map); 2851 + } 2852 + kfree(ndev->irqp.entries); 2853 + } 2854 + 2908 2855 static void mlx5_vdpa_free(struct vdpa_device *vdev) 2909 2856 { 2910 2857 struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); ··· 2939 2848 mlx5_mpfs_del_mac(pfmdev, ndev->config.mac); 2940 2849 } 2941 2850 mlx5_vdpa_free_resources(&ndev->mvdev); 2851 + free_irqs(ndev); 2942 2852 kfree(ndev->event_cbs); 2943 2853 kfree(ndev->vqs); 2944 2854 } ··· 2968 2876 return ret; 2969 2877 } 2970 2878 2971 - static int mlx5_get_vq_irq(struct vdpa_device *vdv, u16 idx) 2879 + static int mlx5_get_vq_irq(struct vdpa_device *vdev, u16 idx) 2972 2880 { 2973 - return -EOPNOTSUPP; 2881 + struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); 2882 + struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); 2883 + struct mlx5_vdpa_virtqueue *mvq; 2884 + 2885 + if (!is_index_valid(mvdev, idx)) 2886 + return -EINVAL; 2887 + 2888 + if (is_ctrl_vq_idx(mvdev, idx)) 2889 + return -EOPNOTSUPP; 2890 + 2891 + mvq = &ndev->vqs[idx]; 2892 + if (!mvq->map.virq) 2893 + return -EOPNOTSUPP; 2894 + 2895 + return mvq->map.virq; 2974 2896 } 2975 2897 2976 2898 static u64 mlx5_vdpa_get_driver_features(struct vdpa_device *vdev) ··· 3261 3155 return err; 3262 3156 } 3263 3157 3158 + static void allocate_irqs(struct mlx5_vdpa_net *ndev) 3159 + { 3160 + struct mlx5_vdpa_irq_pool_entry *ent; 3161 + int i; 3162 + 3163 + if (!msix_mode_supported(&ndev->mvdev)) 3164 + return; 3165 + 3166 + if (!ndev->mvdev.mdev->pdev) 3167 + return; 3168 + 3169 + ndev->irqp.entries = kcalloc(ndev->mvdev.max_vqs, sizeof(*ndev->irqp.entries), GFP_KERNEL); 3170 + if (!ndev->irqp.entries) 3171 + return; 3172 + 3173 + 3174 + for (i = 0; i < ndev->mvdev.max_vqs; i++) { 3175 + ent = ndev->irqp.entries + i; 3176 + snprintf(ent->name, MLX5_VDPA_IRQ_NAME_LEN, "%s-vq-%d", 3177 + dev_name(&ndev->mvdev.vdev.dev), i); 3178 + ent->map = pci_msix_alloc_irq_at(ndev->mvdev.mdev->pdev, MSI_ANY_INDEX, NULL); 3179 + if (!ent->map.virq) 3180 + return; 3181 + 3182 + ndev->irqp.num_ent++; 3183 + } 3184 + } 3185 + 3264 3186 static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, 3265 3187 const struct vdpa_dev_set_config *add_config) 3266 3188 { ··· 3367 3233 } 3368 3234 3369 3235 init_mvqs(ndev); 3236 + allocate_irqs(ndev); 3370 3237 init_rwsem(&ndev->reslock); 3371 3238 config = &ndev->config; 3372 3239 ··· 3548 3413 kfree(mgtdev); 3549 3414 } 3550 3415 3416 + static void mlx5v_shutdown(struct auxiliary_device *auxdev) 3417 + { 3418 + struct mlx5_vdpa_mgmtdev *mgtdev; 3419 + struct mlx5_vdpa_net *ndev; 3420 + 3421 + mgtdev = auxiliary_get_drvdata(auxdev); 3422 + ndev = mgtdev->ndev; 3423 + 3424 + free_irqs(ndev); 3425 + } 3426 + 3551 3427 static const struct auxiliary_device_id mlx5v_id_table[] = { 3552 3428 { .name = MLX5_ADEV_NAME ".vnet", }, 3553 3429 {}, ··· 3570 3424 .name = "vnet", 3571 3425 .probe = mlx5v_probe, 3572 3426 .remove = mlx5v_remove, 3427 + .shutdown = mlx5v_shutdown, 3573 3428 .id_table = mlx5v_id_table, 3574 3429 }; 3575 3430

+15

drivers/vdpa/mlx5/net/mlx5_vnet.h

··· 26 26 return (u16)(key >> 48) & 0xfff; 27 27 } 28 28 29 + #define MLX5_VDPA_IRQ_NAME_LEN 32 30 + 31 + struct mlx5_vdpa_irq_pool_entry { 32 + struct msi_map map; 33 + bool used; 34 + char name[MLX5_VDPA_IRQ_NAME_LEN]; 35 + void *dev_id; 36 + }; 37 + 38 + struct mlx5_vdpa_irq_pool { 39 + int num_ent; 40 + struct mlx5_vdpa_irq_pool_entry *entries; 41 + }; 42 + 29 43 struct mlx5_vdpa_net { 30 44 struct mlx5_vdpa_dev mvdev; 31 45 struct mlx5_vdpa_net_resources res; ··· 63 49 struct vdpa_callback config_cb; 64 50 struct mlx5_vdpa_wq_ent cvq_ent; 65 51 struct hlist_head macvlan_hash[MLX5V_MACVLAN_SIZE]; 52 + struct mlx5_vdpa_irq_pool irqp; 66 53 struct dentry *debugfs; 67 54 }; 68 55

+10

drivers/vdpa/pds/Makefile

··· 1 + # SPDX-License-Identifier: GPL-2.0-only 2 + # Copyright(c) 2023 Advanced Micro Devices, Inc 3 + 4 + obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o 5 + 6 + pds_vdpa-y := aux_drv.o \ 7 + cmds.o \ 8 + vdpa_dev.o 9 + 10 + pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o

+140

drivers/vdpa/pds/aux_drv.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 + 4 + #include <linux/auxiliary_bus.h> 5 + #include <linux/pci.h> 6 + #include <linux/vdpa.h> 7 + #include <linux/virtio_pci_modern.h> 8 + 9 + #include <linux/pds/pds_common.h> 10 + #include <linux/pds/pds_core_if.h> 11 + #include <linux/pds/pds_adminq.h> 12 + #include <linux/pds/pds_auxbus.h> 13 + 14 + #include "aux_drv.h" 15 + #include "debugfs.h" 16 + #include "vdpa_dev.h" 17 + 18 + static const struct auxiliary_device_id pds_vdpa_id_table[] = { 19 + { .name = PDS_VDPA_DEV_NAME, }, 20 + {}, 21 + }; 22 + 23 + static int pds_vdpa_device_id_check(struct pci_dev *pdev) 24 + { 25 + if (pdev->device != PCI_DEVICE_ID_PENSANDO_VDPA_VF || 26 + pdev->vendor != PCI_VENDOR_ID_PENSANDO) 27 + return -ENODEV; 28 + 29 + return PCI_DEVICE_ID_PENSANDO_VDPA_VF; 30 + } 31 + 32 + static int pds_vdpa_probe(struct auxiliary_device *aux_dev, 33 + const struct auxiliary_device_id *id) 34 + 35 + { 36 + struct pds_auxiliary_dev *padev = 37 + container_of(aux_dev, struct pds_auxiliary_dev, aux_dev); 38 + struct device *dev = &aux_dev->dev; 39 + struct pds_vdpa_aux *vdpa_aux; 40 + int err; 41 + 42 + vdpa_aux = kzalloc(sizeof(*vdpa_aux), GFP_KERNEL); 43 + if (!vdpa_aux) 44 + return -ENOMEM; 45 + 46 + vdpa_aux->padev = padev; 47 + vdpa_aux->vf_id = pci_iov_vf_id(padev->vf_pdev); 48 + auxiliary_set_drvdata(aux_dev, vdpa_aux); 49 + 50 + /* Get device ident info and set up the vdpa_mgmt_dev */ 51 + err = pds_vdpa_get_mgmt_info(vdpa_aux); 52 + if (err) 53 + goto err_free_mem; 54 + 55 + /* Find the virtio configuration */ 56 + vdpa_aux->vd_mdev.pci_dev = padev->vf_pdev; 57 + vdpa_aux->vd_mdev.device_id_check = pds_vdpa_device_id_check; 58 + vdpa_aux->vd_mdev.dma_mask = DMA_BIT_MASK(PDS_CORE_ADDR_LEN); 59 + err = vp_modern_probe(&vdpa_aux->vd_mdev); 60 + if (err) { 61 + dev_err(dev, "Unable to probe for virtio configuration: %pe\n", 62 + ERR_PTR(err)); 63 + goto err_free_mgmt_info; 64 + } 65 + 66 + /* Let vdpa know that we can provide devices */ 67 + err = vdpa_mgmtdev_register(&vdpa_aux->vdpa_mdev); 68 + if (err) { 69 + dev_err(dev, "%s: Failed to initialize vdpa_mgmt interface: %pe\n", 70 + __func__, ERR_PTR(err)); 71 + goto err_free_virtio; 72 + } 73 + 74 + pds_vdpa_debugfs_add_pcidev(vdpa_aux); 75 + pds_vdpa_debugfs_add_ident(vdpa_aux); 76 + 77 + return 0; 78 + 79 + err_free_virtio: 80 + vp_modern_remove(&vdpa_aux->vd_mdev); 81 + err_free_mgmt_info: 82 + pci_free_irq_vectors(padev->vf_pdev); 83 + err_free_mem: 84 + kfree(vdpa_aux); 85 + auxiliary_set_drvdata(aux_dev, NULL); 86 + 87 + return err; 88 + } 89 + 90 + static void pds_vdpa_remove(struct auxiliary_device *aux_dev) 91 + { 92 + struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev); 93 + struct device *dev = &aux_dev->dev; 94 + 95 + vdpa_mgmtdev_unregister(&vdpa_aux->vdpa_mdev); 96 + vp_modern_remove(&vdpa_aux->vd_mdev); 97 + pci_free_irq_vectors(vdpa_aux->padev->vf_pdev); 98 + 99 + pds_vdpa_debugfs_del_vdpadev(vdpa_aux); 100 + kfree(vdpa_aux); 101 + auxiliary_set_drvdata(aux_dev, NULL); 102 + 103 + dev_info(dev, "Removed\n"); 104 + } 105 + 106 + static struct auxiliary_driver pds_vdpa_driver = { 107 + .name = PDS_DEV_TYPE_VDPA_STR, 108 + .probe = pds_vdpa_probe, 109 + .remove = pds_vdpa_remove, 110 + .id_table = pds_vdpa_id_table, 111 + }; 112 + 113 + static void __exit pds_vdpa_cleanup(void) 114 + { 115 + auxiliary_driver_unregister(&pds_vdpa_driver); 116 + 117 + pds_vdpa_debugfs_destroy(); 118 + } 119 + module_exit(pds_vdpa_cleanup); 120 + 121 + static int __init pds_vdpa_init(void) 122 + { 123 + int err; 124 + 125 + pds_vdpa_debugfs_create(); 126 + 127 + err = auxiliary_driver_register(&pds_vdpa_driver); 128 + if (err) { 129 + pr_err("%s: aux driver register failed: %pe\n", 130 + PDS_VDPA_DRV_NAME, ERR_PTR(err)); 131 + pds_vdpa_debugfs_destroy(); 132 + } 133 + 134 + return err; 135 + } 136 + module_init(pds_vdpa_init); 137 + 138 + MODULE_DESCRIPTION(PDS_VDPA_DRV_DESCRIPTION); 139 + MODULE_AUTHOR("Advanced Micro Devices, Inc"); 140 + MODULE_LICENSE("GPL");

+26

drivers/vdpa/pds/aux_drv.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 + 4 + #ifndef _AUX_DRV_H_ 5 + #define _AUX_DRV_H_ 6 + 7 + #include <linux/virtio_pci_modern.h> 8 + 9 + #define PDS_VDPA_DRV_DESCRIPTION "AMD/Pensando vDPA VF Device Driver" 10 + #define PDS_VDPA_DRV_NAME KBUILD_MODNAME 11 + 12 + struct pds_vdpa_aux { 13 + struct pds_auxiliary_dev *padev; 14 + 15 + struct vdpa_mgmt_dev vdpa_mdev; 16 + struct pds_vdpa_device *pdsv; 17 + 18 + struct pds_vdpa_ident ident; 19 + 20 + int vf_id; 21 + struct dentry *dentry; 22 + struct virtio_pci_modern_device vd_mdev; 23 + 24 + int nintrs; 25 + }; 26 + #endif /* _AUX_DRV_H_ */

+185

drivers/vdpa/pds/cmds.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 + 4 + #include <linux/vdpa.h> 5 + #include <linux/virtio_pci_modern.h> 6 + 7 + #include <linux/pds/pds_common.h> 8 + #include <linux/pds/pds_core_if.h> 9 + #include <linux/pds/pds_adminq.h> 10 + #include <linux/pds/pds_auxbus.h> 11 + 12 + #include "vdpa_dev.h" 13 + #include "aux_drv.h" 14 + #include "cmds.h" 15 + 16 + int pds_vdpa_init_hw(struct pds_vdpa_device *pdsv) 17 + { 18 + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; 19 + struct device *dev = &padev->aux_dev.dev; 20 + union pds_core_adminq_cmd cmd = { 21 + .vdpa_init.opcode = PDS_VDPA_CMD_INIT, 22 + .vdpa_init.vdpa_index = pdsv->vdpa_index, 23 + .vdpa_init.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), 24 + }; 25 + union pds_core_adminq_comp comp = {}; 26 + int err; 27 + 28 + /* Initialize the vdpa/virtio device */ 29 + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_init), 30 + &comp, 0); 31 + if (err) 32 + dev_dbg(dev, "Failed to init hw, status %d: %pe\n", 33 + comp.status, ERR_PTR(err)); 34 + 35 + return err; 36 + } 37 + 38 + int pds_vdpa_cmd_reset(struct pds_vdpa_device *pdsv) 39 + { 40 + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; 41 + struct device *dev = &padev->aux_dev.dev; 42 + union pds_core_adminq_cmd cmd = { 43 + .vdpa.opcode = PDS_VDPA_CMD_RESET, 44 + .vdpa.vdpa_index = pdsv->vdpa_index, 45 + .vdpa.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), 46 + }; 47 + union pds_core_adminq_comp comp = {}; 48 + int err; 49 + 50 + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa), &comp, 0); 51 + if (err) 52 + dev_dbg(dev, "Failed to reset hw, status %d: %pe\n", 53 + comp.status, ERR_PTR(err)); 54 + 55 + return err; 56 + } 57 + 58 + int pds_vdpa_cmd_set_status(struct pds_vdpa_device *pdsv, u8 status) 59 + { 60 + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; 61 + struct device *dev = &padev->aux_dev.dev; 62 + union pds_core_adminq_cmd cmd = { 63 + .vdpa_status.opcode = PDS_VDPA_CMD_STATUS_UPDATE, 64 + .vdpa_status.vdpa_index = pdsv->vdpa_index, 65 + .vdpa_status.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), 66 + .vdpa_status.status = status, 67 + }; 68 + union pds_core_adminq_comp comp = {}; 69 + int err; 70 + 71 + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_status), &comp, 0); 72 + if (err) 73 + dev_dbg(dev, "Failed to set status to %#x, error status %d: %pe\n", 74 + status, comp.status, ERR_PTR(err)); 75 + 76 + return err; 77 + } 78 + 79 + int pds_vdpa_cmd_set_mac(struct pds_vdpa_device *pdsv, u8 *mac) 80 + { 81 + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; 82 + struct device *dev = &padev->aux_dev.dev; 83 + union pds_core_adminq_cmd cmd = { 84 + .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR, 85 + .vdpa_setattr.vdpa_index = pdsv->vdpa_index, 86 + .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), 87 + .vdpa_setattr.attr = PDS_VDPA_ATTR_MAC, 88 + }; 89 + union pds_core_adminq_comp comp = {}; 90 + int err; 91 + 92 + ether_addr_copy(cmd.vdpa_setattr.mac, mac); 93 + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_setattr), 94 + &comp, 0); 95 + if (err) 96 + dev_dbg(dev, "Failed to set mac address %pM, status %d: %pe\n", 97 + mac, comp.status, ERR_PTR(err)); 98 + 99 + return err; 100 + } 101 + 102 + int pds_vdpa_cmd_set_max_vq_pairs(struct pds_vdpa_device *pdsv, u16 max_vqp) 103 + { 104 + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; 105 + struct device *dev = &padev->aux_dev.dev; 106 + union pds_core_adminq_cmd cmd = { 107 + .vdpa_setattr.opcode = PDS_VDPA_CMD_SET_ATTR, 108 + .vdpa_setattr.vdpa_index = pdsv->vdpa_index, 109 + .vdpa_setattr.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), 110 + .vdpa_setattr.attr = PDS_VDPA_ATTR_MAX_VQ_PAIRS, 111 + .vdpa_setattr.max_vq_pairs = cpu_to_le16(max_vqp), 112 + }; 113 + union pds_core_adminq_comp comp = {}; 114 + int err; 115 + 116 + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_setattr), 117 + &comp, 0); 118 + if (err) 119 + dev_dbg(dev, "Failed to set max vq pairs %u, status %d: %pe\n", 120 + max_vqp, comp.status, ERR_PTR(err)); 121 + 122 + return err; 123 + } 124 + 125 + int pds_vdpa_cmd_init_vq(struct pds_vdpa_device *pdsv, u16 qid, u16 invert_idx, 126 + struct pds_vdpa_vq_info *vq_info) 127 + { 128 + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; 129 + struct device *dev = &padev->aux_dev.dev; 130 + union pds_core_adminq_cmd cmd = { 131 + .vdpa_vq_init.opcode = PDS_VDPA_CMD_VQ_INIT, 132 + .vdpa_vq_init.vdpa_index = pdsv->vdpa_index, 133 + .vdpa_vq_init.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), 134 + .vdpa_vq_init.qid = cpu_to_le16(qid), 135 + .vdpa_vq_init.len = cpu_to_le16(ilog2(vq_info->q_len)), 136 + .vdpa_vq_init.desc_addr = cpu_to_le64(vq_info->desc_addr), 137 + .vdpa_vq_init.avail_addr = cpu_to_le64(vq_info->avail_addr), 138 + .vdpa_vq_init.used_addr = cpu_to_le64(vq_info->used_addr), 139 + .vdpa_vq_init.intr_index = cpu_to_le16(qid), 140 + .vdpa_vq_init.avail_index = cpu_to_le16(vq_info->avail_idx ^ invert_idx), 141 + .vdpa_vq_init.used_index = cpu_to_le16(vq_info->used_idx ^ invert_idx), 142 + }; 143 + union pds_core_adminq_comp comp = {}; 144 + int err; 145 + 146 + dev_dbg(dev, "%s: qid %d len %d desc_addr %#llx avail_addr %#llx used_addr %#llx\n", 147 + __func__, qid, ilog2(vq_info->q_len), 148 + vq_info->desc_addr, vq_info->avail_addr, vq_info->used_addr); 149 + 150 + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_vq_init), 151 + &comp, 0); 152 + if (err) 153 + dev_dbg(dev, "Failed to init vq %d, status %d: %pe\n", 154 + qid, comp.status, ERR_PTR(err)); 155 + 156 + return err; 157 + } 158 + 159 + int pds_vdpa_cmd_reset_vq(struct pds_vdpa_device *pdsv, u16 qid, u16 invert_idx, 160 + struct pds_vdpa_vq_info *vq_info) 161 + { 162 + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; 163 + struct device *dev = &padev->aux_dev.dev; 164 + union pds_core_adminq_cmd cmd = { 165 + .vdpa_vq_reset.opcode = PDS_VDPA_CMD_VQ_RESET, 166 + .vdpa_vq_reset.vdpa_index = pdsv->vdpa_index, 167 + .vdpa_vq_reset.vf_id = cpu_to_le16(pdsv->vdpa_aux->vf_id), 168 + .vdpa_vq_reset.qid = cpu_to_le16(qid), 169 + }; 170 + union pds_core_adminq_comp comp = {}; 171 + int err; 172 + 173 + err = pds_client_adminq_cmd(padev, &cmd, sizeof(cmd.vdpa_vq_reset), 174 + &comp, 0); 175 + if (err) { 176 + dev_dbg(dev, "Failed to reset vq %d, status %d: %pe\n", 177 + qid, comp.status, ERR_PTR(err)); 178 + return err; 179 + } 180 + 181 + vq_info->avail_idx = le16_to_cpu(comp.vdpa_vq_reset.avail_index) ^ invert_idx; 182 + vq_info->used_idx = le16_to_cpu(comp.vdpa_vq_reset.used_index) ^ invert_idx; 183 + 184 + return 0; 185 + }

+18

drivers/vdpa/pds/cmds.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 + 4 + #ifndef _VDPA_CMDS_H_ 5 + #define _VDPA_CMDS_H_ 6 + 7 + int pds_vdpa_init_hw(struct pds_vdpa_device *pdsv); 8 + 9 + int pds_vdpa_cmd_reset(struct pds_vdpa_device *pdsv); 10 + int pds_vdpa_cmd_set_status(struct pds_vdpa_device *pdsv, u8 status); 11 + int pds_vdpa_cmd_set_mac(struct pds_vdpa_device *pdsv, u8 *mac); 12 + int pds_vdpa_cmd_set_max_vq_pairs(struct pds_vdpa_device *pdsv, u16 max_vqp); 13 + int pds_vdpa_cmd_init_vq(struct pds_vdpa_device *pdsv, u16 qid, u16 invert_idx, 14 + struct pds_vdpa_vq_info *vq_info); 15 + int pds_vdpa_cmd_reset_vq(struct pds_vdpa_device *pdsv, u16 qid, u16 invert_idx, 16 + struct pds_vdpa_vq_info *vq_info); 17 + int pds_vdpa_cmd_set_features(struct pds_vdpa_device *pdsv, u64 features); 18 + #endif /* _VDPA_CMDS_H_ */

+289

drivers/vdpa/pds/debugfs.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 + 4 + #include <linux/pci.h> 5 + #include <linux/vdpa.h> 6 + 7 + #include <linux/pds/pds_common.h> 8 + #include <linux/pds/pds_core_if.h> 9 + #include <linux/pds/pds_adminq.h> 10 + #include <linux/pds/pds_auxbus.h> 11 + 12 + #include "aux_drv.h" 13 + #include "vdpa_dev.h" 14 + #include "debugfs.h" 15 + 16 + static struct dentry *dbfs_dir; 17 + 18 + void pds_vdpa_debugfs_create(void) 19 + { 20 + dbfs_dir = debugfs_create_dir(PDS_VDPA_DRV_NAME, NULL); 21 + } 22 + 23 + void pds_vdpa_debugfs_destroy(void) 24 + { 25 + debugfs_remove_recursive(dbfs_dir); 26 + dbfs_dir = NULL; 27 + } 28 + 29 + #define PRINT_SBIT_NAME(__seq, __f, __name) \ 30 + do { \ 31 + if ((__f) & (__name)) \ 32 + seq_printf(__seq, " %s", &#__name[16]); \ 33 + } while (0) 34 + 35 + static void print_status_bits(struct seq_file *seq, u8 status) 36 + { 37 + seq_puts(seq, "status:"); 38 + PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_ACKNOWLEDGE); 39 + PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_DRIVER); 40 + PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_DRIVER_OK); 41 + PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_FEATURES_OK); 42 + PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_NEEDS_RESET); 43 + PRINT_SBIT_NAME(seq, status, VIRTIO_CONFIG_S_FAILED); 44 + seq_puts(seq, "\n"); 45 + } 46 + 47 + static void print_feature_bits_all(struct seq_file *seq, u64 features) 48 + { 49 + int i; 50 + 51 + seq_puts(seq, "features:"); 52 + 53 + for (i = 0; i < (sizeof(u64) * 8); i++) { 54 + u64 mask = BIT_ULL(i); 55 + 56 + switch (features & mask) { 57 + case BIT_ULL(VIRTIO_NET_F_CSUM): 58 + seq_puts(seq, " VIRTIO_NET_F_CSUM"); 59 + break; 60 + case BIT_ULL(VIRTIO_NET_F_GUEST_CSUM): 61 + seq_puts(seq, " VIRTIO_NET_F_GUEST_CSUM"); 62 + break; 63 + case BIT_ULL(VIRTIO_NET_F_CTRL_GUEST_OFFLOADS): 64 + seq_puts(seq, " VIRTIO_NET_F_CTRL_GUEST_OFFLOADS"); 65 + break; 66 + case BIT_ULL(VIRTIO_NET_F_MTU): 67 + seq_puts(seq, " VIRTIO_NET_F_MTU"); 68 + break; 69 + case BIT_ULL(VIRTIO_NET_F_MAC): 70 + seq_puts(seq, " VIRTIO_NET_F_MAC"); 71 + break; 72 + case BIT_ULL(VIRTIO_NET_F_GUEST_TSO4): 73 + seq_puts(seq, " VIRTIO_NET_F_GUEST_TSO4"); 74 + break; 75 + case BIT_ULL(VIRTIO_NET_F_GUEST_TSO6): 76 + seq_puts(seq, " VIRTIO_NET_F_GUEST_TSO6"); 77 + break; 78 + case BIT_ULL(VIRTIO_NET_F_GUEST_ECN): 79 + seq_puts(seq, " VIRTIO_NET_F_GUEST_ECN"); 80 + break; 81 + case BIT_ULL(VIRTIO_NET_F_GUEST_UFO): 82 + seq_puts(seq, " VIRTIO_NET_F_GUEST_UFO"); 83 + break; 84 + case BIT_ULL(VIRTIO_NET_F_HOST_TSO4): 85 + seq_puts(seq, " VIRTIO_NET_F_HOST_TSO4"); 86 + break; 87 + case BIT_ULL(VIRTIO_NET_F_HOST_TSO6): 88 + seq_puts(seq, " VIRTIO_NET_F_HOST_TSO6"); 89 + break; 90 + case BIT_ULL(VIRTIO_NET_F_HOST_ECN): 91 + seq_puts(seq, " VIRTIO_NET_F_HOST_ECN"); 92 + break; 93 + case BIT_ULL(VIRTIO_NET_F_HOST_UFO): 94 + seq_puts(seq, " VIRTIO_NET_F_HOST_UFO"); 95 + break; 96 + case BIT_ULL(VIRTIO_NET_F_MRG_RXBUF): 97 + seq_puts(seq, " VIRTIO_NET_F_MRG_RXBUF"); 98 + break; 99 + case BIT_ULL(VIRTIO_NET_F_STATUS): 100 + seq_puts(seq, " VIRTIO_NET_F_STATUS"); 101 + break; 102 + case BIT_ULL(VIRTIO_NET_F_CTRL_VQ): 103 + seq_puts(seq, " VIRTIO_NET_F_CTRL_VQ"); 104 + break; 105 + case BIT_ULL(VIRTIO_NET_F_CTRL_RX): 106 + seq_puts(seq, " VIRTIO_NET_F_CTRL_RX"); 107 + break; 108 + case BIT_ULL(VIRTIO_NET_F_CTRL_VLAN): 109 + seq_puts(seq, " VIRTIO_NET_F_CTRL_VLAN"); 110 + break; 111 + case BIT_ULL(VIRTIO_NET_F_CTRL_RX_EXTRA): 112 + seq_puts(seq, " VIRTIO_NET_F_CTRL_RX_EXTRA"); 113 + break; 114 + case BIT_ULL(VIRTIO_NET_F_GUEST_ANNOUNCE): 115 + seq_puts(seq, " VIRTIO_NET_F_GUEST_ANNOUNCE"); 116 + break; 117 + case BIT_ULL(VIRTIO_NET_F_MQ): 118 + seq_puts(seq, " VIRTIO_NET_F_MQ"); 119 + break; 120 + case BIT_ULL(VIRTIO_NET_F_CTRL_MAC_ADDR): 121 + seq_puts(seq, " VIRTIO_NET_F_CTRL_MAC_ADDR"); 122 + break; 123 + case BIT_ULL(VIRTIO_NET_F_HASH_REPORT): 124 + seq_puts(seq, " VIRTIO_NET_F_HASH_REPORT"); 125 + break; 126 + case BIT_ULL(VIRTIO_NET_F_RSS): 127 + seq_puts(seq, " VIRTIO_NET_F_RSS"); 128 + break; 129 + case BIT_ULL(VIRTIO_NET_F_RSC_EXT): 130 + seq_puts(seq, " VIRTIO_NET_F_RSC_EXT"); 131 + break; 132 + case BIT_ULL(VIRTIO_NET_F_STANDBY): 133 + seq_puts(seq, " VIRTIO_NET_F_STANDBY"); 134 + break; 135 + case BIT_ULL(VIRTIO_NET_F_SPEED_DUPLEX): 136 + seq_puts(seq, " VIRTIO_NET_F_SPEED_DUPLEX"); 137 + break; 138 + case BIT_ULL(VIRTIO_F_NOTIFY_ON_EMPTY): 139 + seq_puts(seq, " VIRTIO_F_NOTIFY_ON_EMPTY"); 140 + break; 141 + case BIT_ULL(VIRTIO_F_ANY_LAYOUT): 142 + seq_puts(seq, " VIRTIO_F_ANY_LAYOUT"); 143 + break; 144 + case BIT_ULL(VIRTIO_F_VERSION_1): 145 + seq_puts(seq, " VIRTIO_F_VERSION_1"); 146 + break; 147 + case BIT_ULL(VIRTIO_F_ACCESS_PLATFORM): 148 + seq_puts(seq, " VIRTIO_F_ACCESS_PLATFORM"); 149 + break; 150 + case BIT_ULL(VIRTIO_F_RING_PACKED): 151 + seq_puts(seq, " VIRTIO_F_RING_PACKED"); 152 + break; 153 + case BIT_ULL(VIRTIO_F_ORDER_PLATFORM): 154 + seq_puts(seq, " VIRTIO_F_ORDER_PLATFORM"); 155 + break; 156 + case BIT_ULL(VIRTIO_F_SR_IOV): 157 + seq_puts(seq, " VIRTIO_F_SR_IOV"); 158 + break; 159 + case 0: 160 + break; 161 + default: 162 + seq_printf(seq, " bit_%d", i); 163 + break; 164 + } 165 + } 166 + 167 + seq_puts(seq, "\n"); 168 + } 169 + 170 + void pds_vdpa_debugfs_add_pcidev(struct pds_vdpa_aux *vdpa_aux) 171 + { 172 + vdpa_aux->dentry = debugfs_create_dir(pci_name(vdpa_aux->padev->vf_pdev), dbfs_dir); 173 + } 174 + 175 + static int identity_show(struct seq_file *seq, void *v) 176 + { 177 + struct pds_vdpa_aux *vdpa_aux = seq->private; 178 + struct vdpa_mgmt_dev *mgmt; 179 + 180 + seq_printf(seq, "aux_dev: %s\n", 181 + dev_name(&vdpa_aux->padev->aux_dev.dev)); 182 + 183 + mgmt = &vdpa_aux->vdpa_mdev; 184 + seq_printf(seq, "max_vqs: %d\n", mgmt->max_supported_vqs); 185 + seq_printf(seq, "config_attr_mask: %#llx\n", mgmt->config_attr_mask); 186 + seq_printf(seq, "supported_features: %#llx\n", mgmt->supported_features); 187 + print_feature_bits_all(seq, mgmt->supported_features); 188 + 189 + return 0; 190 + } 191 + DEFINE_SHOW_ATTRIBUTE(identity); 192 + 193 + void pds_vdpa_debugfs_add_ident(struct pds_vdpa_aux *vdpa_aux) 194 + { 195 + debugfs_create_file("identity", 0400, vdpa_aux->dentry, 196 + vdpa_aux, &identity_fops); 197 + } 198 + 199 + static int config_show(struct seq_file *seq, void *v) 200 + { 201 + struct pds_vdpa_device *pdsv = seq->private; 202 + struct virtio_net_config vc; 203 + u64 driver_features; 204 + u8 status; 205 + 206 + memcpy_fromio(&vc, pdsv->vdpa_aux->vd_mdev.device, 207 + sizeof(struct virtio_net_config)); 208 + 209 + seq_printf(seq, "mac: %pM\n", vc.mac); 210 + seq_printf(seq, "max_virtqueue_pairs: %d\n", 211 + __virtio16_to_cpu(true, vc.max_virtqueue_pairs)); 212 + seq_printf(seq, "mtu: %d\n", __virtio16_to_cpu(true, vc.mtu)); 213 + seq_printf(seq, "speed: %d\n", le32_to_cpu(vc.speed)); 214 + seq_printf(seq, "duplex: %d\n", vc.duplex); 215 + seq_printf(seq, "rss_max_key_size: %d\n", vc.rss_max_key_size); 216 + seq_printf(seq, "rss_max_indirection_table_length: %d\n", 217 + le16_to_cpu(vc.rss_max_indirection_table_length)); 218 + seq_printf(seq, "supported_hash_types: %#x\n", 219 + le32_to_cpu(vc.supported_hash_types)); 220 + seq_printf(seq, "vn_status: %#x\n", 221 + __virtio16_to_cpu(true, vc.status)); 222 + 223 + status = vp_modern_get_status(&pdsv->vdpa_aux->vd_mdev); 224 + seq_printf(seq, "dev_status: %#x\n", status); 225 + print_status_bits(seq, status); 226 + 227 + seq_printf(seq, "req_features: %#llx\n", pdsv->req_features); 228 + print_feature_bits_all(seq, pdsv->req_features); 229 + driver_features = vp_modern_get_driver_features(&pdsv->vdpa_aux->vd_mdev); 230 + seq_printf(seq, "driver_features: %#llx\n", driver_features); 231 + print_feature_bits_all(seq, driver_features); 232 + seq_printf(seq, "vdpa_index: %d\n", pdsv->vdpa_index); 233 + seq_printf(seq, "num_vqs: %d\n", pdsv->num_vqs); 234 + 235 + return 0; 236 + } 237 + DEFINE_SHOW_ATTRIBUTE(config); 238 + 239 + static int vq_show(struct seq_file *seq, void *v) 240 + { 241 + struct pds_vdpa_vq_info *vq = seq->private; 242 + 243 + seq_printf(seq, "ready: %d\n", vq->ready); 244 + seq_printf(seq, "desc_addr: %#llx\n", vq->desc_addr); 245 + seq_printf(seq, "avail_addr: %#llx\n", vq->avail_addr); 246 + seq_printf(seq, "used_addr: %#llx\n", vq->used_addr); 247 + seq_printf(seq, "q_len: %d\n", vq->q_len); 248 + seq_printf(seq, "qid: %d\n", vq->qid); 249 + 250 + seq_printf(seq, "doorbell: %#llx\n", vq->doorbell); 251 + seq_printf(seq, "avail_idx: %d\n", vq->avail_idx); 252 + seq_printf(seq, "used_idx: %d\n", vq->used_idx); 253 + seq_printf(seq, "irq: %d\n", vq->irq); 254 + seq_printf(seq, "irq-name: %s\n", vq->irq_name); 255 + 256 + return 0; 257 + } 258 + DEFINE_SHOW_ATTRIBUTE(vq); 259 + 260 + void pds_vdpa_debugfs_add_vdpadev(struct pds_vdpa_aux *vdpa_aux) 261 + { 262 + int i; 263 + 264 + debugfs_create_file("config", 0400, vdpa_aux->dentry, vdpa_aux->pdsv, &config_fops); 265 + 266 + for (i = 0; i < vdpa_aux->pdsv->num_vqs; i++) { 267 + char name[8]; 268 + 269 + snprintf(name, sizeof(name), "vq%02d", i); 270 + debugfs_create_file(name, 0400, vdpa_aux->dentry, 271 + &vdpa_aux->pdsv->vqs[i], &vq_fops); 272 + } 273 + } 274 + 275 + void pds_vdpa_debugfs_del_vdpadev(struct pds_vdpa_aux *vdpa_aux) 276 + { 277 + debugfs_remove_recursive(vdpa_aux->dentry); 278 + vdpa_aux->dentry = NULL; 279 + } 280 + 281 + void pds_vdpa_debugfs_reset_vdpadev(struct pds_vdpa_aux *vdpa_aux) 282 + { 283 + /* we don't keep track of the entries, so remove it all 284 + * then rebuild the basics 285 + */ 286 + pds_vdpa_debugfs_del_vdpadev(vdpa_aux); 287 + pds_vdpa_debugfs_add_pcidev(vdpa_aux); 288 + pds_vdpa_debugfs_add_ident(vdpa_aux); 289 + }

+17

drivers/vdpa/pds/debugfs.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0 */ 2 + /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 + 4 + #ifndef _PDS_VDPA_DEBUGFS_H_ 5 + #define _PDS_VDPA_DEBUGFS_H_ 6 + 7 + #include <linux/debugfs.h> 8 + 9 + void pds_vdpa_debugfs_create(void); 10 + void pds_vdpa_debugfs_destroy(void); 11 + void pds_vdpa_debugfs_add_pcidev(struct pds_vdpa_aux *vdpa_aux); 12 + void pds_vdpa_debugfs_add_ident(struct pds_vdpa_aux *vdpa_aux); 13 + void pds_vdpa_debugfs_add_vdpadev(struct pds_vdpa_aux *vdpa_aux); 14 + void pds_vdpa_debugfs_del_vdpadev(struct pds_vdpa_aux *vdpa_aux); 15 + void pds_vdpa_debugfs_reset_vdpadev(struct pds_vdpa_aux *vdpa_aux); 16 + 17 + #endif /* _PDS_VDPA_DEBUGFS_H_ */

+769

drivers/vdpa/pds/vdpa_dev.c

··· 1 + // SPDX-License-Identifier: GPL-2.0-only 2 + /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 + 4 + #include <linux/pci.h> 5 + #include <linux/vdpa.h> 6 + #include <uapi/linux/vdpa.h> 7 + #include <linux/virtio_pci_modern.h> 8 + 9 + #include <linux/pds/pds_common.h> 10 + #include <linux/pds/pds_core_if.h> 11 + #include <linux/pds/pds_adminq.h> 12 + #include <linux/pds/pds_auxbus.h> 13 + 14 + #include "vdpa_dev.h" 15 + #include "aux_drv.h" 16 + #include "cmds.h" 17 + #include "debugfs.h" 18 + 19 + static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev); 20 + 21 + static struct pds_vdpa_device *vdpa_to_pdsv(struct vdpa_device *vdpa_dev) 22 + { 23 + return container_of(vdpa_dev, struct pds_vdpa_device, vdpa_dev); 24 + } 25 + 26 + static int pds_vdpa_notify_handler(struct notifier_block *nb, 27 + unsigned long ecode, 28 + void *data) 29 + { 30 + struct pds_vdpa_device *pdsv = container_of(nb, struct pds_vdpa_device, nb); 31 + struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev; 32 + 33 + dev_dbg(dev, "%s: event code %lu\n", __func__, ecode); 34 + 35 + if (ecode == PDS_EVENT_RESET || ecode == PDS_EVENT_LINK_CHANGE) { 36 + if (pdsv->config_cb.callback) 37 + pdsv->config_cb.callback(pdsv->config_cb.private); 38 + } 39 + 40 + return 0; 41 + } 42 + 43 + static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv) 44 + { 45 + struct device *dev = &pdsv->vdpa_aux->padev->aux_dev.dev; 46 + struct notifier_block *nb = &pdsv->nb; 47 + int err; 48 + 49 + if (!nb->notifier_call) { 50 + nb->notifier_call = pds_vdpa_notify_handler; 51 + err = pdsc_register_notify(nb); 52 + if (err) { 53 + nb->notifier_call = NULL; 54 + dev_err(dev, "failed to register pds event handler: %ps\n", 55 + ERR_PTR(err)); 56 + return -EINVAL; 57 + } 58 + dev_dbg(dev, "pds event handler registered\n"); 59 + } 60 + 61 + return 0; 62 + } 63 + 64 + static void pds_vdpa_unregister_event_handler(struct pds_vdpa_device *pdsv) 65 + { 66 + if (pdsv->nb.notifier_call) { 67 + pdsc_unregister_notify(&pdsv->nb); 68 + pdsv->nb.notifier_call = NULL; 69 + } 70 + } 71 + 72 + static int pds_vdpa_set_vq_address(struct vdpa_device *vdpa_dev, u16 qid, 73 + u64 desc_addr, u64 driver_addr, u64 device_addr) 74 + { 75 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 76 + 77 + pdsv->vqs[qid].desc_addr = desc_addr; 78 + pdsv->vqs[qid].avail_addr = driver_addr; 79 + pdsv->vqs[qid].used_addr = device_addr; 80 + 81 + return 0; 82 + } 83 + 84 + static void pds_vdpa_set_vq_num(struct vdpa_device *vdpa_dev, u16 qid, u32 num) 85 + { 86 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 87 + 88 + pdsv->vqs[qid].q_len = num; 89 + } 90 + 91 + static void pds_vdpa_kick_vq(struct vdpa_device *vdpa_dev, u16 qid) 92 + { 93 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 94 + 95 + iowrite16(qid, pdsv->vqs[qid].notify); 96 + } 97 + 98 + static void pds_vdpa_set_vq_cb(struct vdpa_device *vdpa_dev, u16 qid, 99 + struct vdpa_callback *cb) 100 + { 101 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 102 + 103 + pdsv->vqs[qid].event_cb = *cb; 104 + } 105 + 106 + static irqreturn_t pds_vdpa_isr(int irq, void *data) 107 + { 108 + struct pds_vdpa_vq_info *vq; 109 + 110 + vq = data; 111 + if (vq->event_cb.callback) 112 + vq->event_cb.callback(vq->event_cb.private); 113 + 114 + return IRQ_HANDLED; 115 + } 116 + 117 + static void pds_vdpa_release_irq(struct pds_vdpa_device *pdsv, int qid) 118 + { 119 + if (pdsv->vqs[qid].irq == VIRTIO_MSI_NO_VECTOR) 120 + return; 121 + 122 + free_irq(pdsv->vqs[qid].irq, &pdsv->vqs[qid]); 123 + pdsv->vqs[qid].irq = VIRTIO_MSI_NO_VECTOR; 124 + } 125 + 126 + static void pds_vdpa_set_vq_ready(struct vdpa_device *vdpa_dev, u16 qid, bool ready) 127 + { 128 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 129 + struct pci_dev *pdev = pdsv->vdpa_aux->padev->vf_pdev; 130 + struct device *dev = &pdsv->vdpa_dev.dev; 131 + u64 driver_features; 132 + u16 invert_idx = 0; 133 + int irq; 134 + int err; 135 + 136 + dev_dbg(dev, "%s: qid %d ready %d => %d\n", 137 + __func__, qid, pdsv->vqs[qid].ready, ready); 138 + if (ready == pdsv->vqs[qid].ready) 139 + return; 140 + 141 + driver_features = pds_vdpa_get_driver_features(vdpa_dev); 142 + if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) 143 + invert_idx = PDS_VDPA_PACKED_INVERT_IDX; 144 + 145 + if (ready) { 146 + irq = pci_irq_vector(pdev, qid); 147 + snprintf(pdsv->vqs[qid].irq_name, sizeof(pdsv->vqs[qid].irq_name), 148 + "vdpa-%s-%d", dev_name(dev), qid); 149 + 150 + err = request_irq(irq, pds_vdpa_isr, 0, 151 + pdsv->vqs[qid].irq_name, &pdsv->vqs[qid]); 152 + if (err) { 153 + dev_err(dev, "%s: no irq for qid %d: %pe\n", 154 + __func__, qid, ERR_PTR(err)); 155 + return; 156 + } 157 + pdsv->vqs[qid].irq = irq; 158 + 159 + /* Pass vq setup info to DSC using adminq to gather up and 160 + * send all info at once so FW can do its full set up in 161 + * one easy operation 162 + */ 163 + err = pds_vdpa_cmd_init_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]); 164 + if (err) { 165 + dev_err(dev, "Failed to init vq %d: %pe\n", 166 + qid, ERR_PTR(err)); 167 + pds_vdpa_release_irq(pdsv, qid); 168 + ready = false; 169 + } 170 + } else { 171 + err = pds_vdpa_cmd_reset_vq(pdsv, qid, invert_idx, &pdsv->vqs[qid]); 172 + if (err) 173 + dev_err(dev, "%s: reset_vq failed qid %d: %pe\n", 174 + __func__, qid, ERR_PTR(err)); 175 + pds_vdpa_release_irq(pdsv, qid); 176 + } 177 + 178 + pdsv->vqs[qid].ready = ready; 179 + } 180 + 181 + static bool pds_vdpa_get_vq_ready(struct vdpa_device *vdpa_dev, u16 qid) 182 + { 183 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 184 + 185 + return pdsv->vqs[qid].ready; 186 + } 187 + 188 + static int pds_vdpa_set_vq_state(struct vdpa_device *vdpa_dev, u16 qid, 189 + const struct vdpa_vq_state *state) 190 + { 191 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 192 + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; 193 + struct device *dev = &padev->aux_dev.dev; 194 + u64 driver_features; 195 + u16 avail; 196 + u16 used; 197 + 198 + if (pdsv->vqs[qid].ready) { 199 + dev_err(dev, "Setting device position is denied while vq is enabled\n"); 200 + return -EINVAL; 201 + } 202 + 203 + driver_features = pds_vdpa_get_driver_features(vdpa_dev); 204 + if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 205 + avail = state->packed.last_avail_idx | 206 + (state->packed.last_avail_counter << 15); 207 + used = state->packed.last_used_idx | 208 + (state->packed.last_used_counter << 15); 209 + 210 + /* The avail and used index are stored with the packed wrap 211 + * counter bit inverted. This way, in case set_vq_state is 212 + * not called, the initial value can be set to zero prior to 213 + * feature negotiation, and it is good for both packed and 214 + * split vq. 215 + */ 216 + avail ^= PDS_VDPA_PACKED_INVERT_IDX; 217 + used ^= PDS_VDPA_PACKED_INVERT_IDX; 218 + } else { 219 + avail = state->split.avail_index; 220 + /* state->split does not provide a used_index: 221 + * the vq will be set to "empty" here, and the vq will read 222 + * the current used index the next time the vq is kicked. 223 + */ 224 + used = avail; 225 + } 226 + 227 + if (used != avail) { 228 + dev_dbg(dev, "Setting used equal to avail, for interoperability\n"); 229 + used = avail; 230 + } 231 + 232 + pdsv->vqs[qid].avail_idx = avail; 233 + pdsv->vqs[qid].used_idx = used; 234 + 235 + return 0; 236 + } 237 + 238 + static int pds_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid, 239 + struct vdpa_vq_state *state) 240 + { 241 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 242 + struct pds_auxiliary_dev *padev = pdsv->vdpa_aux->padev; 243 + struct device *dev = &padev->aux_dev.dev; 244 + u64 driver_features; 245 + u16 avail; 246 + u16 used; 247 + 248 + if (pdsv->vqs[qid].ready) { 249 + dev_err(dev, "Getting device position is denied while vq is enabled\n"); 250 + return -EINVAL; 251 + } 252 + 253 + avail = pdsv->vqs[qid].avail_idx; 254 + used = pdsv->vqs[qid].used_idx; 255 + 256 + driver_features = pds_vdpa_get_driver_features(vdpa_dev); 257 + if (driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) { 258 + avail ^= PDS_VDPA_PACKED_INVERT_IDX; 259 + used ^= PDS_VDPA_PACKED_INVERT_IDX; 260 + 261 + state->packed.last_avail_idx = avail & 0x7fff; 262 + state->packed.last_avail_counter = avail >> 15; 263 + state->packed.last_used_idx = used & 0x7fff; 264 + state->packed.last_used_counter = used >> 15; 265 + } else { 266 + state->split.avail_index = avail; 267 + /* state->split does not provide a used_index. */ 268 + } 269 + 270 + return 0; 271 + } 272 + 273 + static struct vdpa_notification_area 274 + pds_vdpa_get_vq_notification(struct vdpa_device *vdpa_dev, u16 qid) 275 + { 276 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 277 + struct virtio_pci_modern_device *vd_mdev; 278 + struct vdpa_notification_area area; 279 + 280 + area.addr = pdsv->vqs[qid].notify_pa; 281 + 282 + vd_mdev = &pdsv->vdpa_aux->vd_mdev; 283 + if (!vd_mdev->notify_offset_multiplier) 284 + area.size = PDS_PAGE_SIZE; 285 + else 286 + area.size = vd_mdev->notify_offset_multiplier; 287 + 288 + return area; 289 + } 290 + 291 + static int pds_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev, u16 qid) 292 + { 293 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 294 + 295 + return pdsv->vqs[qid].irq; 296 + } 297 + 298 + static u32 pds_vdpa_get_vq_align(struct vdpa_device *vdpa_dev) 299 + { 300 + return PDS_PAGE_SIZE; 301 + } 302 + 303 + static u32 pds_vdpa_get_vq_group(struct vdpa_device *vdpa_dev, u16 idx) 304 + { 305 + return 0; 306 + } 307 + 308 + static u64 pds_vdpa_get_device_features(struct vdpa_device *vdpa_dev) 309 + { 310 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 311 + 312 + return pdsv->supported_features; 313 + } 314 + 315 + static int pds_vdpa_set_driver_features(struct vdpa_device *vdpa_dev, u64 features) 316 + { 317 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 318 + struct device *dev = &pdsv->vdpa_dev.dev; 319 + u64 driver_features; 320 + u64 nego_features; 321 + u64 missing; 322 + 323 + if (!(features & BIT_ULL(VIRTIO_F_ACCESS_PLATFORM)) && features) { 324 + dev_err(dev, "VIRTIO_F_ACCESS_PLATFORM is not negotiated\n"); 325 + return -EOPNOTSUPP; 326 + } 327 + 328 + pdsv->req_features = features; 329 + 330 + /* Check for valid feature bits */ 331 + nego_features = features & le64_to_cpu(pdsv->vdpa_aux->ident.hw_features); 332 + missing = pdsv->req_features & ~nego_features; 333 + if (missing) { 334 + dev_err(dev, "Can't support all requested features in %#llx, missing %#llx features\n", 335 + pdsv->req_features, missing); 336 + return -EOPNOTSUPP; 337 + } 338 + 339 + driver_features = pds_vdpa_get_driver_features(vdpa_dev); 340 + dev_dbg(dev, "%s: %#llx => %#llx\n", 341 + __func__, driver_features, nego_features); 342 + 343 + if (driver_features == nego_features) 344 + return 0; 345 + 346 + vp_modern_set_features(&pdsv->vdpa_aux->vd_mdev, nego_features); 347 + 348 + return 0; 349 + } 350 + 351 + static u64 pds_vdpa_get_driver_features(struct vdpa_device *vdpa_dev) 352 + { 353 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 354 + 355 + return vp_modern_get_driver_features(&pdsv->vdpa_aux->vd_mdev); 356 + } 357 + 358 + static void pds_vdpa_set_config_cb(struct vdpa_device *vdpa_dev, 359 + struct vdpa_callback *cb) 360 + { 361 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 362 + 363 + pdsv->config_cb.callback = cb->callback; 364 + pdsv->config_cb.private = cb->private; 365 + } 366 + 367 + static u16 pds_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev) 368 + { 369 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 370 + 371 + /* qemu has assert() that vq_num_max <= VIRTQUEUE_MAX_SIZE (1024) */ 372 + return min_t(u16, 1024, BIT(le16_to_cpu(pdsv->vdpa_aux->ident.max_qlen))); 373 + } 374 + 375 + static u32 pds_vdpa_get_device_id(struct vdpa_device *vdpa_dev) 376 + { 377 + return VIRTIO_ID_NET; 378 + } 379 + 380 + static u32 pds_vdpa_get_vendor_id(struct vdpa_device *vdpa_dev) 381 + { 382 + return PCI_VENDOR_ID_PENSANDO; 383 + } 384 + 385 + static u8 pds_vdpa_get_status(struct vdpa_device *vdpa_dev) 386 + { 387 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 388 + 389 + return vp_modern_get_status(&pdsv->vdpa_aux->vd_mdev); 390 + } 391 + 392 + static void pds_vdpa_set_status(struct vdpa_device *vdpa_dev, u8 status) 393 + { 394 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 395 + struct device *dev = &pdsv->vdpa_dev.dev; 396 + u8 old_status; 397 + int i; 398 + 399 + old_status = pds_vdpa_get_status(vdpa_dev); 400 + dev_dbg(dev, "%s: old %#x new %#x\n", __func__, old_status, status); 401 + 402 + pds_vdpa_cmd_set_status(pdsv, status); 403 + 404 + /* Note: still working with FW on the need for this reset cmd */ 405 + if (status == 0) { 406 + pds_vdpa_cmd_reset(pdsv); 407 + 408 + for (i = 0; i < pdsv->num_vqs; i++) { 409 + pdsv->vqs[i].avail_idx = 0; 410 + pdsv->vqs[i].used_idx = 0; 411 + } 412 + } 413 + 414 + if (status & ~old_status & VIRTIO_CONFIG_S_FEATURES_OK) { 415 + for (i = 0; i < pdsv->num_vqs; i++) { 416 + pdsv->vqs[i].notify = 417 + vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev, 418 + i, &pdsv->vqs[i].notify_pa); 419 + } 420 + } 421 + } 422 + 423 + static int pds_vdpa_reset(struct vdpa_device *vdpa_dev) 424 + { 425 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 426 + struct device *dev; 427 + int err = 0; 428 + u8 status; 429 + int i; 430 + 431 + dev = &pdsv->vdpa_aux->padev->aux_dev.dev; 432 + status = pds_vdpa_get_status(vdpa_dev); 433 + 434 + if (status == 0) 435 + return 0; 436 + 437 + if (status & VIRTIO_CONFIG_S_DRIVER_OK) { 438 + /* Reset the vqs */ 439 + for (i = 0; i < pdsv->num_vqs && !err; i++) { 440 + err = pds_vdpa_cmd_reset_vq(pdsv, i, 0, &pdsv->vqs[i]); 441 + if (err) 442 + dev_err(dev, "%s: reset_vq failed qid %d: %pe\n", 443 + __func__, i, ERR_PTR(err)); 444 + pds_vdpa_release_irq(pdsv, i); 445 + memset(&pdsv->vqs[i], 0, sizeof(pdsv->vqs[0])); 446 + pdsv->vqs[i].ready = false; 447 + } 448 + } 449 + 450 + pds_vdpa_set_status(vdpa_dev, 0); 451 + 452 + return 0; 453 + } 454 + 455 + static size_t pds_vdpa_get_config_size(struct vdpa_device *vdpa_dev) 456 + { 457 + return sizeof(struct virtio_net_config); 458 + } 459 + 460 + static void pds_vdpa_get_config(struct vdpa_device *vdpa_dev, 461 + unsigned int offset, 462 + void *buf, unsigned int len) 463 + { 464 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 465 + void __iomem *device; 466 + 467 + if (offset + len > sizeof(struct virtio_net_config)) { 468 + WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len); 469 + return; 470 + } 471 + 472 + device = pdsv->vdpa_aux->vd_mdev.device; 473 + memcpy_fromio(buf, device + offset, len); 474 + } 475 + 476 + static void pds_vdpa_set_config(struct vdpa_device *vdpa_dev, 477 + unsigned int offset, const void *buf, 478 + unsigned int len) 479 + { 480 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 481 + void __iomem *device; 482 + 483 + if (offset + len > sizeof(struct virtio_net_config)) { 484 + WARN(true, "%s: bad read, offset %d len %d\n", __func__, offset, len); 485 + return; 486 + } 487 + 488 + device = pdsv->vdpa_aux->vd_mdev.device; 489 + memcpy_toio(device + offset, buf, len); 490 + } 491 + 492 + static const struct vdpa_config_ops pds_vdpa_ops = { 493 + .set_vq_address = pds_vdpa_set_vq_address, 494 + .set_vq_num = pds_vdpa_set_vq_num, 495 + .kick_vq = pds_vdpa_kick_vq, 496 + .set_vq_cb = pds_vdpa_set_vq_cb, 497 + .set_vq_ready = pds_vdpa_set_vq_ready, 498 + .get_vq_ready = pds_vdpa_get_vq_ready, 499 + .set_vq_state = pds_vdpa_set_vq_state, 500 + .get_vq_state = pds_vdpa_get_vq_state, 501 + .get_vq_notification = pds_vdpa_get_vq_notification, 502 + .get_vq_irq = pds_vdpa_get_vq_irq, 503 + .get_vq_align = pds_vdpa_get_vq_align, 504 + .get_vq_group = pds_vdpa_get_vq_group, 505 + 506 + .get_device_features = pds_vdpa_get_device_features, 507 + .set_driver_features = pds_vdpa_set_driver_features, 508 + .get_driver_features = pds_vdpa_get_driver_features, 509 + .set_config_cb = pds_vdpa_set_config_cb, 510 + .get_vq_num_max = pds_vdpa_get_vq_num_max, 511 + .get_device_id = pds_vdpa_get_device_id, 512 + .get_vendor_id = pds_vdpa_get_vendor_id, 513 + .get_status = pds_vdpa_get_status, 514 + .set_status = pds_vdpa_set_status, 515 + .reset = pds_vdpa_reset, 516 + .get_config_size = pds_vdpa_get_config_size, 517 + .get_config = pds_vdpa_get_config, 518 + .set_config = pds_vdpa_set_config, 519 + }; 520 + static struct virtio_device_id pds_vdpa_id_table[] = { 521 + {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID}, 522 + {0}, 523 + }; 524 + 525 + static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, 526 + const struct vdpa_dev_set_config *add_config) 527 + { 528 + struct pds_vdpa_aux *vdpa_aux; 529 + struct pds_vdpa_device *pdsv; 530 + struct vdpa_mgmt_dev *mgmt; 531 + u16 fw_max_vqs, vq_pairs; 532 + struct device *dma_dev; 533 + struct pci_dev *pdev; 534 + struct device *dev; 535 + u8 mac[ETH_ALEN]; 536 + int err; 537 + int i; 538 + 539 + vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev); 540 + dev = &vdpa_aux->padev->aux_dev.dev; 541 + mgmt = &vdpa_aux->vdpa_mdev; 542 + 543 + if (vdpa_aux->pdsv) { 544 + dev_warn(dev, "Multiple vDPA devices on a VF is not supported.\n"); 545 + return -EOPNOTSUPP; 546 + } 547 + 548 + pdsv = vdpa_alloc_device(struct pds_vdpa_device, vdpa_dev, 549 + dev, &pds_vdpa_ops, 1, 1, name, false); 550 + if (IS_ERR(pdsv)) { 551 + dev_err(dev, "Failed to allocate vDPA structure: %pe\n", pdsv); 552 + return PTR_ERR(pdsv); 553 + } 554 + 555 + vdpa_aux->pdsv = pdsv; 556 + pdsv->vdpa_aux = vdpa_aux; 557 + 558 + pdev = vdpa_aux->padev->vf_pdev; 559 + dma_dev = &pdev->dev; 560 + pdsv->vdpa_dev.dma_dev = dma_dev; 561 + 562 + pdsv->supported_features = mgmt->supported_features; 563 + 564 + if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) { 565 + u64 unsupp_features = 566 + add_config->device_features & ~mgmt->supported_features; 567 + 568 + if (unsupp_features) { 569 + dev_err(dev, "Unsupported features: %#llx\n", unsupp_features); 570 + err = -EOPNOTSUPP; 571 + goto err_unmap; 572 + } 573 + 574 + pdsv->supported_features = add_config->device_features; 575 + } 576 + 577 + err = pds_vdpa_cmd_reset(pdsv); 578 + if (err) { 579 + dev_err(dev, "Failed to reset hw: %pe\n", ERR_PTR(err)); 580 + goto err_unmap; 581 + } 582 + 583 + err = pds_vdpa_init_hw(pdsv); 584 + if (err) { 585 + dev_err(dev, "Failed to init hw: %pe\n", ERR_PTR(err)); 586 + goto err_unmap; 587 + } 588 + 589 + fw_max_vqs = le16_to_cpu(pdsv->vdpa_aux->ident.max_vqs); 590 + vq_pairs = fw_max_vqs / 2; 591 + 592 + /* Make sure we have the queues being requested */ 593 + if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MAX_VQP)) 594 + vq_pairs = add_config->net.max_vq_pairs; 595 + 596 + pdsv->num_vqs = 2 * vq_pairs; 597 + if (pdsv->supported_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ)) 598 + pdsv->num_vqs++; 599 + 600 + if (pdsv->num_vqs > fw_max_vqs) { 601 + dev_err(dev, "%s: queue count requested %u greater than max %u\n", 602 + __func__, pdsv->num_vqs, fw_max_vqs); 603 + err = -ENOSPC; 604 + goto err_unmap; 605 + } 606 + 607 + if (pdsv->num_vqs != fw_max_vqs) { 608 + err = pds_vdpa_cmd_set_max_vq_pairs(pdsv, vq_pairs); 609 + if (err) { 610 + dev_err(dev, "Failed to set max_vq_pairs: %pe\n", 611 + ERR_PTR(err)); 612 + goto err_unmap; 613 + } 614 + } 615 + 616 + /* Set a mac, either from the user config if provided 617 + * or set a random mac if default is 00:..:00 618 + */ 619 + if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR)) { 620 + ether_addr_copy(mac, add_config->net.mac); 621 + pds_vdpa_cmd_set_mac(pdsv, mac); 622 + } else { 623 + struct virtio_net_config __iomem *vc; 624 + 625 + vc = pdsv->vdpa_aux->vd_mdev.device; 626 + memcpy_fromio(mac, vc->mac, sizeof(mac)); 627 + if (is_zero_ether_addr(mac)) { 628 + eth_random_addr(mac); 629 + dev_info(dev, "setting random mac %pM\n", mac); 630 + pds_vdpa_cmd_set_mac(pdsv, mac); 631 + } 632 + } 633 + 634 + for (i = 0; i < pdsv->num_vqs; i++) { 635 + pdsv->vqs[i].qid = i; 636 + pdsv->vqs[i].pdsv = pdsv; 637 + pdsv->vqs[i].irq = VIRTIO_MSI_NO_VECTOR; 638 + pdsv->vqs[i].notify = vp_modern_map_vq_notify(&pdsv->vdpa_aux->vd_mdev, 639 + i, &pdsv->vqs[i].notify_pa); 640 + } 641 + 642 + pdsv->vdpa_dev.mdev = &vdpa_aux->vdpa_mdev; 643 + 644 + err = pds_vdpa_register_event_handler(pdsv); 645 + if (err) { 646 + dev_err(dev, "Failed to register for PDS events: %pe\n", ERR_PTR(err)); 647 + goto err_unmap; 648 + } 649 + 650 + /* We use the _vdpa_register_device() call rather than the 651 + * vdpa_register_device() to avoid a deadlock because our 652 + * dev_add() is called with the vdpa_dev_lock already set 653 + * by vdpa_nl_cmd_dev_add_set_doit() 654 + */ 655 + err = _vdpa_register_device(&pdsv->vdpa_dev, pdsv->num_vqs); 656 + if (err) { 657 + dev_err(dev, "Failed to register to vDPA bus: %pe\n", ERR_PTR(err)); 658 + goto err_unevent; 659 + } 660 + 661 + pds_vdpa_debugfs_add_vdpadev(vdpa_aux); 662 + 663 + return 0; 664 + 665 + err_unevent: 666 + pds_vdpa_unregister_event_handler(pdsv); 667 + err_unmap: 668 + put_device(&pdsv->vdpa_dev.dev); 669 + vdpa_aux->pdsv = NULL; 670 + return err; 671 + } 672 + 673 + static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, 674 + struct vdpa_device *vdpa_dev) 675 + { 676 + struct pds_vdpa_device *pdsv = vdpa_to_pdsv(vdpa_dev); 677 + struct pds_vdpa_aux *vdpa_aux; 678 + 679 + pds_vdpa_unregister_event_handler(pdsv); 680 + 681 + vdpa_aux = container_of(mdev, struct pds_vdpa_aux, vdpa_mdev); 682 + _vdpa_unregister_device(vdpa_dev); 683 + 684 + pds_vdpa_cmd_reset(vdpa_aux->pdsv); 685 + pds_vdpa_debugfs_reset_vdpadev(vdpa_aux); 686 + 687 + vdpa_aux->pdsv = NULL; 688 + 689 + dev_info(&vdpa_aux->padev->aux_dev.dev, "Removed vdpa device\n"); 690 + } 691 + 692 + static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = { 693 + .dev_add = pds_vdpa_dev_add, 694 + .dev_del = pds_vdpa_dev_del 695 + }; 696 + 697 + int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux) 698 + { 699 + union pds_core_adminq_cmd cmd = { 700 + .vdpa_ident.opcode = PDS_VDPA_CMD_IDENT, 701 + .vdpa_ident.vf_id = cpu_to_le16(vdpa_aux->vf_id), 702 + }; 703 + union pds_core_adminq_comp comp = {}; 704 + struct vdpa_mgmt_dev *mgmt; 705 + struct pci_dev *pf_pdev; 706 + struct device *pf_dev; 707 + struct pci_dev *pdev; 708 + dma_addr_t ident_pa; 709 + struct device *dev; 710 + u16 dev_intrs; 711 + u16 max_vqs; 712 + int err; 713 + 714 + dev = &vdpa_aux->padev->aux_dev.dev; 715 + pdev = vdpa_aux->padev->vf_pdev; 716 + mgmt = &vdpa_aux->vdpa_mdev; 717 + 718 + /* Get resource info through the PF's adminq. It is a block of info, 719 + * so we need to map some memory for PF to make available to the 720 + * firmware for writing the data. 721 + */ 722 + pf_pdev = pci_physfn(vdpa_aux->padev->vf_pdev); 723 + pf_dev = &pf_pdev->dev; 724 + ident_pa = dma_map_single(pf_dev, &vdpa_aux->ident, 725 + sizeof(vdpa_aux->ident), DMA_FROM_DEVICE); 726 + if (dma_mapping_error(pf_dev, ident_pa)) { 727 + dev_err(dev, "Failed to map ident space\n"); 728 + return -ENOMEM; 729 + } 730 + 731 + cmd.vdpa_ident.ident_pa = cpu_to_le64(ident_pa); 732 + cmd.vdpa_ident.len = cpu_to_le32(sizeof(vdpa_aux->ident)); 733 + err = pds_client_adminq_cmd(vdpa_aux->padev, &cmd, 734 + sizeof(cmd.vdpa_ident), &comp, 0); 735 + dma_unmap_single(pf_dev, ident_pa, 736 + sizeof(vdpa_aux->ident), DMA_FROM_DEVICE); 737 + if (err) { 738 + dev_err(dev, "Failed to ident hw, status %d: %pe\n", 739 + comp.status, ERR_PTR(err)); 740 + return err; 741 + } 742 + 743 + max_vqs = le16_to_cpu(vdpa_aux->ident.max_vqs); 744 + dev_intrs = pci_msix_vec_count(pdev); 745 + dev_dbg(dev, "ident.max_vqs %d dev_intrs %d\n", max_vqs, dev_intrs); 746 + 747 + max_vqs = min_t(u16, dev_intrs, max_vqs); 748 + mgmt->max_supported_vqs = min_t(u16, PDS_VDPA_MAX_QUEUES, max_vqs); 749 + vdpa_aux->nintrs = mgmt->max_supported_vqs; 750 + 751 + mgmt->ops = &pds_vdpa_mgmt_dev_ops; 752 + mgmt->id_table = pds_vdpa_id_table; 753 + mgmt->device = dev; 754 + mgmt->supported_features = le64_to_cpu(vdpa_aux->ident.hw_features); 755 + mgmt->config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR); 756 + mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP); 757 + mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_FEATURES); 758 + 759 + err = pci_alloc_irq_vectors(pdev, vdpa_aux->nintrs, vdpa_aux->nintrs, 760 + PCI_IRQ_MSIX); 761 + if (err < 0) { 762 + dev_err(dev, "Couldn't get %d msix vectors: %pe\n", 763 + vdpa_aux->nintrs, ERR_PTR(err)); 764 + return err; 765 + } 766 + vdpa_aux->nintrs = err; 767 + 768 + return 0; 769 + }

+49

drivers/vdpa/pds/vdpa_dev.h

··· 1 + /* SPDX-License-Identifier: GPL-2.0-only */ 2 + /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 + 4 + #ifndef _VDPA_DEV_H_ 5 + #define _VDPA_DEV_H_ 6 + 7 + #include <linux/pci.h> 8 + #include <linux/vdpa.h> 9 + 10 + struct pds_vdpa_vq_info { 11 + bool ready; 12 + u64 desc_addr; 13 + u64 avail_addr; 14 + u64 used_addr; 15 + u32 q_len; 16 + u16 qid; 17 + int irq; 18 + char irq_name[32]; 19 + 20 + void __iomem *notify; 21 + dma_addr_t notify_pa; 22 + 23 + u64 doorbell; 24 + u16 avail_idx; 25 + u16 used_idx; 26 + 27 + struct vdpa_callback event_cb; 28 + struct pds_vdpa_device *pdsv; 29 + }; 30 + 31 + #define PDS_VDPA_MAX_QUEUES 65 32 + #define PDS_VDPA_MAX_QLEN 32768 33 + struct pds_vdpa_device { 34 + struct vdpa_device vdpa_dev; 35 + struct pds_vdpa_aux *vdpa_aux; 36 + 37 + struct pds_vdpa_vq_info vqs[PDS_VDPA_MAX_QUEUES]; 38 + u64 supported_features; /* specified device features */ 39 + u64 req_features; /* features requested by vdpa */ 40 + u8 vdpa_index; /* rsvd for future subdevice use */ 41 + u8 num_vqs; /* num vqs in use */ 42 + struct vdpa_callback config_cb; 43 + struct notifier_block nb; 44 + }; 45 + 46 + #define PDS_VDPA_PACKED_INVERT_IDX 0x8000 47 + 48 + int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux); 49 + #endif /* _VDPA_DEV_H_ */

+6

drivers/vdpa/solidrun/snet_ctrl.c

··· 16 16 SNET_CTRL_OP_DESTROY = 1, 17 17 SNET_CTRL_OP_READ_VQ_STATE, 18 18 SNET_CTRL_OP_SUSPEND, 19 + SNET_CTRL_OP_RESUME, 19 20 }; 20 21 21 22 #define SNET_CTRL_TIMEOUT 2000000 ··· 328 327 int snet_suspend_dev(struct snet *snet) 329 328 { 330 329 return snet_send_ctrl_msg(snet, SNET_CTRL_OP_SUSPEND, 0); 330 + } 331 + 332 + int snet_resume_dev(struct snet *snet) 333 + { 334 + return snet_send_ctrl_msg(snet, SNET_CTRL_OP_RESUME, 0); 331 335 }

+1 -1

drivers/vdpa/solidrun/snet_hwmon.c

··· 159 159 .read_string = snet_hwmon_read_string 160 160 }; 161 161 162 - static const struct hwmon_channel_info *snet_hwmon_info[] = { 162 + static const struct hwmon_channel_info * const snet_hwmon_info[] = { 163 163 HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT | HWMON_T_MAX | HWMON_T_CRIT | HWMON_T_LABEL, 164 164 HWMON_T_INPUT | HWMON_T_CRIT | HWMON_T_LABEL), 165 165 HWMON_CHANNEL_INFO(power, HWMON_P_INPUT | HWMON_P_LABEL),

+15

drivers/vdpa/solidrun/snet_main.c

··· 509 509 return ret; 510 510 } 511 511 512 + static int snet_resume(struct vdpa_device *vdev) 513 + { 514 + struct snet *snet = vdpa_to_snet(vdev); 515 + int ret; 516 + 517 + ret = snet_resume_dev(snet); 518 + if (ret) 519 + SNET_ERR(snet->pdev, "SNET[%u] resume failed, err: %d\n", snet->sid, ret); 520 + else 521 + SNET_DBG(snet->pdev, "Resume SNET[%u] device\n", snet->sid); 522 + 523 + return ret; 524 + } 525 + 512 526 static const struct vdpa_config_ops snet_config_ops = { 513 527 .set_vq_address = snet_set_vq_address, 514 528 .set_vq_num = snet_set_vq_num, ··· 550 536 .get_config = snet_get_config, 551 537 .set_config = snet_set_config, 552 538 .suspend = snet_suspend, 539 + .resume = snet_resume, 553 540 }; 554 541 555 542 static int psnet_open_pf_bar(struct pci_dev *pdev, struct psnet *psnet)

+1

drivers/vdpa/solidrun/snet_vdpa.h

··· 204 204 int snet_destroy_dev(struct snet *snet); 205 205 int snet_read_vq_state(struct snet *snet, u16 idx, struct vdpa_vq_state *state); 206 206 int snet_suspend_dev(struct snet *snet); 207 + int snet_resume_dev(struct snet *snet); 207 208 208 209 #endif //_SNET_VDPA_H_

+5 -1

drivers/vdpa/vdpa_user/vduse_dev.c

··· 726 726 { 727 727 struct vduse_dev *dev = vdpa_to_vduse(vdpa); 728 728 729 - cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask); 729 + if (cpu_mask) 730 + cpumask_copy(&dev->vqs[idx]->irq_affinity, cpu_mask); 731 + else 732 + cpumask_setall(&dev->vqs[idx]->irq_affinity); 733 + 730 734 return 0; 731 735 } 732 736

+5 -3

drivers/vhost/net.c

··· 546 546 endtime = busy_clock() + busyloop_timeout; 547 547 548 548 while (vhost_can_busy_poll(endtime)) { 549 - if (vhost_has_work(&net->dev)) { 549 + if (vhost_vq_has_work(vq)) { 550 550 *busyloop_intr = true; 551 551 break; 552 552 } ··· 1347 1347 VHOST_NET_PKT_WEIGHT, VHOST_NET_WEIGHT, true, 1348 1348 NULL); 1349 1349 1350 - vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, EPOLLOUT, dev); 1351 - vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, EPOLLIN, dev); 1350 + vhost_poll_init(n->poll + VHOST_NET_VQ_TX, handle_tx_net, EPOLLOUT, dev, 1351 + vqs[VHOST_NET_VQ_TX]); 1352 + vhost_poll_init(n->poll + VHOST_NET_VQ_RX, handle_rx_net, EPOLLIN, dev, 1353 + vqs[VHOST_NET_VQ_RX]); 1352 1354 1353 1355 f->private_data = n; 1354 1356 n->page_frag.page = NULL;

+61 -42

drivers/vhost/scsi.c

··· 167 167 168 168 struct vhost_scsi_virtqueue { 169 169 struct vhost_virtqueue vq; 170 + struct vhost_scsi *vs; 170 171 /* 171 172 * Reference counting for inflight reqs, used for flush operation. At 172 173 * each time, one reference tracks new commands submitted, while we ··· 182 181 struct vhost_scsi_cmd *scsi_cmds; 183 182 struct sbitmap scsi_tags; 184 183 int max_cmds; 184 + 185 + struct vhost_work completion_work; 186 + struct llist_head completion_list; 185 187 }; 186 188 187 189 struct vhost_scsi { ··· 194 190 195 191 struct vhost_dev dev; 196 192 struct vhost_scsi_virtqueue *vqs; 197 - unsigned long *compl_bitmap; 198 193 struct vhost_scsi_inflight **old_inflight; 199 - 200 - struct vhost_work vs_completion_work; /* cmd completion work item */ 201 - struct llist_head vs_completion_list; /* cmd completion queue */ 202 194 203 195 struct vhost_work vs_event_work; /* evt injection work item */ 204 196 struct llist_head vs_event_list; /* evt injection queue */ ··· 353 353 if (se_cmd->se_cmd_flags & SCF_SCSI_TMR_CDB) { 354 354 struct vhost_scsi_tmf *tmf = container_of(se_cmd, 355 355 struct vhost_scsi_tmf, se_cmd); 356 + struct vhost_virtqueue *vq = &tmf->svq->vq; 356 357 357 - vhost_work_queue(&tmf->vhost->dev, &tmf->vwork); 358 + vhost_vq_work_queue(vq, &tmf->vwork); 358 359 } else { 359 360 struct vhost_scsi_cmd *cmd = container_of(se_cmd, 360 361 struct vhost_scsi_cmd, tvc_se_cmd); 361 - struct vhost_scsi *vs = cmd->tvc_vhost; 362 + struct vhost_scsi_virtqueue *svq = container_of(cmd->tvc_vq, 363 + struct vhost_scsi_virtqueue, vq); 362 364 363 - llist_add(&cmd->tvc_completion_list, &vs->vs_completion_list); 364 - vhost_work_queue(&vs->dev, &vs->vs_completion_work); 365 + llist_add(&cmd->tvc_completion_list, &svq->completion_list); 366 + vhost_vq_work_queue(&svq->vq, &svq->completion_work); 365 367 } 366 368 } 367 369 ··· 511 509 */ 512 510 static void vhost_scsi_complete_cmd_work(struct vhost_work *work) 513 511 { 514 - struct vhost_scsi *vs = container_of(work, struct vhost_scsi, 515 - vs_completion_work); 512 + struct vhost_scsi_virtqueue *svq = container_of(work, 513 + struct vhost_scsi_virtqueue, completion_work); 516 514 struct virtio_scsi_cmd_resp v_rsp; 517 515 struct vhost_scsi_cmd *cmd, *t; 518 516 struct llist_node *llnode; 519 517 struct se_cmd *se_cmd; 520 518 struct iov_iter iov_iter; 521 - int ret, vq; 519 + bool signal = false; 520 + int ret; 522 521 523 - bitmap_zero(vs->compl_bitmap, vs->dev.nvqs); 524 - llnode = llist_del_all(&vs->vs_completion_list); 522 + llnode = llist_del_all(&svq->completion_list); 525 523 llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) { 526 524 se_cmd = &cmd->tvc_se_cmd; 527 525 ··· 541 539 cmd->tvc_in_iovs, sizeof(v_rsp)); 542 540 ret = copy_to_iter(&v_rsp, sizeof(v_rsp), &iov_iter); 543 541 if (likely(ret == sizeof(v_rsp))) { 544 - struct vhost_scsi_virtqueue *q; 542 + signal = true; 543 + 545 544 vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0); 546 - q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq); 547 - vq = q - vs->vqs; 548 - __set_bit(vq, vs->compl_bitmap); 549 545 } else 550 546 pr_err("Faulted on virtio_scsi_cmd_resp\n"); 551 547 552 548 vhost_scsi_release_cmd_res(se_cmd); 553 549 } 554 550 555 - vq = -1; 556 - while ((vq = find_next_bit(vs->compl_bitmap, vs->dev.nvqs, vq + 1)) 557 - < vs->dev.nvqs) 558 - vhost_signal(&vs->dev, &vs->vqs[vq].vq); 551 + if (signal) 552 + vhost_signal(&svq->vs->dev, &svq->vq); 559 553 } 560 554 561 555 static struct vhost_scsi_cmd * ··· 1133 1135 { 1134 1136 struct vhost_scsi_tmf *tmf = container_of(work, struct vhost_scsi_tmf, 1135 1137 vwork); 1136 - int resp_code; 1138 + struct vhost_virtqueue *ctl_vq, *vq; 1139 + int resp_code, i; 1137 1140 1138 - if (tmf->scsi_resp == TMR_FUNCTION_COMPLETE) 1141 + if (tmf->scsi_resp == TMR_FUNCTION_COMPLETE) { 1142 + /* 1143 + * Flush IO vqs that don't share a worker with the ctl to make 1144 + * sure they have sent their responses before us. 1145 + */ 1146 + ctl_vq = &tmf->vhost->vqs[VHOST_SCSI_VQ_CTL].vq; 1147 + for (i = VHOST_SCSI_VQ_IO; i < tmf->vhost->dev.nvqs; i++) { 1148 + vq = &tmf->vhost->vqs[i].vq; 1149 + 1150 + if (vhost_vq_is_setup(vq) && 1151 + vq->worker != ctl_vq->worker) 1152 + vhost_vq_flush(vq); 1153 + } 1154 + 1139 1155 resp_code = VIRTIO_SCSI_S_FUNCTION_SUCCEEDED; 1140 - else 1156 + } else { 1141 1157 resp_code = VIRTIO_SCSI_S_FUNCTION_REJECTED; 1158 + } 1142 1159 1143 1160 vhost_scsi_send_tmf_resp(tmf->vhost, &tmf->svq->vq, tmf->in_iovs, 1144 1161 tmf->vq_desc, &tmf->resp_iov, resp_code); ··· 1348 1335 } 1349 1336 1350 1337 static void 1351 - vhost_scsi_send_evt(struct vhost_scsi *vs, 1352 - struct vhost_scsi_tpg *tpg, 1353 - struct se_lun *lun, 1354 - u32 event, 1355 - u32 reason) 1338 + vhost_scsi_send_evt(struct vhost_scsi *vs, struct vhost_virtqueue *vq, 1339 + struct vhost_scsi_tpg *tpg, struct se_lun *lun, 1340 + u32 event, u32 reason) 1356 1341 { 1357 1342 struct vhost_scsi_evt *evt; 1358 1343 ··· 1372 1361 } 1373 1362 1374 1363 llist_add(&evt->list, &vs->vs_event_list); 1375 - vhost_work_queue(&vs->dev, &vs->vs_event_work); 1364 + vhost_vq_work_queue(vq, &vs->vs_event_work); 1376 1365 } 1377 1366 1378 1367 static void vhost_scsi_evt_handle_kick(struct vhost_work *work) ··· 1386 1375 goto out; 1387 1376 1388 1377 if (vs->vs_events_missed) 1389 - vhost_scsi_send_evt(vs, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT, 0); 1378 + vhost_scsi_send_evt(vs, vq, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT, 1379 + 0); 1390 1380 out: 1391 1381 mutex_unlock(&vq->mutex); 1392 1382 } ··· 1782 1770 1783 1771 static int vhost_scsi_open(struct inode *inode, struct file *f) 1784 1772 { 1773 + struct vhost_scsi_virtqueue *svq; 1785 1774 struct vhost_scsi *vs; 1786 1775 struct vhost_virtqueue **vqs; 1787 1776 int r = -ENOMEM, i, nvqs = vhost_scsi_max_io_vqs; ··· 1801 1788 } 1802 1789 nvqs += VHOST_SCSI_VQ_IO; 1803 1790 1804 - vs->compl_bitmap = bitmap_alloc(nvqs, GFP_KERNEL); 1805 - if (!vs->compl_bitmap) 1806 - goto err_compl_bitmap; 1807 - 1808 1791 vs->old_inflight = kmalloc_array(nvqs, sizeof(*vs->old_inflight), 1809 1792 GFP_KERNEL | __GFP_ZERO); 1810 1793 if (!vs->old_inflight) ··· 1815 1806 if (!vqs) 1816 1807 goto err_local_vqs; 1817 1808 1818 - vhost_work_init(&vs->vs_completion_work, vhost_scsi_complete_cmd_work); 1819 1809 vhost_work_init(&vs->vs_event_work, vhost_scsi_evt_work); 1820 1810 1821 1811 vs->vs_events_nr = 0; ··· 1825 1817 vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick; 1826 1818 vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick; 1827 1819 for (i = VHOST_SCSI_VQ_IO; i < nvqs; i++) { 1828 - vqs[i] = &vs->vqs[i].vq; 1829 - vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick; 1820 + svq = &vs->vqs[i]; 1821 + 1822 + vqs[i] = &svq->vq; 1823 + svq->vs = vs; 1824 + init_llist_head(&svq->completion_list); 1825 + vhost_work_init(&svq->completion_work, 1826 + vhost_scsi_complete_cmd_work); 1827 + svq->vq.handle_kick = vhost_scsi_handle_kick; 1830 1828 } 1831 1829 vhost_dev_init(&vs->dev, vqs, nvqs, UIO_MAXIOV, 1832 1830 VHOST_SCSI_WEIGHT, 0, true, NULL); ··· 1847 1833 err_vqs: 1848 1834 kfree(vs->old_inflight); 1849 1835 err_inflight: 1850 - bitmap_free(vs->compl_bitmap); 1851 - err_compl_bitmap: 1852 1836 kvfree(vs); 1853 1837 err_vs: 1854 1838 return r; ··· 1866 1854 kfree(vs->dev.vqs); 1867 1855 kfree(vs->vqs); 1868 1856 kfree(vs->old_inflight); 1869 - bitmap_free(vs->compl_bitmap); 1870 1857 kvfree(vs); 1871 1858 return 0; 1872 1859 } ··· 1927 1916 if (copy_from_user(&features, featurep, sizeof features)) 1928 1917 return -EFAULT; 1929 1918 return vhost_scsi_set_features(vs, features); 1919 + case VHOST_NEW_WORKER: 1920 + case VHOST_FREE_WORKER: 1921 + case VHOST_ATTACH_VRING_WORKER: 1922 + case VHOST_GET_VRING_WORKER: 1923 + mutex_lock(&vs->dev.mutex); 1924 + r = vhost_worker_ioctl(&vs->dev, ioctl, argp); 1925 + mutex_unlock(&vs->dev.mutex); 1926 + return r; 1930 1927 default: 1931 1928 mutex_lock(&vs->dev.mutex); 1932 1929 r = vhost_dev_ioctl(&vs->dev, ioctl, argp); ··· 2014 1995 goto unlock; 2015 1996 2016 1997 if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG)) 2017 - vhost_scsi_send_evt(vs, tpg, lun, 1998 + vhost_scsi_send_evt(vs, vq, tpg, lun, 2018 1999 VIRTIO_SCSI_T_TRANSPORT_RESET, reason); 2019 2000 unlock: 2020 2001 mutex_unlock(&vq->mutex);

+367 -58

drivers/vhost/vhost.c

··· 187 187 188 188 /* Init poll structure */ 189 189 void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, 190 - __poll_t mask, struct vhost_dev *dev) 190 + __poll_t mask, struct vhost_dev *dev, 191 + struct vhost_virtqueue *vq) 191 192 { 192 193 init_waitqueue_func_entry(&poll->wait, vhost_poll_wakeup); 193 194 init_poll_funcptr(&poll->table, vhost_poll_func); 194 195 poll->mask = mask; 195 196 poll->dev = dev; 196 197 poll->wqh = NULL; 198 + poll->vq = vq; 197 199 198 200 vhost_work_init(&poll->work, fn); 199 201 } ··· 233 231 } 234 232 EXPORT_SYMBOL_GPL(vhost_poll_stop); 235 233 236 - void vhost_dev_flush(struct vhost_dev *dev) 234 + static void vhost_worker_queue(struct vhost_worker *worker, 235 + struct vhost_work *work) 237 236 { 238 - struct vhost_flush_struct flush; 239 - 240 - if (dev->worker.vtsk) { 241 - init_completion(&flush.wait_event); 242 - vhost_work_init(&flush.work, vhost_flush_work); 243 - 244 - vhost_work_queue(dev, &flush.work); 245 - wait_for_completion(&flush.wait_event); 246 - } 247 - } 248 - EXPORT_SYMBOL_GPL(vhost_dev_flush); 249 - 250 - void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work) 251 - { 252 - if (!dev->worker.vtsk) 253 - return; 254 - 255 237 if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) { 256 238 /* We can only add the work to the list after we're 257 239 * sure it was not in the list. 258 240 * test_and_set_bit() implies a memory barrier. 259 241 */ 260 - llist_add(&work->node, &dev->worker.work_list); 261 - vhost_task_wake(dev->worker.vtsk); 242 + llist_add(&work->node, &worker->work_list); 243 + vhost_task_wake(worker->vtsk); 262 244 } 263 245 } 264 - EXPORT_SYMBOL_GPL(vhost_work_queue); 246 + 247 + bool vhost_vq_work_queue(struct vhost_virtqueue *vq, struct vhost_work *work) 248 + { 249 + struct vhost_worker *worker; 250 + bool queued = false; 251 + 252 + rcu_read_lock(); 253 + worker = rcu_dereference(vq->worker); 254 + if (worker) { 255 + queued = true; 256 + vhost_worker_queue(worker, work); 257 + } 258 + rcu_read_unlock(); 259 + 260 + return queued; 261 + } 262 + EXPORT_SYMBOL_GPL(vhost_vq_work_queue); 263 + 264 + void vhost_vq_flush(struct vhost_virtqueue *vq) 265 + { 266 + struct vhost_flush_struct flush; 267 + 268 + init_completion(&flush.wait_event); 269 + vhost_work_init(&flush.work, vhost_flush_work); 270 + 271 + if (vhost_vq_work_queue(vq, &flush.work)) 272 + wait_for_completion(&flush.wait_event); 273 + } 274 + EXPORT_SYMBOL_GPL(vhost_vq_flush); 275 + 276 + /** 277 + * vhost_worker_flush - flush a worker 278 + * @worker: worker to flush 279 + * 280 + * This does not use RCU to protect the worker, so the device or worker 281 + * mutex must be held. 282 + */ 283 + static void vhost_worker_flush(struct vhost_worker *worker) 284 + { 285 + struct vhost_flush_struct flush; 286 + 287 + init_completion(&flush.wait_event); 288 + vhost_work_init(&flush.work, vhost_flush_work); 289 + 290 + vhost_worker_queue(worker, &flush.work); 291 + wait_for_completion(&flush.wait_event); 292 + } 293 + 294 + void vhost_dev_flush(struct vhost_dev *dev) 295 + { 296 + struct vhost_worker *worker; 297 + unsigned long i; 298 + 299 + xa_for_each(&dev->worker_xa, i, worker) { 300 + mutex_lock(&worker->mutex); 301 + if (!worker->attachment_cnt) { 302 + mutex_unlock(&worker->mutex); 303 + continue; 304 + } 305 + vhost_worker_flush(worker); 306 + mutex_unlock(&worker->mutex); 307 + } 308 + } 309 + EXPORT_SYMBOL_GPL(vhost_dev_flush); 265 310 266 311 /* A lockless hint for busy polling code to exit the loop */ 267 - bool vhost_has_work(struct vhost_dev *dev) 312 + bool vhost_vq_has_work(struct vhost_virtqueue *vq) 268 313 { 269 - return !llist_empty(&dev->worker.work_list); 314 + struct vhost_worker *worker; 315 + bool has_work = false; 316 + 317 + rcu_read_lock(); 318 + worker = rcu_dereference(vq->worker); 319 + if (worker && !llist_empty(&worker->work_list)) 320 + has_work = true; 321 + rcu_read_unlock(); 322 + 323 + return has_work; 270 324 } 271 - EXPORT_SYMBOL_GPL(vhost_has_work); 325 + EXPORT_SYMBOL_GPL(vhost_vq_has_work); 272 326 273 327 void vhost_poll_queue(struct vhost_poll *poll) 274 328 { 275 - vhost_work_queue(poll->dev, &poll->work); 329 + vhost_vq_work_queue(poll->vq, &poll->work); 276 330 } 277 331 EXPORT_SYMBOL_GPL(vhost_poll_queue); 278 332 ··· 387 329 vq->busyloop_timeout = 0; 388 330 vq->umem = NULL; 389 331 vq->iotlb = NULL; 332 + rcu_assign_pointer(vq->worker, NULL); 390 333 vhost_vring_call_reset(&vq->call_ctx); 391 334 __vhost_vq_meta_reset(vq); 392 335 } ··· 517 458 dev->umem = NULL; 518 459 dev->iotlb = NULL; 519 460 dev->mm = NULL; 520 - memset(&dev->worker, 0, sizeof(dev->worker)); 521 - init_llist_head(&dev->worker.work_list); 522 461 dev->iov_limit = iov_limit; 523 462 dev->weight = weight; 524 463 dev->byte_weight = byte_weight; ··· 526 469 INIT_LIST_HEAD(&dev->read_list); 527 470 INIT_LIST_HEAD(&dev->pending_list); 528 471 spin_lock_init(&dev->iotlb_lock); 529 - 472 + xa_init_flags(&dev->worker_xa, XA_FLAGS_ALLOC); 530 473 531 474 for (i = 0; i < dev->nvqs; ++i) { 532 475 vq = dev->vqs[i]; ··· 538 481 vhost_vq_reset(dev, vq); 539 482 if (vq->handle_kick) 540 483 vhost_poll_init(&vq->poll, vq->handle_kick, 541 - EPOLLIN, dev); 484 + EPOLLIN, dev, vq); 542 485 } 543 486 } 544 487 EXPORT_SYMBOL_GPL(vhost_dev_init); ··· 588 531 dev->mm = NULL; 589 532 } 590 533 591 - static void vhost_worker_free(struct vhost_dev *dev) 534 + static void vhost_worker_destroy(struct vhost_dev *dev, 535 + struct vhost_worker *worker) 592 536 { 593 - if (!dev->worker.vtsk) 537 + if (!worker) 594 538 return; 595 539 596 - WARN_ON(!llist_empty(&dev->worker.work_list)); 597 - vhost_task_stop(dev->worker.vtsk); 598 - dev->worker.kcov_handle = 0; 599 - dev->worker.vtsk = NULL; 540 + WARN_ON(!llist_empty(&worker->work_list)); 541 + xa_erase(&dev->worker_xa, worker->id); 542 + vhost_task_stop(worker->vtsk); 543 + kfree(worker); 600 544 } 601 545 602 - static int vhost_worker_create(struct vhost_dev *dev) 546 + static void vhost_workers_free(struct vhost_dev *dev) 603 547 { 548 + struct vhost_worker *worker; 549 + unsigned long i; 550 + 551 + if (!dev->use_worker) 552 + return; 553 + 554 + for (i = 0; i < dev->nvqs; i++) 555 + rcu_assign_pointer(dev->vqs[i]->worker, NULL); 556 + /* 557 + * Free the default worker we created and cleanup workers userspace 558 + * created but couldn't clean up (it forgot or crashed). 559 + */ 560 + xa_for_each(&dev->worker_xa, i, worker) 561 + vhost_worker_destroy(dev, worker); 562 + xa_destroy(&dev->worker_xa); 563 + } 564 + 565 + static struct vhost_worker *vhost_worker_create(struct vhost_dev *dev) 566 + { 567 + struct vhost_worker *worker; 604 568 struct vhost_task *vtsk; 605 569 char name[TASK_COMM_LEN]; 570 + int ret; 571 + u32 id; 572 + 573 + worker = kzalloc(sizeof(*worker), GFP_KERNEL_ACCOUNT); 574 + if (!worker) 575 + return NULL; 606 576 607 577 snprintf(name, sizeof(name), "vhost-%d", current->pid); 608 578 609 - vtsk = vhost_task_create(vhost_worker, &dev->worker, name); 579 + vtsk = vhost_task_create(vhost_worker, worker, name); 610 580 if (!vtsk) 611 - return -ENOMEM; 581 + goto free_worker; 612 582 613 - dev->worker.kcov_handle = kcov_common_handle(); 614 - dev->worker.vtsk = vtsk; 583 + mutex_init(&worker->mutex); 584 + init_llist_head(&worker->work_list); 585 + worker->kcov_handle = kcov_common_handle(); 586 + worker->vtsk = vtsk; 587 + 615 588 vhost_task_start(vtsk); 589 + 590 + ret = xa_alloc(&dev->worker_xa, &id, worker, xa_limit_32b, GFP_KERNEL); 591 + if (ret < 0) 592 + goto stop_worker; 593 + worker->id = id; 594 + 595 + return worker; 596 + 597 + stop_worker: 598 + vhost_task_stop(vtsk); 599 + free_worker: 600 + kfree(worker); 601 + return NULL; 602 + } 603 + 604 + /* Caller must have device mutex */ 605 + static void __vhost_vq_attach_worker(struct vhost_virtqueue *vq, 606 + struct vhost_worker *worker) 607 + { 608 + struct vhost_worker *old_worker; 609 + 610 + old_worker = rcu_dereference_check(vq->worker, 611 + lockdep_is_held(&vq->dev->mutex)); 612 + 613 + mutex_lock(&worker->mutex); 614 + worker->attachment_cnt++; 615 + mutex_unlock(&worker->mutex); 616 + rcu_assign_pointer(vq->worker, worker); 617 + 618 + if (!old_worker) 619 + return; 620 + /* 621 + * Take the worker mutex to make sure we see the work queued from 622 + * device wide flushes which doesn't use RCU for execution. 623 + */ 624 + mutex_lock(&old_worker->mutex); 625 + old_worker->attachment_cnt--; 626 + /* 627 + * We don't want to call synchronize_rcu for every vq during setup 628 + * because it will slow down VM startup. If we haven't done 629 + * VHOST_SET_VRING_KICK and not done the driver specific 630 + * SET_ENDPOINT/RUNNUNG then we can skip the sync since there will 631 + * not be any works queued for scsi and net. 632 + */ 633 + mutex_lock(&vq->mutex); 634 + if (!vhost_vq_get_backend(vq) && !vq->kick) { 635 + mutex_unlock(&vq->mutex); 636 + mutex_unlock(&old_worker->mutex); 637 + /* 638 + * vsock can queue anytime after VHOST_VSOCK_SET_GUEST_CID. 639 + * Warn if it adds support for multiple workers but forgets to 640 + * handle the early queueing case. 641 + */ 642 + WARN_ON(!old_worker->attachment_cnt && 643 + !llist_empty(&old_worker->work_list)); 644 + return; 645 + } 646 + mutex_unlock(&vq->mutex); 647 + 648 + /* Make sure new vq queue/flush/poll calls see the new worker */ 649 + synchronize_rcu(); 650 + /* Make sure whatever was queued gets run */ 651 + vhost_worker_flush(old_worker); 652 + mutex_unlock(&old_worker->mutex); 653 + } 654 + 655 + /* Caller must have device mutex */ 656 + static int vhost_vq_attach_worker(struct vhost_virtqueue *vq, 657 + struct vhost_vring_worker *info) 658 + { 659 + unsigned long index = info->worker_id; 660 + struct vhost_dev *dev = vq->dev; 661 + struct vhost_worker *worker; 662 + 663 + if (!dev->use_worker) 664 + return -EINVAL; 665 + 666 + worker = xa_find(&dev->worker_xa, &index, UINT_MAX, XA_PRESENT); 667 + if (!worker || worker->id != info->worker_id) 668 + return -ENODEV; 669 + 670 + __vhost_vq_attach_worker(vq, worker); 616 671 return 0; 617 672 } 673 + 674 + /* Caller must have device mutex */ 675 + static int vhost_new_worker(struct vhost_dev *dev, 676 + struct vhost_worker_state *info) 677 + { 678 + struct vhost_worker *worker; 679 + 680 + worker = vhost_worker_create(dev); 681 + if (!worker) 682 + return -ENOMEM; 683 + 684 + info->worker_id = worker->id; 685 + return 0; 686 + } 687 + 688 + /* Caller must have device mutex */ 689 + static int vhost_free_worker(struct vhost_dev *dev, 690 + struct vhost_worker_state *info) 691 + { 692 + unsigned long index = info->worker_id; 693 + struct vhost_worker *worker; 694 + 695 + worker = xa_find(&dev->worker_xa, &index, UINT_MAX, XA_PRESENT); 696 + if (!worker || worker->id != info->worker_id) 697 + return -ENODEV; 698 + 699 + mutex_lock(&worker->mutex); 700 + if (worker->attachment_cnt) { 701 + mutex_unlock(&worker->mutex); 702 + return -EBUSY; 703 + } 704 + mutex_unlock(&worker->mutex); 705 + 706 + vhost_worker_destroy(dev, worker); 707 + return 0; 708 + } 709 + 710 + static int vhost_get_vq_from_user(struct vhost_dev *dev, void __user *argp, 711 + struct vhost_virtqueue **vq, u32 *id) 712 + { 713 + u32 __user *idxp = argp; 714 + u32 idx; 715 + long r; 716 + 717 + r = get_user(idx, idxp); 718 + if (r < 0) 719 + return r; 720 + 721 + if (idx >= dev->nvqs) 722 + return -ENOBUFS; 723 + 724 + idx = array_index_nospec(idx, dev->nvqs); 725 + 726 + *vq = dev->vqs[idx]; 727 + *id = idx; 728 + return 0; 729 + } 730 + 731 + /* Caller must have device mutex */ 732 + long vhost_worker_ioctl(struct vhost_dev *dev, unsigned int ioctl, 733 + void __user *argp) 734 + { 735 + struct vhost_vring_worker ring_worker; 736 + struct vhost_worker_state state; 737 + struct vhost_worker *worker; 738 + struct vhost_virtqueue *vq; 739 + long ret; 740 + u32 idx; 741 + 742 + if (!dev->use_worker) 743 + return -EINVAL; 744 + 745 + if (!vhost_dev_has_owner(dev)) 746 + return -EINVAL; 747 + 748 + ret = vhost_dev_check_owner(dev); 749 + if (ret) 750 + return ret; 751 + 752 + switch (ioctl) { 753 + /* dev worker ioctls */ 754 + case VHOST_NEW_WORKER: 755 + ret = vhost_new_worker(dev, &state); 756 + if (!ret && copy_to_user(argp, &state, sizeof(state))) 757 + ret = -EFAULT; 758 + return ret; 759 + case VHOST_FREE_WORKER: 760 + if (copy_from_user(&state, argp, sizeof(state))) 761 + return -EFAULT; 762 + return vhost_free_worker(dev, &state); 763 + /* vring worker ioctls */ 764 + case VHOST_ATTACH_VRING_WORKER: 765 + case VHOST_GET_VRING_WORKER: 766 + break; 767 + default: 768 + return -ENOIOCTLCMD; 769 + } 770 + 771 + ret = vhost_get_vq_from_user(dev, argp, &vq, &idx); 772 + if (ret) 773 + return ret; 774 + 775 + switch (ioctl) { 776 + case VHOST_ATTACH_VRING_WORKER: 777 + if (copy_from_user(&ring_worker, argp, sizeof(ring_worker))) { 778 + ret = -EFAULT; 779 + break; 780 + } 781 + 782 + ret = vhost_vq_attach_worker(vq, &ring_worker); 783 + break; 784 + case VHOST_GET_VRING_WORKER: 785 + worker = rcu_dereference_check(vq->worker, 786 + lockdep_is_held(&dev->mutex)); 787 + if (!worker) { 788 + ret = -EINVAL; 789 + break; 790 + } 791 + 792 + ring_worker.index = idx; 793 + ring_worker.worker_id = worker->id; 794 + 795 + if (copy_to_user(argp, &ring_worker, sizeof(ring_worker))) 796 + ret = -EFAULT; 797 + break; 798 + default: 799 + ret = -ENOIOCTLCMD; 800 + break; 801 + } 802 + 803 + return ret; 804 + } 805 + EXPORT_SYMBOL_GPL(vhost_worker_ioctl); 618 806 619 807 /* Caller should have device mutex */ 620 808 long vhost_dev_set_owner(struct vhost_dev *dev) 621 809 { 622 - int err; 810 + struct vhost_worker *worker; 811 + int err, i; 623 812 624 813 /* Is there an owner already? */ 625 814 if (vhost_dev_has_owner(dev)) { ··· 875 572 876 573 vhost_attach_mm(dev); 877 574 878 - if (dev->use_worker) { 879 - err = vhost_worker_create(dev); 880 - if (err) 881 - goto err_worker; 882 - } 883 - 884 575 err = vhost_dev_alloc_iovecs(dev); 885 576 if (err) 886 577 goto err_iovecs; 887 578 579 + if (dev->use_worker) { 580 + /* 581 + * This should be done last, because vsock can queue work 582 + * before VHOST_SET_OWNER so it simplifies the failure path 583 + * below since we don't have to worry about vsock queueing 584 + * while we free the worker. 585 + */ 586 + worker = vhost_worker_create(dev); 587 + if (!worker) { 588 + err = -ENOMEM; 589 + goto err_worker; 590 + } 591 + 592 + for (i = 0; i < dev->nvqs; i++) 593 + __vhost_vq_attach_worker(dev->vqs[i], worker); 594 + } 595 + 888 596 return 0; 889 - err_iovecs: 890 - vhost_worker_free(dev); 597 + 891 598 err_worker: 599 + vhost_dev_free_iovecs(dev); 600 + err_iovecs: 892 601 vhost_detach_mm(dev); 893 602 err_mm: 894 603 return err; ··· 992 677 dev->iotlb = NULL; 993 678 vhost_clear_msg(dev); 994 679 wake_up_interruptible_poll(&dev->wait, EPOLLIN | EPOLLRDNORM); 995 - vhost_worker_free(dev); 680 + vhost_workers_free(dev); 996 681 vhost_detach_mm(dev); 997 682 } 998 683 EXPORT_SYMBOL_GPL(vhost_dev_cleanup); ··· 1880 1565 struct file *eventfp, *filep = NULL; 1881 1566 bool pollstart = false, pollstop = false; 1882 1567 struct eventfd_ctx *ctx = NULL; 1883 - u32 __user *idxp = argp; 1884 1568 struct vhost_virtqueue *vq; 1885 1569 struct vhost_vring_state s; 1886 1570 struct vhost_vring_file f; 1887 1571 u32 idx; 1888 1572 long r; 1889 1573 1890 - r = get_user(idx, idxp); 1574 + r = vhost_get_vq_from_user(d, argp, &vq, &idx); 1891 1575 if (r < 0) 1892 1576 return r; 1893 - if (idx >= d->nvqs) 1894 - return -ENOBUFS; 1895 - 1896 - idx = array_index_nospec(idx, d->nvqs); 1897 - vq = d->vqs[idx]; 1898 1577 1899 1578 if (ioctl == VHOST_SET_VRING_NUM || 1900 1579 ioctl == VHOST_SET_VRING_ADDR) {

+17 -7

drivers/vhost/vhost.h

··· 28 28 29 29 struct vhost_worker { 30 30 struct vhost_task *vtsk; 31 + /* Used to serialize device wide flushing with worker swapping. */ 32 + struct mutex mutex; 31 33 struct llist_head work_list; 32 34 u64 kcov_handle; 35 + u32 id; 36 + int attachment_cnt; 33 37 }; 34 38 35 39 /* Poll a file (eventfd or socket) */ ··· 45 41 struct vhost_work work; 46 42 __poll_t mask; 47 43 struct vhost_dev *dev; 44 + struct vhost_virtqueue *vq; 48 45 }; 49 46 50 - void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn); 51 - void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work); 52 - bool vhost_has_work(struct vhost_dev *dev); 53 - 54 47 void vhost_poll_init(struct vhost_poll *poll, vhost_work_fn_t fn, 55 - __poll_t mask, struct vhost_dev *dev); 48 + __poll_t mask, struct vhost_dev *dev, 49 + struct vhost_virtqueue *vq); 56 50 int vhost_poll_start(struct vhost_poll *poll, struct file *file); 57 51 void vhost_poll_stop(struct vhost_poll *poll); 58 52 void vhost_poll_queue(struct vhost_poll *poll); 53 + 54 + void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn); 59 55 void vhost_dev_flush(struct vhost_dev *dev); 60 56 61 57 struct vhost_log { ··· 78 74 /* The virtqueue structure describes a queue attached to a device. */ 79 75 struct vhost_virtqueue { 80 76 struct vhost_dev *dev; 77 + struct vhost_worker __rcu *worker; 81 78 82 79 /* The actual ring of buffers. */ 83 80 struct mutex mutex; ··· 163 158 struct vhost_virtqueue **vqs; 164 159 int nvqs; 165 160 struct eventfd_ctx *log_ctx; 166 - struct vhost_worker worker; 167 161 struct vhost_iotlb *umem; 168 162 struct vhost_iotlb *iotlb; 169 163 spinlock_t iotlb_lock; ··· 172 168 int iov_limit; 173 169 int weight; 174 170 int byte_weight; 171 + struct xarray worker_xa; 175 172 bool use_worker; 176 173 int (*msg_handler)(struct vhost_dev *dev, u32 asid, 177 174 struct vhost_iotlb_msg *msg); ··· 193 188 void vhost_dev_stop(struct vhost_dev *); 194 189 long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp); 195 190 long vhost_vring_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp); 191 + long vhost_worker_ioctl(struct vhost_dev *dev, unsigned int ioctl, 192 + void __user *argp); 196 193 bool vhost_vq_access_ok(struct vhost_virtqueue *vq); 197 194 bool vhost_log_access_ok(struct vhost_dev *); 198 195 void vhost_clear_msg(struct vhost_dev *dev); 199 196 200 197 int vhost_get_vq_desc(struct vhost_virtqueue *, 201 - struct iovec iov[], unsigned int iov_count, 198 + struct iovec iov[], unsigned int iov_size, 202 199 unsigned int *out_num, unsigned int *in_num, 203 200 struct vhost_log *log, unsigned int *log_num); 204 201 void vhost_discard_vq_desc(struct vhost_virtqueue *, int n); 205 202 203 + void vhost_vq_flush(struct vhost_virtqueue *vq); 204 + bool vhost_vq_work_queue(struct vhost_virtqueue *vq, struct vhost_work *work); 205 + bool vhost_vq_has_work(struct vhost_virtqueue *vq); 206 206 bool vhost_vq_is_setup(struct vhost_virtqueue *vq); 207 207 int vhost_vq_init_access(struct vhost_virtqueue *); 208 208 int vhost_add_used(struct vhost_virtqueue *, unsigned int head, int len);

+2 -2

drivers/vhost/vsock.c

··· 285 285 atomic_inc(&vsock->queued_replies); 286 286 287 287 virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb); 288 - vhost_work_queue(&vsock->dev, &vsock->send_pkt_work); 288 + vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work); 289 289 290 290 rcu_read_unlock(); 291 291 return len; ··· 583 583 /* Some packets may have been queued before the device was started, 584 584 * let's kick the send worker to send them. 585 585 */ 586 - vhost_work_queue(&vsock->dev, &vsock->send_pkt_work); 586 + vhost_vq_work_queue(&vsock->vqs[VSOCK_VQ_RX], &vsock->send_pkt_work); 587 587 588 588 mutex_unlock(&vsock->dev.mutex); 589 589 return 0;

+4 -3

drivers/virtio/virtio_pci_common.h

··· 45 45 struct virtio_pci_device { 46 46 struct virtio_device vdev; 47 47 struct pci_dev *pci_dev; 48 - struct virtio_pci_legacy_device ldev; 49 - struct virtio_pci_modern_device mdev; 50 - 48 + union { 49 + struct virtio_pci_legacy_device ldev; 50 + struct virtio_pci_modern_device mdev; 51 + }; 51 52 bool is_legacy; 52 53 53 54 /* Where to read and clear interrupt */

+21 -12

drivers/virtio/virtio_pci_modern_dev.c

··· 218 218 int err, common, isr, notify, device; 219 219 u32 notify_length; 220 220 u32 notify_offset; 221 + int devid; 221 222 222 223 check_offsets(); 223 224 224 - /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */ 225 - if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f) 226 - return -ENODEV; 227 - 228 - if (pci_dev->device < 0x1040) { 229 - /* Transitional devices: use the PCI subsystem device id as 230 - * virtio device id, same as legacy driver always did. 231 - */ 232 - mdev->id.device = pci_dev->subsystem_device; 225 + if (mdev->device_id_check) { 226 + devid = mdev->device_id_check(pci_dev); 227 + if (devid < 0) 228 + return devid; 229 + mdev->id.device = devid; 233 230 } else { 234 - /* Modern devices: simply use PCI device id, but start from 0x1040. */ 235 - mdev->id.device = pci_dev->device - 0x1040; 231 + /* We only own devices >= 0x1000 and <= 0x107f: leave the rest. */ 232 + if (pci_dev->device < 0x1000 || pci_dev->device > 0x107f) 233 + return -ENODEV; 234 + 235 + if (pci_dev->device < 0x1040) { 236 + /* Transitional devices: use the PCI subsystem device id as 237 + * virtio device id, same as legacy driver always did. 238 + */ 239 + mdev->id.device = pci_dev->subsystem_device; 240 + } else { 241 + /* Modern devices: simply use PCI device id, but start from 0x1040. */ 242 + mdev->id.device = pci_dev->device - 0x1040; 243 + } 236 244 } 237 245 mdev->id.vendor = pci_dev->subsystem_vendor; 238 246 ··· 268 260 return -EINVAL; 269 261 } 270 262 271 - err = dma_set_mask_and_coherent(&pci_dev->dev, DMA_BIT_MASK(64)); 263 + err = dma_set_mask_and_coherent(&pci_dev->dev, 264 + mdev->dma_mask ? : DMA_BIT_MASK(64)); 272 265 if (err) 273 266 err = dma_set_mask_and_coherent(&pci_dev->dev, 274 267 DMA_BIT_MASK(32));

+3 -1

drivers/virtio/virtio_vdpa.c

··· 385 385 err = PTR_ERR(vqs[i]); 386 386 goto err_setup_vq; 387 387 } 388 - ops->set_vq_affinity(vdpa, i, &masks[i]); 388 + 389 + if (ops->set_vq_affinity) 390 + ops->set_vq_affinity(vdpa, i, &masks[i]); 389 391 } 390 392 391 393 cb.callback = virtio_vdpa_config_cb;

+247

include/linux/pds/pds_adminq.h

··· 222 222 PDS_CORE_LIF_TYPE_DEFAULT = 0, 223 223 }; 224 224 225 + #define PDS_CORE_IFNAMSIZ 16 226 + 227 + /** 228 + * enum pds_core_logical_qtype - Logical Queue Types 229 + * @PDS_CORE_QTYPE_ADMINQ: Administrative Queue 230 + * @PDS_CORE_QTYPE_NOTIFYQ: Notify Queue 231 + * @PDS_CORE_QTYPE_RXQ: Receive Queue 232 + * @PDS_CORE_QTYPE_TXQ: Transmit Queue 233 + * @PDS_CORE_QTYPE_EQ: Event Queue 234 + * @PDS_CORE_QTYPE_MAX: Max queue type supported 235 + */ 236 + enum pds_core_logical_qtype { 237 + PDS_CORE_QTYPE_ADMINQ = 0, 238 + PDS_CORE_QTYPE_NOTIFYQ = 1, 239 + PDS_CORE_QTYPE_RXQ = 2, 240 + PDS_CORE_QTYPE_TXQ = 3, 241 + PDS_CORE_QTYPE_EQ = 4, 242 + 243 + PDS_CORE_QTYPE_MAX = 16 /* don't change - used in struct size */ 244 + }; 245 + 225 246 /** 226 247 * union pds_core_lif_config - LIF configuration 227 248 * @state: LIF state (enum pds_core_lif_state) ··· 605 584 u8 color; 606 585 }; 607 586 587 + /* 588 + * enum pds_vdpa_cmd_opcode - vDPA Device commands 589 + */ 590 + enum pds_vdpa_cmd_opcode { 591 + PDS_VDPA_CMD_INIT = 48, 592 + PDS_VDPA_CMD_IDENT = 49, 593 + PDS_VDPA_CMD_RESET = 51, 594 + PDS_VDPA_CMD_VQ_RESET = 52, 595 + PDS_VDPA_CMD_VQ_INIT = 53, 596 + PDS_VDPA_CMD_STATUS_UPDATE = 54, 597 + PDS_VDPA_CMD_SET_FEATURES = 55, 598 + PDS_VDPA_CMD_SET_ATTR = 56, 599 + }; 600 + 601 + /** 602 + * struct pds_vdpa_cmd - generic command 603 + * @opcode: Opcode 604 + * @vdpa_index: Index for vdpa subdevice 605 + * @vf_id: VF id 606 + */ 607 + struct pds_vdpa_cmd { 608 + u8 opcode; 609 + u8 vdpa_index; 610 + __le16 vf_id; 611 + }; 612 + 613 + /** 614 + * struct pds_vdpa_init_cmd - INIT command 615 + * @opcode: Opcode PDS_VDPA_CMD_INIT 616 + * @vdpa_index: Index for vdpa subdevice 617 + * @vf_id: VF id 618 + */ 619 + struct pds_vdpa_init_cmd { 620 + u8 opcode; 621 + u8 vdpa_index; 622 + __le16 vf_id; 623 + }; 624 + 625 + /** 626 + * struct pds_vdpa_ident - vDPA identification data 627 + * @hw_features: vDPA features supported by device 628 + * @max_vqs: max queues available (2 queues for a single queuepair) 629 + * @max_qlen: log(2) of maximum number of descriptors 630 + * @min_qlen: log(2) of minimum number of descriptors 631 + * 632 + * This struct is used in a DMA block that is set up for the PDS_VDPA_CMD_IDENT 633 + * transaction. Set up the DMA block and send the address in the IDENT cmd 634 + * data, the DSC will write the ident information, then we can remove the DMA 635 + * block after reading the answer. If the completion status is 0, then there 636 + * is valid information, else there was an error and the data should be invalid. 637 + */ 638 + struct pds_vdpa_ident { 639 + __le64 hw_features; 640 + __le16 max_vqs; 641 + __le16 max_qlen; 642 + __le16 min_qlen; 643 + }; 644 + 645 + /** 646 + * struct pds_vdpa_ident_cmd - IDENT command 647 + * @opcode: Opcode PDS_VDPA_CMD_IDENT 648 + * @rsvd: Word boundary padding 649 + * @vf_id: VF id 650 + * @len: length of ident info DMA space 651 + * @ident_pa: address for DMA of ident info (struct pds_vdpa_ident) 652 + * only used for this transaction, then forgotten by DSC 653 + */ 654 + struct pds_vdpa_ident_cmd { 655 + u8 opcode; 656 + u8 rsvd; 657 + __le16 vf_id; 658 + __le32 len; 659 + __le64 ident_pa; 660 + }; 661 + 662 + /** 663 + * struct pds_vdpa_status_cmd - STATUS_UPDATE command 664 + * @opcode: Opcode PDS_VDPA_CMD_STATUS_UPDATE 665 + * @vdpa_index: Index for vdpa subdevice 666 + * @vf_id: VF id 667 + * @status: new status bits 668 + */ 669 + struct pds_vdpa_status_cmd { 670 + u8 opcode; 671 + u8 vdpa_index; 672 + __le16 vf_id; 673 + u8 status; 674 + }; 675 + 676 + /** 677 + * enum pds_vdpa_attr - List of VDPA device attributes 678 + * @PDS_VDPA_ATTR_MAC: MAC address 679 + * @PDS_VDPA_ATTR_MAX_VQ_PAIRS: Max virtqueue pairs 680 + */ 681 + enum pds_vdpa_attr { 682 + PDS_VDPA_ATTR_MAC = 1, 683 + PDS_VDPA_ATTR_MAX_VQ_PAIRS = 2, 684 + }; 685 + 686 + /** 687 + * struct pds_vdpa_setattr_cmd - SET_ATTR command 688 + * @opcode: Opcode PDS_VDPA_CMD_SET_ATTR 689 + * @vdpa_index: Index for vdpa subdevice 690 + * @vf_id: VF id 691 + * @attr: attribute to be changed (enum pds_vdpa_attr) 692 + * @pad: Word boundary padding 693 + * @mac: new mac address to be assigned as vdpa device address 694 + * @max_vq_pairs: new limit of virtqueue pairs 695 + */ 696 + struct pds_vdpa_setattr_cmd { 697 + u8 opcode; 698 + u8 vdpa_index; 699 + __le16 vf_id; 700 + u8 attr; 701 + u8 pad[3]; 702 + union { 703 + u8 mac[6]; 704 + __le16 max_vq_pairs; 705 + } __packed; 706 + }; 707 + 708 + /** 709 + * struct pds_vdpa_vq_init_cmd - queue init command 710 + * @opcode: Opcode PDS_VDPA_CMD_VQ_INIT 711 + * @vdpa_index: Index for vdpa subdevice 712 + * @vf_id: VF id 713 + * @qid: Queue id (bit0 clear = rx, bit0 set = tx, qid=N is ctrlq) 714 + * @len: log(2) of max descriptor count 715 + * @desc_addr: DMA address of descriptor area 716 + * @avail_addr: DMA address of available descriptors (aka driver area) 717 + * @used_addr: DMA address of used descriptors (aka device area) 718 + * @intr_index: interrupt index 719 + * @avail_index: initial device position in available ring 720 + * @used_index: initial device position in used ring 721 + */ 722 + struct pds_vdpa_vq_init_cmd { 723 + u8 opcode; 724 + u8 vdpa_index; 725 + __le16 vf_id; 726 + __le16 qid; 727 + __le16 len; 728 + __le64 desc_addr; 729 + __le64 avail_addr; 730 + __le64 used_addr; 731 + __le16 intr_index; 732 + __le16 avail_index; 733 + __le16 used_index; 734 + }; 735 + 736 + /** 737 + * struct pds_vdpa_vq_init_comp - queue init completion 738 + * @status: Status of the command (enum pds_core_status_code) 739 + * @hw_qtype: HW queue type, used in doorbell selection 740 + * @hw_qindex: HW queue index, used in doorbell selection 741 + * @rsvd: Word boundary padding 742 + * @color: Color bit 743 + */ 744 + struct pds_vdpa_vq_init_comp { 745 + u8 status; 746 + u8 hw_qtype; 747 + __le16 hw_qindex; 748 + u8 rsvd[11]; 749 + u8 color; 750 + }; 751 + 752 + /** 753 + * struct pds_vdpa_vq_reset_cmd - queue reset command 754 + * @opcode: Opcode PDS_VDPA_CMD_VQ_RESET 755 + * @vdpa_index: Index for vdpa subdevice 756 + * @vf_id: VF id 757 + * @qid: Queue id 758 + */ 759 + struct pds_vdpa_vq_reset_cmd { 760 + u8 opcode; 761 + u8 vdpa_index; 762 + __le16 vf_id; 763 + __le16 qid; 764 + }; 765 + 766 + /** 767 + * struct pds_vdpa_vq_reset_comp - queue reset completion 768 + * @status: Status of the command (enum pds_core_status_code) 769 + * @rsvd0: Word boundary padding 770 + * @avail_index: current device position in available ring 771 + * @used_index: current device position in used ring 772 + * @rsvd: Word boundary padding 773 + * @color: Color bit 774 + */ 775 + struct pds_vdpa_vq_reset_comp { 776 + u8 status; 777 + u8 rsvd0; 778 + __le16 avail_index; 779 + __le16 used_index; 780 + u8 rsvd[9]; 781 + u8 color; 782 + }; 783 + 784 + /** 785 + * struct pds_vdpa_set_features_cmd - set hw features 786 + * @opcode: Opcode PDS_VDPA_CMD_SET_FEATURES 787 + * @vdpa_index: Index for vdpa subdevice 788 + * @vf_id: VF id 789 + * @rsvd: Word boundary padding 790 + * @features: Feature bit mask 791 + */ 792 + struct pds_vdpa_set_features_cmd { 793 + u8 opcode; 794 + u8 vdpa_index; 795 + __le16 vf_id; 796 + __le32 rsvd; 797 + __le64 features; 798 + }; 799 + 608 800 union pds_core_adminq_cmd { 609 801 u8 opcode; 610 802 u8 bytes[64]; ··· 834 600 835 601 struct pds_core_q_identify_cmd q_ident; 836 602 struct pds_core_q_init_cmd q_init; 603 + 604 + struct pds_vdpa_cmd vdpa; 605 + struct pds_vdpa_init_cmd vdpa_init; 606 + struct pds_vdpa_ident_cmd vdpa_ident; 607 + struct pds_vdpa_status_cmd vdpa_status; 608 + struct pds_vdpa_setattr_cmd vdpa_setattr; 609 + struct pds_vdpa_set_features_cmd vdpa_set_features; 610 + struct pds_vdpa_vq_init_cmd vdpa_vq_init; 611 + struct pds_vdpa_vq_reset_cmd vdpa_vq_reset; 612 + 837 613 }; 838 614 839 615 union pds_core_adminq_comp { ··· 865 621 866 622 struct pds_core_q_identify_comp q_ident; 867 623 struct pds_core_q_init_comp q_init; 624 + 625 + struct pds_vdpa_vq_init_comp vdpa_vq_init; 626 + struct pds_vdpa_vq_reset_comp vdpa_vq_reset; 868 627 }; 869 628 870 629 #ifndef __CHECKER__

+1 -20

include/linux/pds/pds_common.h

··· 39 39 #define PDS_DEV_TYPE_RDMA_STR "RDMA" 40 40 #define PDS_DEV_TYPE_LM_STR "LM" 41 41 42 - #define PDS_CORE_IFNAMSIZ 16 43 - 44 - /** 45 - * enum pds_core_logical_qtype - Logical Queue Types 46 - * @PDS_CORE_QTYPE_ADMINQ: Administrative Queue 47 - * @PDS_CORE_QTYPE_NOTIFYQ: Notify Queue 48 - * @PDS_CORE_QTYPE_RXQ: Receive Queue 49 - * @PDS_CORE_QTYPE_TXQ: Transmit Queue 50 - * @PDS_CORE_QTYPE_EQ: Event Queue 51 - * @PDS_CORE_QTYPE_MAX: Max queue type supported 52 - */ 53 - enum pds_core_logical_qtype { 54 - PDS_CORE_QTYPE_ADMINQ = 0, 55 - PDS_CORE_QTYPE_NOTIFYQ = 1, 56 - PDS_CORE_QTYPE_RXQ = 2, 57 - PDS_CORE_QTYPE_TXQ = 3, 58 - PDS_CORE_QTYPE_EQ = 4, 59 - 60 - PDS_CORE_QTYPE_MAX = 16 /* don't change - used in struct size */ 61 - }; 42 + #define PDS_VDPA_DEV_NAME PDS_CORE_DRV_NAME "." PDS_DEV_TYPE_VDPA_STR 62 43 63 44 int pdsc_register_notify(struct notifier_block *nb); 64 45 void pdsc_unregister_notify(struct notifier_block *nb);

+4 -1

include/linux/virtio.h

··· 103 103 * @config_enabled: configuration change reporting enabled 104 104 * @config_change_pending: configuration change reported while disabled 105 105 * @config_lock: protects configuration change reporting 106 + * @vqs_list_lock: protects @vqs. 106 107 * @dev: underlying device. 107 108 * @id: the device type identification (used to match it with a driver). 108 109 * @config: the configuration ops for this device. ··· 118 117 bool config_enabled; 119 118 bool config_change_pending; 120 119 spinlock_t config_lock; 121 - spinlock_t vqs_list_lock; /* Protects VQs list access */ 120 + spinlock_t vqs_list_lock; 122 121 struct device dev; 123 122 struct virtio_device_id id; 124 123 const struct virtio_config_ops *config; ··· 161 160 * @feature_table_size: number of entries in the feature table array. 162 161 * @feature_table_legacy: same as feature_table but when working in legacy mode. 163 162 * @feature_table_size_legacy: number of entries in feature table legacy array. 163 + * @validate: the function to call to validate features and config space. 164 + * Returns 0 or -errno. 164 165 * @probe: the function to call when a device is found. Returns 0 or -errno. 165 166 * @scan: optional function to call after successful probe; intended 166 167 * for virtio-scsi to invoke a scan.

+6

include/linux/virtio_pci_modern.h

··· 38 38 int modern_bars; 39 39 40 40 struct virtio_device_id id; 41 + 42 + /* optional check for vendor virtio device, returns dev_id or -ERRNO */ 43 + int (*device_id_check)(struct pci_dev *pdev); 44 + 45 + /* optional mask for devices with limited DMA space */ 46 + u64 dma_mask; 41 47 }; 42 48 43 49 /*

+31

include/uapi/linux/vhost.h

··· 45 45 #define VHOST_SET_LOG_BASE _IOW(VHOST_VIRTIO, 0x04, __u64) 46 46 /* Specify an eventfd file descriptor to signal on log write. */ 47 47 #define VHOST_SET_LOG_FD _IOW(VHOST_VIRTIO, 0x07, int) 48 + /* By default, a device gets one vhost_worker that its virtqueues share. This 49 + * command allows the owner of the device to create an additional vhost_worker 50 + * for the device. It can later be bound to 1 or more of its virtqueues using 51 + * the VHOST_ATTACH_VRING_WORKER command. 52 + * 53 + * This must be called after VHOST_SET_OWNER and the caller must be the owner 54 + * of the device. The new thread will inherit caller's cgroups and namespaces, 55 + * and will share the caller's memory space. The new thread will also be 56 + * counted against the caller's RLIMIT_NPROC value. 57 + * 58 + * The worker's ID used in other commands will be returned in 59 + * vhost_worker_state. 60 + */ 61 + #define VHOST_NEW_WORKER _IOR(VHOST_VIRTIO, 0x8, struct vhost_worker_state) 62 + /* Free a worker created with VHOST_NEW_WORKER if it's not attached to any 63 + * virtqueue. If userspace is not able to call this for workers its created, 64 + * the kernel will free all the device's workers when the device is closed. 65 + */ 66 + #define VHOST_FREE_WORKER _IOW(VHOST_VIRTIO, 0x9, struct vhost_worker_state) 48 67 49 68 /* Ring setup. */ 50 69 /* Set number of descriptors in ring. This parameter can not ··· 89 70 #define VHOST_VRING_BIG_ENDIAN 1 90 71 #define VHOST_SET_VRING_ENDIAN _IOW(VHOST_VIRTIO, 0x13, struct vhost_vring_state) 91 72 #define VHOST_GET_VRING_ENDIAN _IOW(VHOST_VIRTIO, 0x14, struct vhost_vring_state) 73 + /* Attach a vhost_worker created with VHOST_NEW_WORKER to one of the device's 74 + * virtqueues. 75 + * 76 + * This will replace the virtqueue's existing worker. If the replaced worker 77 + * is no longer attached to any virtqueues, it can be freed with 78 + * VHOST_FREE_WORKER. 79 + */ 80 + #define VHOST_ATTACH_VRING_WORKER _IOW(VHOST_VIRTIO, 0x15, \ 81 + struct vhost_vring_worker) 82 + /* Return the vring worker's ID */ 83 + #define VHOST_GET_VRING_WORKER _IOWR(VHOST_VIRTIO, 0x16, \ 84 + struct vhost_vring_worker) 92 85 93 86 /* The following ioctls use eventfd file descriptors to signal and poll 94 87 * for events. */

+16

include/uapi/linux/vhost_types.h

··· 47 47 __u64 log_guest_addr; 48 48 }; 49 49 50 + struct vhost_worker_state { 51 + /* 52 + * For VHOST_NEW_WORKER the kernel will return the new vhost_worker id. 53 + * For VHOST_FREE_WORKER this must be set to the id of the vhost_worker 54 + * to free. 55 + */ 56 + unsigned int worker_id; 57 + }; 58 + 59 + struct vhost_vring_worker { 60 + /* vring index */ 61 + unsigned int index; 62 + /* The id of the vhost_worker returned from VHOST_NEW_WORKER */ 63 + unsigned int worker_id; 64 + }; 65 + 50 66 /* no alignment requirement */ 51 67 struct vhost_iotlb_msg { 52 68 __u64 iova;

+12 -1

tools/virtio/Makefile

··· 4 4 virtio_test: virtio_ring.o virtio_test.o 5 5 vringh_test: vringh_test.o vringh.o virtio_ring.o 6 6 7 - CFLAGS += -g -O2 -Werror -Wno-maybe-uninitialized -Wall -I. -I../include/ -I ../../usr/include/ -Wno-pointer-sign -fno-strict-overflow -fno-strict-aliasing -fno-common -MMD -U_FORTIFY_SOURCE -include ../../include/linux/kconfig.h -mfunction-return=thunk -fcf-protection=none -mindirect-branch-register 7 + try-run = $(shell set -e; \ 8 + if ($(1)) >/dev/null 2>&1; \ 9 + then echo "$(2)"; \ 10 + else echo "$(3)"; \ 11 + fi) 12 + 13 + __cc-option = $(call try-run,\ 14 + $(1) -Werror $(2) -c -x c /dev/null -o /dev/null,$(2),) 15 + cc-option = $(call __cc-option, $(CC),$(1)) 16 + 17 + CFLAGS += -g -O2 -Werror -Wno-maybe-uninitialized -Wall -I. -I../include/ -I ../../usr/include/ -Wno-pointer-sign -fno-strict-overflow -fno-strict-aliasing -fno-common -MMD -U_FORTIFY_SOURCE -include ../../include/linux/kconfig.h $(call cc-option,-mfunction-return=thunk) $(call cc-option,-fcf-protection=none) $(call cc-option,-mindirect-branch-register) 18 + 8 19 CFLAGS += -pthread 9 20 LDFLAGS += -pthread 10 21 vpath %.c ../../drivers/virtio ../../drivers/vhost

Configure Feed

Configure Feed