Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost

+1 -1

drivers/vdpa/octeon_ep/octep_vdpa_hw.c

··· 140 140 val = octep_read_sig(mbox); 141 141 if ((val & 0xFFFF) != MBOX_RSP_SIG) { 142 142 dev_warn(&pdev->dev, "Invalid Signature from mbox : %d response\n", id); 143 - return ret; 143 + return -EINVAL; 144 144 } 145 145 146 146 val = octep_read_sts(mbox);

+2 -26

drivers/virtio/virtio.c

··· 305 305 if (err) 306 306 goto err; 307 307 308 - if (dev->config->create_avq) { 309 - err = dev->config->create_avq(dev); 310 - if (err) 311 - goto err; 312 - } 313 - 314 308 err = drv->probe(dev); 315 309 if (err) 316 - goto err_probe; 310 + goto err; 317 311 318 312 /* If probe didn't do it, mark device DRIVER_OK ourselves. */ 319 313 if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK)) ··· 320 326 321 327 return 0; 322 328 323 - err_probe: 324 - if (dev->config->destroy_avq) 325 - dev->config->destroy_avq(dev); 326 329 err: 327 330 virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED); 328 331 return err; ··· 334 343 virtio_config_disable(dev); 335 344 336 345 drv->remove(dev); 337 - 338 - if (dev->config->destroy_avq) 339 - dev->config->destroy_avq(dev); 340 346 341 347 /* Driver should have reset device. */ 342 348 WARN_ON_ONCE(dev->config->get_status(dev)); ··· 512 524 } 513 525 } 514 526 515 - if (dev->config->destroy_avq) 516 - dev->config->destroy_avq(dev); 517 - 518 527 return 0; 519 528 } 520 529 EXPORT_SYMBOL_GPL(virtio_device_freeze); ··· 547 562 if (ret) 548 563 goto err; 549 564 550 - if (dev->config->create_avq) { 551 - ret = dev->config->create_avq(dev); 552 - if (ret) 553 - goto err; 554 - } 555 - 556 565 if (drv->restore) { 557 566 ret = drv->restore(dev); 558 567 if (ret) 559 - goto err_restore; 568 + goto err; 560 569 } 561 570 562 571 /* If restore didn't do it, mark device DRIVER_OK ourselves. */ ··· 561 582 562 583 return 0; 563 584 564 - err_restore: 565 - if (dev->config->destroy_avq) 566 - dev->config->destroy_avq(dev); 567 585 err: 568 586 virtio_add_status(dev, VIRTIO_CONFIG_S_FAILED); 569 587 return ret;

+151 -41

drivers/virtio/virtio_pci_common.c

··· 46 46 return true; 47 47 } 48 48 49 + /* Notify all slow path virtqueues on an interrupt. */ 50 + static void vp_vring_slow_path_interrupt(int irq, 51 + struct virtio_pci_device *vp_dev) 52 + { 53 + struct virtio_pci_vq_info *info; 54 + unsigned long flags; 55 + 56 + spin_lock_irqsave(&vp_dev->lock, flags); 57 + list_for_each_entry(info, &vp_dev->slow_virtqueues, node) 58 + vring_interrupt(irq, info->vq); 59 + spin_unlock_irqrestore(&vp_dev->lock, flags); 60 + } 61 + 49 62 /* Handle a configuration change: Tell driver if it wants to know. */ 50 63 static irqreturn_t vp_config_changed(int irq, void *opaque) 51 64 { 52 65 struct virtio_pci_device *vp_dev = opaque; 53 66 54 67 virtio_config_changed(&vp_dev->vdev); 68 + vp_vring_slow_path_interrupt(irq, vp_dev); 55 69 return IRQ_HANDLED; 56 70 } 57 71 ··· 139 125 GFP_KERNEL)) 140 126 goto error; 141 127 128 + if (!per_vq_vectors) 129 + desc = NULL; 130 + 142 131 if (desc) { 143 132 flags |= PCI_IRQ_AFFINITY; 144 133 desc->pre_vectors++; /* virtio config vector */ ··· 188 171 return err; 189 172 } 190 173 174 + static bool vp_is_slow_path_vector(u16 msix_vec) 175 + { 176 + return msix_vec == VP_MSIX_CONFIG_VECTOR; 177 + } 178 + 191 179 static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned int index, 192 180 void (*callback)(struct virtqueue *vq), 193 181 const char *name, 194 182 bool ctx, 195 - u16 msix_vec) 183 + u16 msix_vec, 184 + struct virtio_pci_vq_info **p_info) 196 185 { 197 186 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 198 187 struct virtio_pci_vq_info *info = kmalloc(sizeof *info, GFP_KERNEL); ··· 217 194 info->vq = vq; 218 195 if (callback) { 219 196 spin_lock_irqsave(&vp_dev->lock, flags); 220 - list_add(&info->node, &vp_dev->virtqueues); 197 + if (!vp_is_slow_path_vector(msix_vec)) 198 + list_add(&info->node, &vp_dev->virtqueues); 199 + else 200 + list_add(&info->node, &vp_dev->slow_virtqueues); 221 201 spin_unlock_irqrestore(&vp_dev->lock, flags); 222 202 } else { 223 203 INIT_LIST_HEAD(&info->node); 224 204 } 225 205 226 - vp_dev->vqs[index] = info; 206 + *p_info = info; 227 207 return vq; 228 208 229 209 out_info: ··· 262 236 int i; 263 237 264 238 list_for_each_entry_safe(vq, n, &vdev->vqs, list) { 265 - if (vp_dev->is_avq && vp_dev->is_avq(vdev, vq->index)) 266 - continue; 267 - 268 239 if (vp_dev->per_vq_vectors) { 269 240 int v = vp_dev->vqs[vq->index]->msix_vector; 270 241 271 - if (v != VIRTIO_MSI_NO_VECTOR) { 242 + if (v != VIRTIO_MSI_NO_VECTOR && 243 + !vp_is_slow_path_vector(v)) { 272 244 int irq = pci_irq_vector(vp_dev->pci_dev, v); 273 245 274 246 irq_update_affinity_hint(irq, NULL); ··· 308 284 vp_dev->vqs = NULL; 309 285 } 310 286 287 + enum vp_vq_vector_policy { 288 + VP_VQ_VECTOR_POLICY_EACH, 289 + VP_VQ_VECTOR_POLICY_SHARED_SLOW, 290 + VP_VQ_VECTOR_POLICY_SHARED, 291 + }; 292 + 293 + static struct virtqueue * 294 + vp_find_one_vq_msix(struct virtio_device *vdev, int queue_idx, 295 + vq_callback_t *callback, const char *name, bool ctx, 296 + bool slow_path, int *allocated_vectors, 297 + enum vp_vq_vector_policy vector_policy, 298 + struct virtio_pci_vq_info **p_info) 299 + { 300 + struct virtio_pci_device *vp_dev = to_vp_device(vdev); 301 + struct virtqueue *vq; 302 + u16 msix_vec; 303 + int err; 304 + 305 + if (!callback) 306 + msix_vec = VIRTIO_MSI_NO_VECTOR; 307 + else if (vector_policy == VP_VQ_VECTOR_POLICY_EACH || 308 + (vector_policy == VP_VQ_VECTOR_POLICY_SHARED_SLOW && 309 + !slow_path)) 310 + msix_vec = (*allocated_vectors)++; 311 + else if (vector_policy != VP_VQ_VECTOR_POLICY_EACH && 312 + slow_path) 313 + msix_vec = VP_MSIX_CONFIG_VECTOR; 314 + else 315 + msix_vec = VP_MSIX_VQ_VECTOR; 316 + vq = vp_setup_vq(vdev, queue_idx, callback, name, ctx, msix_vec, 317 + p_info); 318 + if (IS_ERR(vq)) 319 + return vq; 320 + 321 + if (vector_policy == VP_VQ_VECTOR_POLICY_SHARED || 322 + msix_vec == VIRTIO_MSI_NO_VECTOR || 323 + vp_is_slow_path_vector(msix_vec)) 324 + return vq; 325 + 326 + /* allocate per-vq irq if available and necessary */ 327 + snprintf(vp_dev->msix_names[msix_vec], sizeof(*vp_dev->msix_names), 328 + "%s-%s", dev_name(&vp_dev->vdev.dev), name); 329 + err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec), 330 + vring_interrupt, 0, 331 + vp_dev->msix_names[msix_vec], vq); 332 + if (err) { 333 + vp_del_vq(vq); 334 + return ERR_PTR(err); 335 + } 336 + 337 + return vq; 338 + } 339 + 311 340 static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs, 312 341 struct virtqueue *vqs[], 313 342 struct virtqueue_info vqs_info[], 314 - bool per_vq_vectors, 343 + enum vp_vq_vector_policy vector_policy, 315 344 struct irq_affinity *desc) 316 345 { 317 346 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 347 + struct virtio_pci_admin_vq *avq = &vp_dev->admin_vq; 318 348 struct virtqueue_info *vqi; 319 - u16 msix_vec; 320 349 int i, err, nvectors, allocated_vectors, queue_idx = 0; 350 + struct virtqueue *vq; 351 + bool per_vq_vectors; 352 + u16 avq_num = 0; 321 353 322 354 vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL); 323 355 if (!vp_dev->vqs) 324 356 return -ENOMEM; 357 + 358 + if (vp_dev->avq_index) { 359 + err = vp_dev->avq_index(vdev, &avq->vq_index, &avq_num); 360 + if (err) 361 + goto error_find; 362 + } 363 + 364 + per_vq_vectors = vector_policy != VP_VQ_VECTOR_POLICY_SHARED; 325 365 326 366 if (per_vq_vectors) { 327 367 /* Best option: one for change interrupt, one per vq. */ ··· 395 307 if (vqi->name && vqi->callback) 396 308 ++nvectors; 397 309 } 310 + if (avq_num && vector_policy == VP_VQ_VECTOR_POLICY_EACH) 311 + ++nvectors; 398 312 } else { 399 313 /* Second best: one for change, shared for all vqs. */ 400 314 nvectors = 2; 401 315 } 402 316 403 - err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors, 404 - per_vq_vectors ? desc : NULL); 317 + err = vp_request_msix_vectors(vdev, nvectors, per_vq_vectors, desc); 405 318 if (err) 406 319 goto error_find; 407 320 ··· 414 325 vqs[i] = NULL; 415 326 continue; 416 327 } 417 - 418 - if (!vqi->callback) 419 - msix_vec = VIRTIO_MSI_NO_VECTOR; 420 - else if (vp_dev->per_vq_vectors) 421 - msix_vec = allocated_vectors++; 422 - else 423 - msix_vec = VP_MSIX_VQ_VECTOR; 424 - vqs[i] = vp_setup_vq(vdev, queue_idx++, vqi->callback, 425 - vqi->name, vqi->ctx, msix_vec); 328 + vqs[i] = vp_find_one_vq_msix(vdev, queue_idx++, vqi->callback, 329 + vqi->name, vqi->ctx, false, 330 + &allocated_vectors, vector_policy, 331 + &vp_dev->vqs[i]); 426 332 if (IS_ERR(vqs[i])) { 427 333 err = PTR_ERR(vqs[i]); 428 334 goto error_find; 429 335 } 430 - 431 - if (!vp_dev->per_vq_vectors || msix_vec == VIRTIO_MSI_NO_VECTOR) 432 - continue; 433 - 434 - /* allocate per-vq irq if available and necessary */ 435 - snprintf(vp_dev->msix_names[msix_vec], 436 - sizeof *vp_dev->msix_names, 437 - "%s-%s", 438 - dev_name(&vp_dev->vdev.dev), vqi->name); 439 - err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec), 440 - vring_interrupt, 0, 441 - vp_dev->msix_names[msix_vec], 442 - vqs[i]); 443 - if (err) { 444 - vp_del_vq(vqs[i]); 445 - goto error_find; 446 - } 447 336 } 337 + 338 + if (!avq_num) 339 + return 0; 340 + sprintf(avq->name, "avq.%u", avq->vq_index); 341 + vq = vp_find_one_vq_msix(vdev, avq->vq_index, vp_modern_avq_done, 342 + avq->name, false, true, &allocated_vectors, 343 + vector_policy, &vp_dev->admin_vq.info); 344 + if (IS_ERR(vq)) { 345 + err = PTR_ERR(vq); 346 + goto error_find; 347 + } 348 + 448 349 return 0; 449 350 450 351 error_find: ··· 447 368 struct virtqueue_info vqs_info[]) 448 369 { 449 370 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 371 + struct virtio_pci_admin_vq *avq = &vp_dev->admin_vq; 450 372 int i, err, queue_idx = 0; 373 + struct virtqueue *vq; 374 + u16 avq_num = 0; 451 375 452 376 vp_dev->vqs = kcalloc(nvqs, sizeof(*vp_dev->vqs), GFP_KERNEL); 453 377 if (!vp_dev->vqs) 454 378 return -ENOMEM; 379 + 380 + if (vp_dev->avq_index) { 381 + err = vp_dev->avq_index(vdev, &avq->vq_index, &avq_num); 382 + if (err) 383 + goto out_del_vqs; 384 + } 455 385 456 386 err = request_irq(vp_dev->pci_dev->irq, vp_interrupt, IRQF_SHARED, 457 387 dev_name(&vdev->dev), vp_dev); ··· 478 390 } 479 391 vqs[i] = vp_setup_vq(vdev, queue_idx++, vqi->callback, 480 392 vqi->name, vqi->ctx, 481 - VIRTIO_MSI_NO_VECTOR); 393 + VIRTIO_MSI_NO_VECTOR, &vp_dev->vqs[i]); 482 394 if (IS_ERR(vqs[i])) { 483 395 err = PTR_ERR(vqs[i]); 484 396 goto out_del_vqs; 485 397 } 398 + } 399 + 400 + if (!avq_num) 401 + return 0; 402 + sprintf(avq->name, "avq.%u", avq->vq_index); 403 + vq = vp_setup_vq(vdev, queue_idx++, vp_modern_avq_done, avq->name, 404 + false, VIRTIO_MSI_NO_VECTOR, 405 + &vp_dev->admin_vq.info); 406 + if (IS_ERR(vq)) { 407 + err = PTR_ERR(vq); 408 + goto out_del_vqs; 486 409 } 487 410 488 411 return 0; ··· 510 411 int err; 511 412 512 413 /* Try MSI-X with one vector per queue. */ 513 - err = vp_find_vqs_msix(vdev, nvqs, vqs, vqs_info, true, desc); 414 + err = vp_find_vqs_msix(vdev, nvqs, vqs, vqs_info, 415 + VP_VQ_VECTOR_POLICY_EACH, desc); 416 + if (!err) 417 + return 0; 418 + /* Fallback: MSI-X with one shared vector for config and 419 + * slow path queues, one vector per queue for the rest. 420 + */ 421 + err = vp_find_vqs_msix(vdev, nvqs, vqs, vqs_info, 422 + VP_VQ_VECTOR_POLICY_SHARED_SLOW, desc); 514 423 if (!err) 515 424 return 0; 516 425 /* Fallback: MSI-X with one vector for config, one shared for queues. */ 517 - err = vp_find_vqs_msix(vdev, nvqs, vqs, vqs_info, false, desc); 426 + err = vp_find_vqs_msix(vdev, nvqs, vqs, vqs_info, 427 + VP_VQ_VECTOR_POLICY_SHARED, desc); 518 428 if (!err) 519 429 return 0; 520 430 /* Is there an interrupt? If not give up. */ ··· 574 466 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 575 467 576 468 if (!vp_dev->per_vq_vectors || 577 - vp_dev->vqs[index]->msix_vector == VIRTIO_MSI_NO_VECTOR) 469 + vp_dev->vqs[index]->msix_vector == VIRTIO_MSI_NO_VECTOR || 470 + vp_is_slow_path_vector(vp_dev->vqs[index]->msix_vector)) 578 471 return NULL; 579 472 580 473 return pci_irq_get_affinity(vp_dev->pci_dev, ··· 683 574 vp_dev->vdev.dev.release = virtio_pci_release_dev; 684 575 vp_dev->pci_dev = pci_dev; 685 576 INIT_LIST_HEAD(&vp_dev->virtqueues); 577 + INIT_LIST_HEAD(&vp_dev->slow_virtqueues); 686 578 spin_lock_init(&vp_dev->lock); 687 579 688 580 /* enable the device */

+10 -6

drivers/virtio/virtio_pci_common.h

··· 35 35 /* the actual virtqueue */ 36 36 struct virtqueue *vq; 37 37 38 - /* the list node for the virtqueues list */ 38 + /* the list node for the virtqueues or slow_virtqueues list */ 39 39 struct list_head node; 40 40 41 41 /* MSI-X vector (or none) */ ··· 44 44 45 45 struct virtio_pci_admin_vq { 46 46 /* Virtqueue info associated with this admin queue. */ 47 - struct virtio_pci_vq_info info; 48 - /* serializing admin commands execution and virtqueue deletion */ 49 - struct mutex cmd_lock; 47 + struct virtio_pci_vq_info *info; 48 + /* Protects virtqueue access. */ 49 + spinlock_t lock; 50 50 u64 supported_cmds; 51 51 /* Name of the admin queue: avq.$vq_index. */ 52 52 char name[10]; ··· 66 66 /* Where to read and clear interrupt */ 67 67 u8 __iomem *isr; 68 68 69 - /* a list of queues so we can dispatch IRQs */ 69 + /* Lists of queues and potentially slow path queues 70 + * so we can dispatch IRQs. 71 + */ 70 72 spinlock_t lock; 71 73 struct list_head virtqueues; 74 + struct list_head slow_virtqueues; 72 75 73 76 /* Array of all virtqueues reported in the 74 77 * PCI common config num_queues field ··· 105 102 void (*del_vq)(struct virtio_pci_vq_info *info); 106 103 107 104 u16 (*config_vector)(struct virtio_pci_device *vp_dev, u16 vector); 108 - bool (*is_avq)(struct virtio_device *vdev, unsigned int index); 105 + int (*avq_index)(struct virtio_device *vdev, u16 *index, u16 *num); 109 106 }; 110 107 111 108 /* Constants for MSI-X */ ··· 178 175 #define VIRTIO_ADMIN_CMD_BITMAP 0 179 176 #endif 180 177 178 + void vp_modern_avq_done(struct virtqueue *vq); 181 179 int vp_modern_admin_cmd_exec(struct virtio_device *vdev, 182 180 struct virtio_admin_cmd *cmd); 183 181

+75 -86

drivers/virtio/virtio_pci_modern.c

··· 28 28 return vp_modern_get_features(&vp_dev->mdev); 29 29 } 30 30 31 + static int vp_avq_index(struct virtio_device *vdev, u16 *index, u16 *num) 32 + { 33 + struct virtio_pci_device *vp_dev = to_vp_device(vdev); 34 + 35 + *num = 0; 36 + if (!virtio_has_feature(vdev, VIRTIO_F_ADMIN_VQ)) 37 + return 0; 38 + 39 + *num = vp_modern_avq_num(&vp_dev->mdev); 40 + if (!(*num)) 41 + return -EINVAL; 42 + *index = vp_modern_avq_index(&vp_dev->mdev); 43 + return 0; 44 + } 45 + 31 46 static bool vp_is_avq(struct virtio_device *vdev, unsigned int index) 32 47 { 33 48 struct virtio_pci_device *vp_dev = to_vp_device(vdev); ··· 53 38 return index == vp_dev->admin_vq.vq_index; 54 39 } 55 40 41 + void vp_modern_avq_done(struct virtqueue *vq) 42 + { 43 + struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 44 + struct virtio_pci_admin_vq *admin_vq = &vp_dev->admin_vq; 45 + struct virtio_admin_cmd *cmd; 46 + unsigned long flags; 47 + unsigned int len; 48 + 49 + spin_lock_irqsave(&admin_vq->lock, flags); 50 + do { 51 + virtqueue_disable_cb(vq); 52 + while ((cmd = virtqueue_get_buf(vq, &len))) 53 + complete(&cmd->completion); 54 + } while (!virtqueue_enable_cb(vq)); 55 + spin_unlock_irqrestore(&admin_vq->lock, flags); 56 + } 57 + 56 58 static int virtqueue_exec_admin_cmd(struct virtio_pci_admin_vq *admin_vq, 57 59 u16 opcode, 58 60 struct scatterlist **sgs, 59 61 unsigned int out_num, 60 62 unsigned int in_num, 61 - void *data) 63 + struct virtio_admin_cmd *cmd) 62 64 { 63 65 struct virtqueue *vq; 64 - int ret, len; 66 + unsigned long flags; 67 + int ret; 65 68 66 - vq = admin_vq->info.vq; 69 + vq = admin_vq->info->vq; 67 70 if (!vq) 68 71 return -EIO; 69 72 ··· 90 57 !((1ULL << opcode) & admin_vq->supported_cmds)) 91 58 return -EOPNOTSUPP; 92 59 93 - ret = virtqueue_add_sgs(vq, sgs, out_num, in_num, data, GFP_KERNEL); 94 - if (ret < 0) 95 - return -EIO; 60 + init_completion(&cmd->completion); 96 61 97 - if (unlikely(!virtqueue_kick(vq))) 98 - return -EIO; 99 - 100 - while (!virtqueue_get_buf(vq, &len) && 101 - !virtqueue_is_broken(vq)) 102 - cpu_relax(); 103 - 62 + again: 104 63 if (virtqueue_is_broken(vq)) 105 64 return -EIO; 106 65 107 - return 0; 66 + spin_lock_irqsave(&admin_vq->lock, flags); 67 + ret = virtqueue_add_sgs(vq, sgs, out_num, in_num, cmd, GFP_KERNEL); 68 + if (ret < 0) { 69 + if (ret == -ENOSPC) { 70 + spin_unlock_irqrestore(&admin_vq->lock, flags); 71 + cpu_relax(); 72 + goto again; 73 + } 74 + goto unlock_err; 75 + } 76 + if (!virtqueue_kick(vq)) 77 + goto unlock_err; 78 + spin_unlock_irqrestore(&admin_vq->lock, flags); 79 + 80 + wait_for_completion(&cmd->completion); 81 + 82 + return cmd->ret; 83 + 84 + unlock_err: 85 + spin_unlock_irqrestore(&admin_vq->lock, flags); 86 + return -EIO; 108 87 } 109 88 110 89 int vp_modern_admin_cmd_exec(struct virtio_device *vdev, ··· 167 122 in_num++; 168 123 } 169 124 170 - mutex_lock(&vp_dev->admin_vq.cmd_lock); 171 125 ret = virtqueue_exec_admin_cmd(&vp_dev->admin_vq, 172 126 le16_to_cpu(cmd->opcode), 173 - sgs, out_num, in_num, sgs); 174 - mutex_unlock(&vp_dev->admin_vq.cmd_lock); 175 - 127 + sgs, out_num, in_num, cmd); 176 128 if (ret) { 177 129 dev_err(&vdev->dev, 178 130 "Failed to execute command on admin vq: %d\n.", ret); ··· 230 188 231 189 static void vp_modern_avq_activate(struct virtio_device *vdev) 232 190 { 233 - struct virtio_pci_device *vp_dev = to_vp_device(vdev); 234 - struct virtio_pci_admin_vq *admin_vq = &vp_dev->admin_vq; 235 - 236 191 if (!virtio_has_feature(vdev, VIRTIO_F_ADMIN_VQ)) 237 192 return; 238 193 239 - __virtqueue_unbreak(admin_vq->info.vq); 240 194 virtio_pci_admin_cmd_list_init(vdev); 241 195 } 242 196 243 - static void vp_modern_avq_deactivate(struct virtio_device *vdev) 197 + static void vp_modern_avq_cleanup(struct virtio_device *vdev) 244 198 { 245 199 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 246 - struct virtio_pci_admin_vq *admin_vq = &vp_dev->admin_vq; 200 + struct virtio_admin_cmd *cmd; 201 + struct virtqueue *vq; 247 202 248 203 if (!virtio_has_feature(vdev, VIRTIO_F_ADMIN_VQ)) 249 204 return; 250 205 251 - __virtqueue_break(admin_vq->info.vq); 206 + vq = vp_dev->vqs[vp_dev->admin_vq.vq_index]->vq; 207 + if (!vq) 208 + return; 209 + 210 + while ((cmd = virtqueue_detach_unused_buf(vq))) { 211 + cmd->ret = -EIO; 212 + complete(&cmd->completion); 213 + } 252 214 } 253 215 254 216 static void vp_transport_features(struct virtio_device *vdev, u64 features) ··· 449 403 while (vp_modern_get_status(mdev)) 450 404 msleep(1); 451 405 452 - vp_modern_avq_deactivate(vdev); 406 + vp_modern_avq_cleanup(vdev); 453 407 454 408 /* Flush pending VQ/configuration callbacks. */ 455 409 vp_synchronize_vectors(vdev); ··· 598 552 if (index >= vp_modern_get_num_queues(mdev) && !is_avq) 599 553 return ERR_PTR(-EINVAL); 600 554 601 - num = is_avq ? 602 - VIRTIO_AVQ_SGS_MAX : vp_modern_get_queue_size(mdev, index); 555 + num = vp_modern_get_queue_size(mdev, index); 603 556 /* Check if queue is either not available or already active. */ 604 557 if (!num || vp_modern_get_queue_enable(mdev, index)) 605 558 return ERR_PTR(-ENOENT); ··· 623 578 if (!vq->priv) { 624 579 err = -ENOMEM; 625 580 goto err; 626 - } 627 - 628 - if (is_avq) { 629 - mutex_lock(&vp_dev->admin_vq.cmd_lock); 630 - vp_dev->admin_vq.info.vq = vq; 631 - mutex_unlock(&vp_dev->admin_vq.cmd_lock); 632 581 } 633 582 634 583 return vq; ··· 658 619 struct virtqueue *vq = info->vq; 659 620 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 660 621 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 661 - 662 - if (vp_is_avq(&vp_dev->vdev, vq->index)) { 663 - mutex_lock(&vp_dev->admin_vq.cmd_lock); 664 - vp_dev->admin_vq.info.vq = NULL; 665 - mutex_unlock(&vp_dev->admin_vq.cmd_lock); 666 - } 667 622 668 623 if (vp_dev->msix_enabled) 669 624 vp_modern_queue_vector(mdev, vq->index, ··· 768 735 return true; 769 736 } 770 737 771 - static int vp_modern_create_avq(struct virtio_device *vdev) 772 - { 773 - struct virtio_pci_device *vp_dev = to_vp_device(vdev); 774 - struct virtio_pci_admin_vq *avq; 775 - struct virtqueue *vq; 776 - u16 admin_q_num; 777 - 778 - if (!virtio_has_feature(vdev, VIRTIO_F_ADMIN_VQ)) 779 - return 0; 780 - 781 - admin_q_num = vp_modern_avq_num(&vp_dev->mdev); 782 - if (!admin_q_num) 783 - return -EINVAL; 784 - 785 - avq = &vp_dev->admin_vq; 786 - avq->vq_index = vp_modern_avq_index(&vp_dev->mdev); 787 - sprintf(avq->name, "avq.%u", avq->vq_index); 788 - vq = vp_dev->setup_vq(vp_dev, &vp_dev->admin_vq.info, avq->vq_index, NULL, 789 - avq->name, NULL, VIRTIO_MSI_NO_VECTOR); 790 - if (IS_ERR(vq)) { 791 - dev_err(&vdev->dev, "failed to setup admin virtqueue, err=%ld", 792 - PTR_ERR(vq)); 793 - return PTR_ERR(vq); 794 - } 795 - 796 - vp_modern_set_queue_enable(&vp_dev->mdev, avq->info.vq->index, true); 797 - return 0; 798 - } 799 - 800 - static void vp_modern_destroy_avq(struct virtio_device *vdev) 801 - { 802 - struct virtio_pci_device *vp_dev = to_vp_device(vdev); 803 - 804 - if (!virtio_has_feature(vdev, VIRTIO_F_ADMIN_VQ)) 805 - return; 806 - 807 - vp_dev->del_vq(&vp_dev->admin_vq.info); 808 - } 809 - 810 738 static const struct virtio_config_ops virtio_pci_config_nodev_ops = { 811 739 .get = NULL, 812 740 .set = NULL, ··· 786 792 .get_shm_region = vp_get_shm_region, 787 793 .disable_vq_and_reset = vp_modern_disable_vq_and_reset, 788 794 .enable_vq_after_reset = vp_modern_enable_vq_after_reset, 789 - .create_avq = vp_modern_create_avq, 790 - .destroy_avq = vp_modern_destroy_avq, 791 795 }; 792 796 793 797 static const struct virtio_config_ops virtio_pci_config_ops = { ··· 806 814 .get_shm_region = vp_get_shm_region, 807 815 .disable_vq_and_reset = vp_modern_disable_vq_and_reset, 808 816 .enable_vq_after_reset = vp_modern_enable_vq_after_reset, 809 - .create_avq = vp_modern_create_avq, 810 - .destroy_avq = vp_modern_destroy_avq, 811 817 }; 812 818 813 819 /* the PCI probing function */ ··· 829 839 vp_dev->config_vector = vp_config_vector; 830 840 vp_dev->setup_vq = setup_vq; 831 841 vp_dev->del_vq = del_vq; 832 - vp_dev->is_avq = vp_is_avq; 842 + vp_dev->avq_index = vp_avq_index; 833 843 vp_dev->isr = mdev->isr; 834 844 vp_dev->vdev.id = mdev->id; 835 845 836 - mutex_init(&vp_dev->admin_vq.cmd_lock); 846 + spin_lock_init(&vp_dev->admin_vq.lock); 837 847 return 0; 838 848 } 839 849 ··· 841 851 { 842 852 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 843 853 844 - mutex_destroy(&vp_dev->admin_vq.cmd_lock); 845 854 vp_modern_remove(mdev); 846 855 }

+3

include/linux/virtio.h

··· 10 10 #include <linux/mod_devicetable.h> 11 11 #include <linux/gfp.h> 12 12 #include <linux/dma-mapping.h> 13 + #include <linux/completion.h> 13 14 14 15 /** 15 16 * struct virtqueue - a queue to register buffers for sending or receiving. ··· 110 109 __le64 group_member_id; 111 110 struct scatterlist *data_sg; 112 111 struct scatterlist *result_sg; 112 + struct completion completion; 113 + int ret; 113 114 }; 114 115 115 116 /**

-4

include/linux/virtio_config.h

··· 104 104 * Returns 0 on success or error status 105 105 * If disable_vq_and_reset is set, then enable_vq_after_reset must also be 106 106 * set. 107 - * @create_avq: create admin virtqueue resource. 108 - * @destroy_avq: destroy admin virtqueue resource. 109 107 */ 110 108 struct virtio_config_ops { 111 109 void (*get)(struct virtio_device *vdev, unsigned offset, ··· 131 133 struct virtio_shm_region *region, u8 id); 132 134 int (*disable_vq_and_reset)(struct virtqueue *vq); 133 135 int (*enable_vq_after_reset)(struct virtqueue *vq); 134 - int (*create_avq)(struct virtio_device *vdev); 135 - void (*destroy_avq)(struct virtio_device *vdev); 136 136 }; 137 137 138 138 /* If driver didn't advertise the feature, it will never appear. */

Configure Feed

Configure Feed