+13

MAINTAINERS

reviewed

··· 12419 12419 F: drivers/media/v4l2-core/videobuf2-* 12420 12420 F: include/media/videobuf2-* 12421 12421 12422 12422 + VIRTIO AND VHOST VSOCK DRIVER 12423 12423 + M: Stefan Hajnoczi <stefanha@redhat.com> 12424 12424 + L: kvm@vger.kernel.org 12425 12425 + L: virtualization@lists.linux-foundation.org 12426 12426 + L: netdev@vger.kernel.org 12427 12427 + S: Maintained 12428 12428 + F: include/linux/virtio_vsock.h 12429 12429 + F: include/uapi/linux/virtio_vsock.h 12430 12430 + F: net/vmw_vsock/virtio_transport_common.c 12431 12431 + F: net/vmw_vsock/virtio_transport.c 12432 12432 + F: drivers/vhost/vsock.c 12433 12433 + F: drivers/vhost/vsock.h 12434 12434 + 12422 12435 VIRTUAL SERIO DEVICE DRIVER 12423 12436 M: Stephen Chandler Paul <thatslyude@gmail.com> 12424 12437 S: Maintained

+1

drivers/Makefile

reviewed

··· 138 138 obj-$(CONFIG_SSB) += ssb/ 139 139 obj-$(CONFIG_BCMA) += bcma/ 140 140 obj-$(CONFIG_VHOST_RING) += vhost/ 141 141 + obj-$(CONFIG_VHOST) += vhost/ 141 142 obj-$(CONFIG_VLYNQ) += vlynq/ 142 143 obj-$(CONFIG_STAGING) += staging/ 143 144 obj-y += platform/

+4

drivers/misc/mic/Kconfig

reviewed

··· 146 146 More information about the Intel MIC family as well as the Linux 147 147 OS and tools for MIC to use with this driver are available from 148 148 <http://software.intel.com/en-us/mic-developer>. 149 149 + 150 150 + if VOP 151 151 + source "drivers/vhost/Kconfig.vringh" 152 152 + endif

+1 -1

drivers/net/caif/Kconfig

reviewed

··· 52 52 The caif driver for CAIF over Virtio. 53 53 54 54 if CAIF_VIRTIO 55 55 - source "drivers/vhost/Kconfig" 55 55 + source "drivers/vhost/Kconfig.vringh" 56 56 endif

+12 -6

drivers/vhost/Kconfig

reviewed

··· 2 2 tristate "Host kernel accelerator for virtio net" 3 3 depends on NET && EVENTFD && (TUN || !TUN) && (MACVTAP || !MACVTAP) 4 4 select VHOST 5 5 - select VHOST_RING 6 5 ---help--- 7 6 This kernel module can be loaded in host kernel to accelerate 8 7 guest networking with virtio_net. Not to be confused with virtio_net ··· 14 15 tristate "VHOST_SCSI TCM fabric driver" 15 16 depends on TARGET_CORE && EVENTFD && m 16 17 select VHOST 17 17 - select VHOST_RING 18 18 default n 19 19 ---help--- 20 20 Say M here to enable the vhost_scsi TCM fabric module 21 21 for use with virtio-scsi guests 22 22 23 23 - config VHOST_RING 24 24 - tristate 23 23 + config VHOST_VSOCK 24 24 + tristate "vhost virtio-vsock driver" 25 25 + depends on VSOCKETS && EVENTFD 26 26 + select VIRTIO_VSOCKETS_COMMON 27 27 + select VHOST 28 28 + default n 25 29 ---help--- 26 26 - This option is selected by any driver which needs to access 27 27 - the host side of a virtio ring. 30 30 + This kernel module can be loaded in the host kernel to provide AF_VSOCK 31 31 + sockets for communicating with guests. The guests must have the 32 32 + virtio_transport.ko driver loaded to use the virtio-vsock device. 33 33 + 34 34 + To compile this driver as a module, choose M here: the module will be called 35 35 + vhost_vsock. 28 36 29 37 config VHOST 30 38 tristate

+5

drivers/vhost/Kconfig.vringh

reviewed

··· 1 1 + config VHOST_RING 2 2 + tristate 3 3 + ---help--- 4 4 + This option is selected by any driver which needs to access 5 5 + the host side of a virtio ring.

+4

drivers/vhost/Makefile

reviewed

··· 4 4 obj-$(CONFIG_VHOST_SCSI) += vhost_scsi.o 5 5 vhost_scsi-y := scsi.o 6 6 7 7 + obj-$(CONFIG_VHOST_VSOCK) += vhost_vsock.o 8 8 + vhost_vsock-y := vsock.o 9 9 + 7 10 obj-$(CONFIG_VHOST_RING) += vringh.o 11 11 + 8 12 obj-$(CONFIG_VHOST) += vhost.o

+57 -10

drivers/vhost/net.c

reviewed

··· 61 61 enum { 62 62 VHOST_NET_FEATURES = VHOST_FEATURES | 63 63 (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) | 64 64 - (1ULL << VIRTIO_NET_F_MRG_RXBUF) 64 64 + (1ULL << VIRTIO_NET_F_MRG_RXBUF) | 65 65 + (1ULL << VIRTIO_F_IOMMU_PLATFORM) 65 66 }; 66 67 67 68 enum { ··· 335 334 { 336 335 unsigned long uninitialized_var(endtime); 337 336 int r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), 338 338 - out_num, in_num, NULL, NULL); 337 337 + out_num, in_num, NULL, NULL); 339 338 340 339 if (r == vq->num && vq->busyloop_timeout) { 341 340 preempt_disable(); ··· 345 344 cpu_relax_lowlatency(); 346 345 preempt_enable(); 347 346 r = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), 348 348 - out_num, in_num, NULL, NULL); 347 347 + out_num, in_num, NULL, NULL); 349 348 } 350 349 351 350 return r; ··· 376 375 mutex_lock(&vq->mutex); 377 376 sock = vq->private_data; 378 377 if (!sock) 378 378 + goto out; 379 379 + 380 380 + if (!vq_iotlb_prefetch(vq)) 379 381 goto out; 380 382 381 383 vhost_disable_notify(&net->dev, vq); ··· 656 652 sock = vq->private_data; 657 653 if (!sock) 658 654 goto out; 655 655 + 656 656 + if (!vq_iotlb_prefetch(vq)) 657 657 + goto out; 658 658 + 659 659 vhost_disable_notify(&net->dev, vq); 660 660 vhost_net_disable_vq(net, vq); 661 661 ··· 1060 1052 struct socket *tx_sock = NULL; 1061 1053 struct socket *rx_sock = NULL; 1062 1054 long err; 1063 1063 - struct vhost_memory *memory; 1055 1055 + struct vhost_umem *umem; 1064 1056 1065 1057 mutex_lock(&n->dev.mutex); 1066 1058 err = vhost_dev_check_owner(&n->dev); 1067 1059 if (err) 1068 1060 goto done; 1069 1069 - memory = vhost_dev_reset_owner_prepare(); 1070 1070 - if (!memory) { 1061 1061 + umem = vhost_dev_reset_owner_prepare(); 1062 1062 + if (!umem) { 1071 1063 err = -ENOMEM; 1072 1064 goto done; 1073 1065 } 1074 1066 vhost_net_stop(n, &tx_sock, &rx_sock); 1075 1067 vhost_net_flush(n); 1076 1076 - vhost_dev_reset_owner(&n->dev, memory); 1068 1068 + vhost_dev_reset_owner(&n->dev, umem); 1077 1069 vhost_net_vq_reset(n); 1078 1070 done: 1079 1071 mutex_unlock(&n->dev.mutex); ··· 1104 1096 } 1105 1097 mutex_lock(&n->dev.mutex); 1106 1098 if ((features & (1 << VHOST_F_LOG_ALL)) && 1107 1107 - !vhost_log_access_ok(&n->dev)) { 1108 1108 - mutex_unlock(&n->dev.mutex); 1109 1109 - return -EFAULT; 1099 1099 + !vhost_log_access_ok(&n->dev)) 1100 1100 + goto out_unlock; 1101 1101 + 1102 1102 + if ((features & (1ULL << VIRTIO_F_IOMMU_PLATFORM))) { 1103 1103 + if (vhost_init_device_iotlb(&n->dev, true)) 1104 1104 + goto out_unlock; 1110 1105 } 1106 1106 + 1111 1107 for (i = 0; i < VHOST_NET_VQ_MAX; ++i) { 1112 1108 mutex_lock(&n->vqs[i].vq.mutex); 1113 1109 n->vqs[i].vq.acked_features = features; ··· 1121 1109 } 1122 1110 mutex_unlock(&n->dev.mutex); 1123 1111 return 0; 1112 1112 + 1113 1113 + out_unlock: 1114 1114 + mutex_unlock(&n->dev.mutex); 1115 1115 + return -EFAULT; 1124 1116 } 1125 1117 1126 1118 static long vhost_net_set_owner(struct vhost_net *n) ··· 1198 1182 } 1199 1183 #endif 1200 1184 1185 1185 + static ssize_t vhost_net_chr_read_iter(struct kiocb *iocb, struct iov_iter *to) 1186 1186 + { 1187 1187 + struct file *file = iocb->ki_filp; 1188 1188 + struct vhost_net *n = file->private_data; 1189 1189 + struct vhost_dev *dev = &n->dev; 1190 1190 + int noblock = file->f_flags & O_NONBLOCK; 1191 1191 + 1192 1192 + return vhost_chr_read_iter(dev, to, noblock); 1193 1193 + } 1194 1194 + 1195 1195 + static ssize_t vhost_net_chr_write_iter(struct kiocb *iocb, 1196 1196 + struct iov_iter *from) 1197 1197 + { 1198 1198 + struct file *file = iocb->ki_filp; 1199 1199 + struct vhost_net *n = file->private_data; 1200 1200 + struct vhost_dev *dev = &n->dev; 1201 1201 + 1202 1202 + return vhost_chr_write_iter(dev, from); 1203 1203 + } 1204 1204 + 1205 1205 + static unsigned int vhost_net_chr_poll(struct file *file, poll_table *wait) 1206 1206 + { 1207 1207 + struct vhost_net *n = file->private_data; 1208 1208 + struct vhost_dev *dev = &n->dev; 1209 1209 + 1210 1210 + return vhost_chr_poll(file, dev, wait); 1211 1211 + } 1212 1212 + 1201 1213 static const struct file_operations vhost_net_fops = { 1202 1214 .owner = THIS_MODULE, 1203 1215 .release = vhost_net_release, 1216 1216 + .read_iter = vhost_net_chr_read_iter, 1217 1217 + .write_iter = vhost_net_chr_write_iter, 1218 1218 + .poll = vhost_net_chr_poll, 1204 1219 .unlocked_ioctl = vhost_net_ioctl, 1205 1220 #ifdef CONFIG_COMPAT 1206 1221 .compat_ioctl = vhost_net_compat_ioctl,

+758 -169

drivers/vhost/vhost.c

reviewed

··· 27 27 #include <linux/cgroup.h> 28 28 #include <linux/module.h> 29 29 #include <linux/sort.h> 30 30 + #include <linux/interval_tree_generic.h> 30 31 31 32 #include "vhost.h" 32 33 ··· 35 34 module_param(max_mem_regions, ushort, 0444); 36 35 MODULE_PARM_DESC(max_mem_regions, 37 36 "Maximum number of memory regions in memory map. (default: 64)"); 37 37 + static int max_iotlb_entries = 2048; 38 38 + module_param(max_iotlb_entries, int, 0444); 39 39 + MODULE_PARM_DESC(max_iotlb_entries, 40 40 + "Maximum number of iotlb entries. (default: 2048)"); 38 41 39 42 enum { 40 43 VHOST_MEMORY_F_LOG = 0x1, ··· 46 41 47 42 #define vhost_used_event(vq) ((__virtio16 __user *)&vq->avail->ring[vq->num]) 48 43 #define vhost_avail_event(vq) ((__virtio16 __user *)&vq->used->ring[vq->num]) 44 44 + 45 45 + INTERVAL_TREE_DEFINE(struct vhost_umem_node, 46 46 + rb, __u64, __subtree_last, 47 47 + START, LAST, , vhost_umem_interval_tree); 49 48 50 49 #ifdef CONFIG_VHOST_CROSS_ENDIAN_LEGACY 51 50 static void vhost_disable_cross_endian(struct vhost_virtqueue *vq) ··· 140 131 vq->is_le = virtio_legacy_is_little_endian(); 141 132 } 142 133 134 134 + struct vhost_flush_struct { 135 135 + struct vhost_work work; 136 136 + struct completion wait_event; 137 137 + }; 138 138 + 139 139 + static void vhost_flush_work(struct vhost_work *work) 140 140 + { 141 141 + struct vhost_flush_struct *s; 142 142 + 143 143 + s = container_of(work, struct vhost_flush_struct, work); 144 144 + complete(&s->wait_event); 145 145 + } 146 146 + 143 147 static void vhost_poll_func(struct file *file, wait_queue_head_t *wqh, 144 148 poll_table *pt) 145 149 { ··· 177 155 178 156 void vhost_work_init(struct vhost_work *work, vhost_work_fn_t fn) 179 157 { 180 180 - INIT_LIST_HEAD(&work->node); 158 158 + clear_bit(VHOST_WORK_QUEUED, &work->flags); 181 159 work->fn = fn; 182 160 init_waitqueue_head(&work->done); 183 183 - work->flushing = 0; 184 184 - work->queue_seq = work->done_seq = 0; 185 161 } 186 162 EXPORT_SYMBOL_GPL(vhost_work_init); 187 163 ··· 231 211 } 232 212 EXPORT_SYMBOL_GPL(vhost_poll_stop); 233 213 234 234 - static bool vhost_work_seq_done(struct vhost_dev *dev, struct vhost_work *work, 235 235 - unsigned seq) 236 236 - { 237 237 - int left; 238 238 - 239 239 - spin_lock_irq(&dev->work_lock); 240 240 - left = seq - work->done_seq; 241 241 - spin_unlock_irq(&dev->work_lock); 242 242 - return left <= 0; 243 243 - } 244 244 - 245 214 void vhost_work_flush(struct vhost_dev *dev, struct vhost_work *work) 246 215 { 247 247 - unsigned seq; 248 248 - int flushing; 216 216 + struct vhost_flush_struct flush; 249 217 250 250 - spin_lock_irq(&dev->work_lock); 251 251 - seq = work->queue_seq; 252 252 - work->flushing++; 253 253 - spin_unlock_irq(&dev->work_lock); 254 254 - wait_event(work->done, vhost_work_seq_done(dev, work, seq)); 255 255 - spin_lock_irq(&dev->work_lock); 256 256 - flushing = --work->flushing; 257 257 - spin_unlock_irq(&dev->work_lock); 258 258 - BUG_ON(flushing < 0); 218 218 + if (dev->worker) { 219 219 + init_completion(&flush.wait_event); 220 220 + vhost_work_init(&flush.work, vhost_flush_work); 221 221 + 222 222 + vhost_work_queue(dev, &flush.work); 223 223 + wait_for_completion(&flush.wait_event); 224 224 + } 259 225 } 260 226 EXPORT_SYMBOL_GPL(vhost_work_flush); 261 227 ··· 255 249 256 250 void vhost_work_queue(struct vhost_dev *dev, struct vhost_work *work) 257 251 { 258 258 - unsigned long flags; 252 252 + if (!dev->worker) 253 253 + return; 259 254 260 260 - spin_lock_irqsave(&dev->work_lock, flags); 261 261 - if (list_empty(&work->node)) { 262 262 - list_add_tail(&work->node, &dev->work_list); 263 263 - work->queue_seq++; 264 264 - spin_unlock_irqrestore(&dev->work_lock, flags); 255 255 + if (!test_and_set_bit(VHOST_WORK_QUEUED, &work->flags)) { 256 256 + /* We can only add the work to the list after we're 257 257 + * sure it was not in the list. 258 258 + */ 259 259 + smp_mb(); 260 260 + llist_add(&work->node, &dev->work_list); 265 261 wake_up_process(dev->worker); 266 266 - } else { 267 267 - spin_unlock_irqrestore(&dev->work_lock, flags); 268 262 } 269 263 } 270 264 EXPORT_SYMBOL_GPL(vhost_work_queue); ··· 272 266 /* A lockless hint for busy polling code to exit the loop */ 273 267 bool vhost_has_work(struct vhost_dev *dev) 274 268 { 275 275 - return !list_empty(&dev->work_list); 269 269 + return !llist_empty(&dev->work_list); 276 270 } 277 271 EXPORT_SYMBOL_GPL(vhost_has_work); 278 272 ··· 306 300 vq->call_ctx = NULL; 307 301 vq->call = NULL; 308 302 vq->log_ctx = NULL; 309 309 - vq->memory = NULL; 310 303 vhost_reset_is_le(vq); 311 304 vhost_disable_cross_endian(vq); 312 305 vq->busyloop_timeout = 0; 306 306 + vq->umem = NULL; 307 307 + vq->iotlb = NULL; 313 308 } 314 309 315 310 static int vhost_worker(void *data) 316 311 { 317 312 struct vhost_dev *dev = data; 318 318 - struct vhost_work *work = NULL; 319 319 - unsigned uninitialized_var(seq); 313 313 + struct vhost_work *work, *work_next; 314 314 + struct llist_node *node; 320 315 mm_segment_t oldfs = get_fs(); 321 316 322 317 set_fs(USER_DS); ··· 327 320 /* mb paired w/ kthread_stop */ 328 321 set_current_state(TASK_INTERRUPTIBLE); 329 322 330 330 - spin_lock_irq(&dev->work_lock); 331 331 - if (work) { 332 332 - work->done_seq = seq; 333 333 - if (work->flushing) 334 334 - wake_up_all(&work->done); 335 335 - } 336 336 - 337 323 if (kthread_should_stop()) { 338 338 - spin_unlock_irq(&dev->work_lock); 339 324 __set_current_state(TASK_RUNNING); 340 325 break; 341 326 } 342 342 - if (!list_empty(&dev->work_list)) { 343 343 - work = list_first_entry(&dev->work_list, 344 344 - struct vhost_work, node); 345 345 - list_del_init(&work->node); 346 346 - seq = work->queue_seq; 347 347 - } else 348 348 - work = NULL; 349 349 - spin_unlock_irq(&dev->work_lock); 350 327 351 351 - if (work) { 328 328 + node = llist_del_all(&dev->work_list); 329 329 + if (!node) 330 330 + schedule(); 331 331 + 332 332 + node = llist_reverse_order(node); 333 333 + /* make sure flag is seen after deletion */ 334 334 + smp_wmb(); 335 335 + llist_for_each_entry_safe(work, work_next, node, node) { 336 336 + clear_bit(VHOST_WORK_QUEUED, &work->flags); 352 337 __set_current_state(TASK_RUNNING); 353 338 work->fn(work); 354 339 if (need_resched()) 355 340 schedule(); 356 356 - } else 357 357 - schedule(); 358 358 - 341 341 + } 359 342 } 360 343 unuse_mm(dev->mm); 361 344 set_fs(oldfs); ··· 404 407 mutex_init(&dev->mutex); 405 408 dev->log_ctx = NULL; 406 409 dev->log_file = NULL; 407 407 - dev->memory = NULL; 410 410 + dev->umem = NULL; 411 411 + dev->iotlb = NULL; 408 412 dev->mm = NULL; 409 409 - spin_lock_init(&dev->work_lock); 410 410 - INIT_LIST_HEAD(&dev->work_list); 411 413 dev->worker = NULL; 414 414 + init_llist_head(&dev->work_list); 415 415 + init_waitqueue_head(&dev->wait); 416 416 + INIT_LIST_HEAD(&dev->read_list); 417 417 + INIT_LIST_HEAD(&dev->pending_list); 418 418 + spin_lock_init(&dev->iotlb_lock); 419 419 + 412 420 413 421 for (i = 0; i < dev->nvqs; ++i) { 414 422 vq = dev->vqs[i]; ··· 514 512 } 515 513 EXPORT_SYMBOL_GPL(vhost_dev_set_owner); 516 514 517 517 - struct vhost_memory *vhost_dev_reset_owner_prepare(void) 515 515 + static void *vhost_kvzalloc(unsigned long size) 518 516 { 519 519 - return kmalloc(offsetof(struct vhost_memory, regions), GFP_KERNEL); 517 517 + void *n = kzalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 518 518 + 519 519 + if (!n) 520 520 + n = vzalloc(size); 521 521 + return n; 522 522 + } 523 523 + 524 524 + struct vhost_umem *vhost_dev_reset_owner_prepare(void) 525 525 + { 526 526 + return vhost_kvzalloc(sizeof(struct vhost_umem)); 520 527 } 521 528 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner_prepare); 522 529 523 530 /* Caller should have device mutex */ 524 524 - void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_memory *memory) 531 531 + void vhost_dev_reset_owner(struct vhost_dev *dev, struct vhost_umem *umem) 525 532 { 526 533 int i; 527 534 528 535 vhost_dev_cleanup(dev, true); 529 536 530 537 /* Restore memory to default empty mapping. */ 531 531 - memory->nregions = 0; 532 532 - dev->memory = memory; 538 538 + INIT_LIST_HEAD(&umem->umem_list); 539 539 + dev->umem = umem; 533 540 /* We don't need VQ locks below since vhost_dev_cleanup makes sure 534 541 * VQs aren't running. 535 542 */ 536 543 for (i = 0; i < dev->nvqs; ++i) 537 537 - dev->vqs[i]->memory = memory; 544 544 + dev->vqs[i]->umem = umem; 538 545 } 539 546 EXPORT_SYMBOL_GPL(vhost_dev_reset_owner); 540 547 ··· 559 548 } 560 549 } 561 550 EXPORT_SYMBOL_GPL(vhost_dev_stop); 551 551 + 552 552 + static void vhost_umem_free(struct vhost_umem *umem, 553 553 + struct vhost_umem_node *node) 554 554 + { 555 555 + vhost_umem_interval_tree_remove(node, &umem->umem_tree); 556 556 + list_del(&node->link); 557 557 + kfree(node); 558 558 + umem->numem--; 559 559 + } 560 560 + 561 561 + static void vhost_umem_clean(struct vhost_umem *umem) 562 562 + { 563 563 + struct vhost_umem_node *node, *tmp; 564 564 + 565 565 + if (!umem) 566 566 + return; 567 567 + 568 568 + list_for_each_entry_safe(node, tmp, &umem->umem_list, link) 569 569 + vhost_umem_free(umem, node); 570 570 + 571 571 + kvfree(umem); 572 572 + } 573 573 + 574 574 + static void vhost_clear_msg(struct vhost_dev *dev) 575 575 + { 576 576 + struct vhost_msg_node *node, *n; 577 577 + 578 578 + spin_lock(&dev->iotlb_lock); 579 579 + 580 580 + list_for_each_entry_safe(node, n, &dev->read_list, node) { 581 581 + list_del(&node->node); 582 582 + kfree(node); 583 583 + } 584 584 + 585 585 + list_for_each_entry_safe(node, n, &dev->pending_list, node) { 586 586 + list_del(&node->node); 587 587 + kfree(node); 588 588 + } 589 589 + 590 590 + spin_unlock(&dev->iotlb_lock); 591 591 + } 562 592 563 593 /* Caller should have device mutex if and only if locked is set */ 564 594 void vhost_dev_cleanup(struct vhost_dev *dev, bool locked) ··· 627 575 fput(dev->log_file); 628 576 dev->log_file = NULL; 629 577 /* No one will access memory at this point */ 630 630 - kvfree(dev->memory); 631 631 - dev->memory = NULL; 632 632 - WARN_ON(!list_empty(&dev->work_list)); 578 578 + vhost_umem_clean(dev->umem); 579 579 + dev->umem = NULL; 580 580 + vhost_umem_clean(dev->iotlb); 581 581 + dev->iotlb = NULL; 582 582 + vhost_clear_msg(dev); 583 583 + wake_up_interruptible_poll(&dev->wait, POLLIN | POLLRDNORM); 584 584 + WARN_ON(!llist_empty(&dev->work_list)); 633 585 if (dev->worker) { 634 586 kthread_stop(dev->worker); 635 587 dev->worker = NULL; ··· 657 601 (sz + VHOST_PAGE_SIZE * 8 - 1) / VHOST_PAGE_SIZE / 8); 658 602 } 659 603 604 604 + static bool vhost_overflow(u64 uaddr, u64 size) 605 605 + { 606 606 + /* Make sure 64 bit math will not overflow. */ 607 607 + return uaddr > ULONG_MAX || size > ULONG_MAX || uaddr > ULONG_MAX - size; 608 608 + } 609 609 + 660 610 /* Caller should have vq mutex and device mutex. */ 661 661 - static int vq_memory_access_ok(void __user *log_base, struct vhost_memory *mem, 611 611 + static int vq_memory_access_ok(void __user *log_base, struct vhost_umem *umem, 662 612 int log_all) 663 613 { 664 664 - int i; 614 614 + struct vhost_umem_node *node; 665 615 666 666 - if (!mem) 616 616 + if (!umem) 667 617 return 0; 668 618 669 669 - for (i = 0; i < mem->nregions; ++i) { 670 670 - struct vhost_memory_region *m = mem->regions + i; 671 671 - unsigned long a = m->userspace_addr; 672 672 - if (m->memory_size > ULONG_MAX) 619 619 + list_for_each_entry(node, &umem->umem_list, link) { 620 620 + unsigned long a = node->userspace_addr; 621 621 + 622 622 + if (vhost_overflow(node->userspace_addr, node->size)) 673 623 return 0; 674 674 - else if (!access_ok(VERIFY_WRITE, (void __user *)a, 675 675 - m->memory_size)) 624 624 + 625 625 + 626 626 + if (!access_ok(VERIFY_WRITE, (void __user *)a, 627 627 + node->size)) 676 628 return 0; 677 629 else if (log_all && !log_access_ok(log_base, 678 678 - m->guest_phys_addr, 679 679 - m->memory_size)) 630 630 + node->start, 631 631 + node->size)) 680 632 return 0; 681 633 } 682 634 return 1; ··· 692 628 693 629 /* Can we switch to this memory table? */ 694 630 /* Caller should have device mutex but not vq mutex */ 695 695 - static int memory_access_ok(struct vhost_dev *d, struct vhost_memory *mem, 631 631 + static int memory_access_ok(struct vhost_dev *d, struct vhost_umem *umem, 696 632 int log_all) 697 633 { 698 634 int i; ··· 705 641 log = log_all || vhost_has_feature(d->vqs[i], VHOST_F_LOG_ALL); 706 642 /* If ring is inactive, will check when it's enabled. */ 707 643 if (d->vqs[i]->private_data) 708 708 - ok = vq_memory_access_ok(d->vqs[i]->log_base, mem, log); 644 644 + ok = vq_memory_access_ok(d->vqs[i]->log_base, 645 645 + umem, log); 709 646 else 710 647 ok = 1; 711 648 mutex_unlock(&d->vqs[i]->mutex); ··· 716 651 return 1; 717 652 } 718 653 654 654 + static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, 655 655 + struct iovec iov[], int iov_size, int access); 656 656 + 657 657 + static int vhost_copy_to_user(struct vhost_virtqueue *vq, void *to, 658 658 + const void *from, unsigned size) 659 659 + { 660 660 + int ret; 661 661 + 662 662 + if (!vq->iotlb) 663 663 + return __copy_to_user(to, from, size); 664 664 + else { 665 665 + /* This function should be called after iotlb 666 666 + * prefetch, which means we're sure that all vq 667 667 + * could be access through iotlb. So -EAGAIN should 668 668 + * not happen in this case. 669 669 + */ 670 670 + /* TODO: more fast path */ 671 671 + struct iov_iter t; 672 672 + ret = translate_desc(vq, (u64)(uintptr_t)to, size, vq->iotlb_iov, 673 673 + ARRAY_SIZE(vq->iotlb_iov), 674 674 + VHOST_ACCESS_WO); 675 675 + if (ret < 0) 676 676 + goto out; 677 677 + iov_iter_init(&t, WRITE, vq->iotlb_iov, ret, size); 678 678 + ret = copy_to_iter(from, size, &t); 679 679 + if (ret == size) 680 680 + ret = 0; 681 681 + } 682 682 + out: 683 683 + return ret; 684 684 + } 685 685 + 686 686 + static int vhost_copy_from_user(struct vhost_virtqueue *vq, void *to, 687 687 + void *from, unsigned size) 688 688 + { 689 689 + int ret; 690 690 + 691 691 + if (!vq->iotlb) 692 692 + return __copy_from_user(to, from, size); 693 693 + else { 694 694 + /* This function should be called after iotlb 695 695 + * prefetch, which means we're sure that vq 696 696 + * could be access through iotlb. So -EAGAIN should 697 697 + * not happen in this case. 698 698 + */ 699 699 + /* TODO: more fast path */ 700 700 + struct iov_iter f; 701 701 + ret = translate_desc(vq, (u64)(uintptr_t)from, size, vq->iotlb_iov, 702 702 + ARRAY_SIZE(vq->iotlb_iov), 703 703 + VHOST_ACCESS_RO); 704 704 + if (ret < 0) { 705 705 + vq_err(vq, "IOTLB translation failure: uaddr " 706 706 + "%p size 0x%llx\n", from, 707 707 + (unsigned long long) size); 708 708 + goto out; 709 709 + } 710 710 + iov_iter_init(&f, READ, vq->iotlb_iov, ret, size); 711 711 + ret = copy_from_iter(to, size, &f); 712 712 + if (ret == size) 713 713 + ret = 0; 714 714 + } 715 715 + 716 716 + out: 717 717 + return ret; 718 718 + } 719 719 + 720 720 + static void __user *__vhost_get_user(struct vhost_virtqueue *vq, 721 721 + void *addr, unsigned size) 722 722 + { 723 723 + int ret; 724 724 + 725 725 + /* This function should be called after iotlb 726 726 + * prefetch, which means we're sure that vq 727 727 + * could be access through iotlb. So -EAGAIN should 728 728 + * not happen in this case. 729 729 + */ 730 730 + /* TODO: more fast path */ 731 731 + ret = translate_desc(vq, (u64)(uintptr_t)addr, size, vq->iotlb_iov, 732 732 + ARRAY_SIZE(vq->iotlb_iov), 733 733 + VHOST_ACCESS_RO); 734 734 + if (ret < 0) { 735 735 + vq_err(vq, "IOTLB translation failure: uaddr " 736 736 + "%p size 0x%llx\n", addr, 737 737 + (unsigned long long) size); 738 738 + return NULL; 739 739 + } 740 740 + 741 741 + if (ret != 1 || vq->iotlb_iov[0].iov_len != size) { 742 742 + vq_err(vq, "Non atomic userspace memory access: uaddr " 743 743 + "%p size 0x%llx\n", addr, 744 744 + (unsigned long long) size); 745 745 + return NULL; 746 746 + } 747 747 + 748 748 + return vq->iotlb_iov[0].iov_base; 749 749 + } 750 750 + 751 751 + #define vhost_put_user(vq, x, ptr) \ 752 752 + ({ \ 753 753 + int ret = -EFAULT; \ 754 754 + if (!vq->iotlb) { \ 755 755 + ret = __put_user(x, ptr); \ 756 756 + } else { \ 757 757 + __typeof__(ptr) to = \ 758 758 + (__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \ 759 759 + if (to != NULL) \ 760 760 + ret = __put_user(x, to); \ 761 761 + else \ 762 762 + ret = -EFAULT; \ 763 763 + } \ 764 764 + ret; \ 765 765 + }) 766 766 + 767 767 + #define vhost_get_user(vq, x, ptr) \ 768 768 + ({ \ 769 769 + int ret; \ 770 770 + if (!vq->iotlb) { \ 771 771 + ret = __get_user(x, ptr); \ 772 772 + } else { \ 773 773 + __typeof__(ptr) from = \ 774 774 + (__typeof__(ptr)) __vhost_get_user(vq, ptr, sizeof(*ptr)); \ 775 775 + if (from != NULL) \ 776 776 + ret = __get_user(x, from); \ 777 777 + else \ 778 778 + ret = -EFAULT; \ 779 779 + } \ 780 780 + ret; \ 781 781 + }) 782 782 + 783 783 + static void vhost_dev_lock_vqs(struct vhost_dev *d) 784 784 + { 785 785 + int i = 0; 786 786 + for (i = 0; i < d->nvqs; ++i) 787 787 + mutex_lock(&d->vqs[i]->mutex); 788 788 + } 789 789 + 790 790 + static void vhost_dev_unlock_vqs(struct vhost_dev *d) 791 791 + { 792 792 + int i = 0; 793 793 + for (i = 0; i < d->nvqs; ++i) 794 794 + mutex_unlock(&d->vqs[i]->mutex); 795 795 + } 796 796 + 797 797 + static int vhost_new_umem_range(struct vhost_umem *umem, 798 798 + u64 start, u64 size, u64 end, 799 799 + u64 userspace_addr, int perm) 800 800 + { 801 801 + struct vhost_umem_node *tmp, *node = kmalloc(sizeof(*node), GFP_ATOMIC); 802 802 + 803 803 + if (!node) 804 804 + return -ENOMEM; 805 805 + 806 806 + if (umem->numem == max_iotlb_entries) { 807 807 + tmp = list_first_entry(&umem->umem_list, typeof(*tmp), link); 808 808 + vhost_umem_free(umem, tmp); 809 809 + } 810 810 + 811 811 + node->start = start; 812 812 + node->size = size; 813 813 + node->last = end; 814 814 + node->userspace_addr = userspace_addr; 815 815 + node->perm = perm; 816 816 + INIT_LIST_HEAD(&node->link); 817 817 + list_add_tail(&node->link, &umem->umem_list); 818 818 + vhost_umem_interval_tree_insert(node, &umem->umem_tree); 819 819 + umem->numem++; 820 820 + 821 821 + return 0; 822 822 + } 823 823 + 824 824 + static void vhost_del_umem_range(struct vhost_umem *umem, 825 825 + u64 start, u64 end) 826 826 + { 827 827 + struct vhost_umem_node *node; 828 828 + 829 829 + while ((node = vhost_umem_interval_tree_iter_first(&umem->umem_tree, 830 830 + start, end))) 831 831 + vhost_umem_free(umem, node); 832 832 + } 833 833 + 834 834 + static void vhost_iotlb_notify_vq(struct vhost_dev *d, 835 835 + struct vhost_iotlb_msg *msg) 836 836 + { 837 837 + struct vhost_msg_node *node, *n; 838 838 + 839 839 + spin_lock(&d->iotlb_lock); 840 840 + 841 841 + list_for_each_entry_safe(node, n, &d->pending_list, node) { 842 842 + struct vhost_iotlb_msg *vq_msg = &node->msg.iotlb; 843 843 + if (msg->iova <= vq_msg->iova && 844 844 + msg->iova + msg->size - 1 > vq_msg->iova && 845 845 + vq_msg->type == VHOST_IOTLB_MISS) { 846 846 + vhost_poll_queue(&node->vq->poll); 847 847 + list_del(&node->node); 848 848 + kfree(node); 849 849 + } 850 850 + } 851 851 + 852 852 + spin_unlock(&d->iotlb_lock); 853 853 + } 854 854 + 855 855 + static int umem_access_ok(u64 uaddr, u64 size, int access) 856 856 + { 857 857 + unsigned long a = uaddr; 858 858 + 859 859 + /* Make sure 64 bit math will not overflow. */ 860 860 + if (vhost_overflow(uaddr, size)) 861 861 + return -EFAULT; 862 862 + 863 863 + if ((access & VHOST_ACCESS_RO) && 864 864 + !access_ok(VERIFY_READ, (void __user *)a, size)) 865 865 + return -EFAULT; 866 866 + if ((access & VHOST_ACCESS_WO) && 867 867 + !access_ok(VERIFY_WRITE, (void __user *)a, size)) 868 868 + return -EFAULT; 869 869 + return 0; 870 870 + } 871 871 + 872 872 + int vhost_process_iotlb_msg(struct vhost_dev *dev, 873 873 + struct vhost_iotlb_msg *msg) 874 874 + { 875 875 + int ret = 0; 876 876 + 877 877 + vhost_dev_lock_vqs(dev); 878 878 + switch (msg->type) { 879 879 + case VHOST_IOTLB_UPDATE: 880 880 + if (!dev->iotlb) { 881 881 + ret = -EFAULT; 882 882 + break; 883 883 + } 884 884 + if (umem_access_ok(msg->uaddr, msg->size, msg->perm)) { 885 885 + ret = -EFAULT; 886 886 + break; 887 887 + } 888 888 + if (vhost_new_umem_range(dev->iotlb, msg->iova, msg->size, 889 889 + msg->iova + msg->size - 1, 890 890 + msg->uaddr, msg->perm)) { 891 891 + ret = -ENOMEM; 892 892 + break; 893 893 + } 894 894 + vhost_iotlb_notify_vq(dev, msg); 895 895 + break; 896 896 + case VHOST_IOTLB_INVALIDATE: 897 897 + vhost_del_umem_range(dev->iotlb, msg->iova, 898 898 + msg->iova + msg->size - 1); 899 899 + break; 900 900 + default: 901 901 + ret = -EINVAL; 902 902 + break; 903 903 + } 904 904 + 905 905 + vhost_dev_unlock_vqs(dev); 906 906 + return ret; 907 907 + } 908 908 + ssize_t vhost_chr_write_iter(struct vhost_dev *dev, 909 909 + struct iov_iter *from) 910 910 + { 911 911 + struct vhost_msg_node node; 912 912 + unsigned size = sizeof(struct vhost_msg); 913 913 + size_t ret; 914 914 + int err; 915 915 + 916 916 + if (iov_iter_count(from) < size) 917 917 + return 0; 918 918 + ret = copy_from_iter(&node.msg, size, from); 919 919 + if (ret != size) 920 920 + goto done; 921 921 + 922 922 + switch (node.msg.type) { 923 923 + case VHOST_IOTLB_MSG: 924 924 + err = vhost_process_iotlb_msg(dev, &node.msg.iotlb); 925 925 + if (err) 926 926 + ret = err; 927 927 + break; 928 928 + default: 929 929 + ret = -EINVAL; 930 930 + break; 931 931 + } 932 932 + 933 933 + done: 934 934 + return ret; 935 935 + } 936 936 + EXPORT_SYMBOL(vhost_chr_write_iter); 937 937 + 938 938 + unsigned int vhost_chr_poll(struct file *file, struct vhost_dev *dev, 939 939 + poll_table *wait) 940 940 + { 941 941 + unsigned int mask = 0; 942 942 + 943 943 + poll_wait(file, &dev->wait, wait); 944 944 + 945 945 + if (!list_empty(&dev->read_list)) 946 946 + mask |= POLLIN | POLLRDNORM; 947 947 + 948 948 + return mask; 949 949 + } 950 950 + EXPORT_SYMBOL(vhost_chr_poll); 951 951 + 952 952 + ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to, 953 953 + int noblock) 954 954 + { 955 955 + DEFINE_WAIT(wait); 956 956 + struct vhost_msg_node *node; 957 957 + ssize_t ret = 0; 958 958 + unsigned size = sizeof(struct vhost_msg); 959 959 + 960 960 + if (iov_iter_count(to) < size) 961 961 + return 0; 962 962 + 963 963 + while (1) { 964 964 + if (!noblock) 965 965 + prepare_to_wait(&dev->wait, &wait, 966 966 + TASK_INTERRUPTIBLE); 967 967 + 968 968 + node = vhost_dequeue_msg(dev, &dev->read_list); 969 969 + if (node) 970 970 + break; 971 971 + if (noblock) { 972 972 + ret = -EAGAIN; 973 973 + break; 974 974 + } 975 975 + if (signal_pending(current)) { 976 976 + ret = -ERESTARTSYS; 977 977 + break; 978 978 + } 979 979 + if (!dev->iotlb) { 980 980 + ret = -EBADFD; 981 981 + break; 982 982 + } 983 983 + 984 984 + schedule(); 985 985 + } 986 986 + 987 987 + if (!noblock) 988 988 + finish_wait(&dev->wait, &wait); 989 989 + 990 990 + if (node) { 991 991 + ret = copy_to_iter(&node->msg, size, to); 992 992 + 993 993 + if (ret != size || node->msg.type != VHOST_IOTLB_MISS) { 994 994 + kfree(node); 995 995 + return ret; 996 996 + } 997 997 + 998 998 + vhost_enqueue_msg(dev, &dev->pending_list, node); 999 999 + } 1000 1000 + 1001 1001 + return ret; 1002 1002 + } 1003 1003 + EXPORT_SYMBOL_GPL(vhost_chr_read_iter); 1004 1004 + 1005 1005 + static int vhost_iotlb_miss(struct vhost_virtqueue *vq, u64 iova, int access) 1006 1006 + { 1007 1007 + struct vhost_dev *dev = vq->dev; 1008 1008 + struct vhost_msg_node *node; 1009 1009 + struct vhost_iotlb_msg *msg; 1010 1010 + 1011 1011 + node = vhost_new_msg(vq, VHOST_IOTLB_MISS); 1012 1012 + if (!node) 1013 1013 + return -ENOMEM; 1014 1014 + 1015 1015 + msg = &node->msg.iotlb; 1016 1016 + msg->type = VHOST_IOTLB_MISS; 1017 1017 + msg->iova = iova; 1018 1018 + msg->perm = access; 1019 1019 + 1020 1020 + vhost_enqueue_msg(dev, &dev->read_list, node); 1021 1021 + 1022 1022 + return 0; 1023 1023 + } 1024 1024 + 719 1025 static int vq_access_ok(struct vhost_virtqueue *vq, unsigned int num, 720 1026 struct vring_desc __user *desc, 721 1027 struct vring_avail __user *avail, 722 1028 struct vring_used __user *used) 1029 1029 + 723 1030 { 724 1031 size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 1032 1032 + 725 1033 return access_ok(VERIFY_READ, desc, num * sizeof *desc) && 726 1034 access_ok(VERIFY_READ, avail, 727 1035 sizeof *avail + num * sizeof *avail->ring + s) && ··· 1102 664 sizeof *used + num * sizeof *used->ring + s); 1103 665 } 1104 666 667 667 + static int iotlb_access_ok(struct vhost_virtqueue *vq, 668 668 + int access, u64 addr, u64 len) 669 669 + { 670 670 + const struct vhost_umem_node *node; 671 671 + struct vhost_umem *umem = vq->iotlb; 672 672 + u64 s = 0, size; 673 673 + 674 674 + while (len > s) { 675 675 + node = vhost_umem_interval_tree_iter_first(&umem->umem_tree, 676 676 + addr, 677 677 + addr + len - 1); 678 678 + if (node == NULL || node->start > addr) { 679 679 + vhost_iotlb_miss(vq, addr, access); 680 680 + return false; 681 681 + } else if (!(node->perm & access)) { 682 682 + /* Report the possible access violation by 683 683 + * request another translation from userspace. 684 684 + */ 685 685 + return false; 686 686 + } 687 687 + 688 688 + size = node->size - addr + node->start; 689 689 + s += size; 690 690 + addr += size; 691 691 + } 692 692 + 693 693 + return true; 694 694 + } 695 695 + 696 696 + int vq_iotlb_prefetch(struct vhost_virtqueue *vq) 697 697 + { 698 698 + size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 699 699 + unsigned int num = vq->num; 700 700 + 701 701 + if (!vq->iotlb) 702 702 + return 1; 703 703 + 704 704 + return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, 705 705 + num * sizeof *vq->desc) && 706 706 + iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->avail, 707 707 + sizeof *vq->avail + 708 708 + num * sizeof *vq->avail->ring + s) && 709 709 + iotlb_access_ok(vq, VHOST_ACCESS_WO, (u64)(uintptr_t)vq->used, 710 710 + sizeof *vq->used + 711 711 + num * sizeof *vq->used->ring + s); 712 712 + } 713 713 + EXPORT_SYMBOL_GPL(vq_iotlb_prefetch); 714 714 + 1105 715 /* Can we log writes? */ 1106 716 /* Caller should have device mutex but not vq mutex */ 1107 717 int vhost_log_access_ok(struct vhost_dev *dev) 1108 718 { 1109 1109 - return memory_access_ok(dev, dev->memory, 1); 719 719 + return memory_access_ok(dev, dev->umem, 1); 1110 720 } 1111 721 EXPORT_SYMBOL_GPL(vhost_log_access_ok); 1112 722 ··· 1165 679 { 1166 680 size_t s = vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX) ? 2 : 0; 1167 681 1168 1168 - return vq_memory_access_ok(log_base, vq->memory, 682 682 + return vq_memory_access_ok(log_base, vq->umem, 1169 683 vhost_has_feature(vq, VHOST_F_LOG_ALL)) && 1170 684 (!vq->log_used || log_access_ok(log_base, vq->log_addr, 1171 685 sizeof *vq->used + ··· 1176 690 /* Caller should have vq mutex and device mutex */ 1177 691 int vhost_vq_access_ok(struct vhost_virtqueue *vq) 1178 692 { 693 693 + if (vq->iotlb) { 694 694 + /* When device IOTLB was used, the access validation 695 695 + * will be validated during prefetching. 696 696 + */ 697 697 + return 1; 698 698 + } 1179 699 return vq_access_ok(vq, vq->num, vq->desc, vq->avail, vq->used) && 1180 700 vq_log_access_ok(vq, vq->log_base); 1181 701 } 1182 702 EXPORT_SYMBOL_GPL(vhost_vq_access_ok); 1183 703 1184 1184 - static int vhost_memory_reg_sort_cmp(const void *p1, const void *p2) 704 704 + static struct vhost_umem *vhost_umem_alloc(void) 1185 705 { 1186 1186 - const struct vhost_memory_region *r1 = p1, *r2 = p2; 1187 1187 - if (r1->guest_phys_addr < r2->guest_phys_addr) 1188 1188 - return 1; 1189 1189 - if (r1->guest_phys_addr > r2->guest_phys_addr) 1190 1190 - return -1; 1191 1191 - return 0; 1192 1192 - } 706 706 + struct vhost_umem *umem = vhost_kvzalloc(sizeof(*umem)); 1193 707 1194 1194 - static void *vhost_kvzalloc(unsigned long size) 1195 1195 - { 1196 1196 - void *n = kzalloc(size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 708 708 + if (!umem) 709 709 + return NULL; 1197 710 1198 1198 - if (!n) 1199 1199 - n = vzalloc(size); 1200 1200 - return n; 711 711 + umem->umem_tree = RB_ROOT; 712 712 + umem->numem = 0; 713 713 + INIT_LIST_HEAD(&umem->umem_list); 714 714 + 715 715 + return umem; 1201 716 } 1202 717 1203 718 static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m) 1204 719 { 1205 1205 - struct vhost_memory mem, *newmem, *oldmem; 720 720 + struct vhost_memory mem, *newmem; 721 721 + struct vhost_memory_region *region; 722 722 + struct vhost_umem *newumem, *oldumem; 1206 723 unsigned long size = offsetof(struct vhost_memory, regions); 1207 724 int i; 1208 725 ··· 1225 736 kvfree(newmem); 1226 737 return -EFAULT; 1227 738 } 1228 1228 - sort(newmem->regions, newmem->nregions, sizeof(*newmem->regions), 1229 1229 - vhost_memory_reg_sort_cmp, NULL); 1230 739 1231 1231 - if (!memory_access_ok(d, newmem, 0)) { 740 740 + newumem = vhost_umem_alloc(); 741 741 + if (!newumem) { 1232 742 kvfree(newmem); 1233 1233 - return -EFAULT; 743 743 + return -ENOMEM; 1234 744 } 1235 1235 - oldmem = d->memory; 1236 1236 - d->memory = newmem; 745 745 + 746 746 + for (region = newmem->regions; 747 747 + region < newmem->regions + mem.nregions; 748 748 + region++) { 749 749 + if (vhost_new_umem_range(newumem, 750 750 + region->guest_phys_addr, 751 751 + region->memory_size, 752 752 + region->guest_phys_addr + 753 753 + region->memory_size - 1, 754 754 + region->userspace_addr, 755 755 + VHOST_ACCESS_RW)) 756 756 + goto err; 757 757 + } 758 758 + 759 759 + if (!memory_access_ok(d, newumem, 0)) 760 760 + goto err; 761 761 + 762 762 + oldumem = d->umem; 763 763 + d->umem = newumem; 1237 764 1238 765 /* All memory accesses are done under some VQ mutex. */ 1239 766 for (i = 0; i < d->nvqs; ++i) { 1240 767 mutex_lock(&d->vqs[i]->mutex); 1241 1241 - d->vqs[i]->memory = newmem; 768 768 + d->vqs[i]->umem = newumem; 1242 769 mutex_unlock(&d->vqs[i]->mutex); 1243 770 } 1244 1244 - kvfree(oldmem); 771 771 + 772 772 + kvfree(newmem); 773 773 + vhost_umem_clean(oldumem); 1245 774 return 0; 775 775 + 776 776 + err: 777 777 + vhost_umem_clean(newumem); 778 778 + kvfree(newmem); 779 779 + return -EFAULT; 1246 780 } 1247 781 1248 782 long vhost_vring_ioctl(struct vhost_dev *d, int ioctl, void __user *argp) ··· 1486 974 } 1487 975 EXPORT_SYMBOL_GPL(vhost_vring_ioctl); 1488 976 977 977 + int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled) 978 978 + { 979 979 + struct vhost_umem *niotlb, *oiotlb; 980 980 + int i; 981 981 + 982 982 + niotlb = vhost_umem_alloc(); 983 983 + if (!niotlb) 984 984 + return -ENOMEM; 985 985 + 986 986 + oiotlb = d->iotlb; 987 987 + d->iotlb = niotlb; 988 988 + 989 989 + for (i = 0; i < d->nvqs; ++i) { 990 990 + mutex_lock(&d->vqs[i]->mutex); 991 991 + d->vqs[i]->iotlb = niotlb; 992 992 + mutex_unlock(&d->vqs[i]->mutex); 993 993 + } 994 994 + 995 995 + vhost_umem_clean(oiotlb); 996 996 + 997 997 + return 0; 998 998 + } 999 999 + EXPORT_SYMBOL_GPL(vhost_init_device_iotlb); 1000 1000 + 1489 1001 /* Caller must have device mutex */ 1490 1002 long vhost_dev_ioctl(struct vhost_dev *d, unsigned int ioctl, void __user *argp) 1491 1003 { ··· 1592 1056 } 1593 1057 EXPORT_SYMBOL_GPL(vhost_dev_ioctl); 1594 1058 1595 1595 - static const struct vhost_memory_region *find_region(struct vhost_memory *mem, 1596 1596 - __u64 addr, __u32 len) 1597 1597 - { 1598 1598 - const struct vhost_memory_region *reg; 1599 1599 - int start = 0, end = mem->nregions; 1600 1600 - 1601 1601 - while (start < end) { 1602 1602 - int slot = start + (end - start) / 2; 1603 1603 - reg = mem->regions + slot; 1604 1604 - if (addr >= reg->guest_phys_addr) 1605 1605 - end = slot; 1606 1606 - else 1607 1607 - start = slot + 1; 1608 1608 - } 1609 1609 - 1610 1610 - reg = mem->regions + start; 1611 1611 - if (addr >= reg->guest_phys_addr && 1612 1612 - reg->guest_phys_addr + reg->memory_size > addr) 1613 1613 - return reg; 1614 1614 - return NULL; 1615 1615 - } 1616 1616 - 1617 1059 /* TODO: This is really inefficient. We need something like get_user() 1618 1060 * (instruction directly accesses the data, with an exception table entry 1619 1061 * returning -EFAULT). See Documentation/x86/exception-tables.txt. ··· 1670 1156 static int vhost_update_used_flags(struct vhost_virtqueue *vq) 1671 1157 { 1672 1158 void __user *used; 1673 1673 - if (__put_user(cpu_to_vhost16(vq, vq->used_flags), &vq->used->flags) < 0) 1159 1159 + if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags), 1160 1160 + &vq->used->flags) < 0) 1674 1161 return -EFAULT; 1675 1162 if (unlikely(vq->log_used)) { 1676 1163 /* Make sure the flag is seen before log. */ ··· 1689 1174 1690 1175 static int vhost_update_avail_event(struct vhost_virtqueue *vq, u16 avail_event) 1691 1176 { 1692 1692 - if (__put_user(cpu_to_vhost16(vq, vq->avail_idx), vhost_avail_event(vq))) 1177 1177 + if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx), 1178 1178 + vhost_avail_event(vq))) 1693 1179 return -EFAULT; 1694 1180 if (unlikely(vq->log_used)) { 1695 1181 void __user *used; ··· 1724 1208 if (r) 1725 1209 goto err; 1726 1210 vq->signalled_used_valid = false; 1727 1727 - if (!access_ok(VERIFY_READ, &vq->used->idx, sizeof vq->used->idx)) { 1211 1211 + if (!vq->iotlb && 1212 1212 + !access_ok(VERIFY_READ, &vq->used->idx, sizeof vq->used->idx)) { 1728 1213 r = -EFAULT; 1729 1214 goto err; 1730 1215 } 1731 1731 - r = __get_user(last_used_idx, &vq->used->idx); 1732 1732 - if (r) 1216 1216 + r = vhost_get_user(vq, last_used_idx, &vq->used->idx); 1217 1217 + if (r) { 1218 1218 + vq_err(vq, "Can't access used idx at %p\n", 1219 1219 + &vq->used->idx); 1733 1220 goto err; 1221 1221 + } 1734 1222 vq->last_used_idx = vhost16_to_cpu(vq, last_used_idx); 1735 1223 return 0; 1224 1224 + 1736 1225 err: 1737 1226 vq->is_le = is_le; 1738 1227 return r; ··· 1745 1224 EXPORT_SYMBOL_GPL(vhost_vq_init_access); 1746 1225 1747 1226 static int translate_desc(struct vhost_virtqueue *vq, u64 addr, u32 len, 1748 1748 - struct iovec iov[], int iov_size) 1227 1227 + struct iovec iov[], int iov_size, int access) 1749 1228 { 1750 1750 - const struct vhost_memory_region *reg; 1751 1751 - struct vhost_memory *mem; 1229 1229 + const struct vhost_umem_node *node; 1230 1230 + struct vhost_dev *dev = vq->dev; 1231 1231 + struct vhost_umem *umem = dev->iotlb ? dev->iotlb : dev->umem; 1752 1232 struct iovec *_iov; 1753 1233 u64 s = 0; 1754 1234 int ret = 0; 1755 1235 1756 1756 - mem = vq->memory; 1757 1236 while ((u64)len > s) { 1758 1237 u64 size; 1759 1238 if (unlikely(ret >= iov_size)) { 1760 1239 ret = -ENOBUFS; 1761 1240 break; 1762 1241 } 1763 1763 - reg = find_region(mem, addr, len); 1764 1764 - if (unlikely(!reg)) { 1765 1765 - ret = -EFAULT; 1242 1242 + 1243 1243 + node = vhost_umem_interval_tree_iter_first(&umem->umem_tree, 1244 1244 + addr, addr + len - 1); 1245 1245 + if (node == NULL || node->start > addr) { 1246 1246 + if (umem != dev->iotlb) { 1247 1247 + ret = -EFAULT; 1248 1248 + break; 1249 1249 + } 1250 1250 + ret = -EAGAIN; 1251 1251 + break; 1252 1252 + } else if (!(node->perm & access)) { 1253 1253 + ret = -EPERM; 1766 1254 break; 1767 1255 } 1256 1256 + 1768 1257 _iov = iov + ret; 1769 1769 - size = reg->memory_size - addr + reg->guest_phys_addr; 1258 1258 + size = node->size - addr + node->start; 1770 1259 _iov->iov_len = min((u64)len - s, size); 1771 1260 _iov->iov_base = (void __user *)(unsigned long) 1772 1772 - (reg->userspace_addr + addr - reg->guest_phys_addr); 1261 1261 + (node->userspace_addr + addr - node->start); 1773 1262 s += size; 1774 1263 addr += size; 1775 1264 ++ret; 1776 1265 } 1777 1266 1267 1267 + if (ret == -EAGAIN) 1268 1268 + vhost_iotlb_miss(vq, addr, access); 1778 1269 return ret; 1779 1270 } 1780 1271 ··· 1821 1288 unsigned int i = 0, count, found = 0; 1822 1289 u32 len = vhost32_to_cpu(vq, indirect->len); 1823 1290 struct iov_iter from; 1824 1824 - int ret; 1291 1291 + int ret, access; 1825 1292 1826 1293 /* Sanity check */ 1827 1294 if (unlikely(len % sizeof desc)) { ··· 1833 1300 } 1834 1301 1835 1302 ret = translate_desc(vq, vhost64_to_cpu(vq, indirect->addr), len, vq->indirect, 1836 1836 - UIO_MAXIOV); 1303 1303 + UIO_MAXIOV, VHOST_ACCESS_RO); 1837 1304 if (unlikely(ret < 0)) { 1838 1838 - vq_err(vq, "Translation failure %d in indirect.\n", ret); 1305 1305 + if (ret != -EAGAIN) 1306 1306 + vq_err(vq, "Translation failure %d in indirect.\n", ret); 1839 1307 return ret; 1840 1308 } 1841 1309 iov_iter_init(&from, READ, vq->indirect, ret, len); ··· 1874 1340 return -EINVAL; 1875 1341 } 1876 1342 1343 1343 + if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) 1344 1344 + access = VHOST_ACCESS_WO; 1345 1345 + else 1346 1346 + access = VHOST_ACCESS_RO; 1347 1347 + 1877 1348 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), 1878 1349 vhost32_to_cpu(vq, desc.len), iov + iov_count, 1879 1879 - iov_size - iov_count); 1350 1350 + iov_size - iov_count, access); 1880 1351 if (unlikely(ret < 0)) { 1881 1881 - vq_err(vq, "Translation failure %d indirect idx %d\n", 1882 1882 - ret, i); 1352 1352 + if (ret != -EAGAIN) 1353 1353 + vq_err(vq, "Translation failure %d indirect idx %d\n", 1354 1354 + ret, i); 1883 1355 return ret; 1884 1356 } 1885 1357 /* If this is an input descriptor, increment that count. */ 1886 1886 - if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) { 1358 1358 + if (access == VHOST_ACCESS_WO) { 1887 1359 *in_num += ret; 1888 1360 if (unlikely(log)) { 1889 1361 log[*log_num].addr = vhost64_to_cpu(vq, desc.addr); ··· 1928 1388 u16 last_avail_idx; 1929 1389 __virtio16 avail_idx; 1930 1390 __virtio16 ring_head; 1931 1931 - int ret; 1391 1391 + int ret, access; 1932 1392 1933 1393 /* Check it isn't doing very strange things with descriptor numbers. */ 1934 1394 last_avail_idx = vq->last_avail_idx; 1935 1935 - if (unlikely(__get_user(avail_idx, &vq->avail->idx))) { 1395 1395 + if (unlikely(vhost_get_user(vq, avail_idx, &vq->avail->idx))) { 1936 1396 vq_err(vq, "Failed to access avail idx at %p\n", 1937 1397 &vq->avail->idx); 1938 1398 return -EFAULT; ··· 1954 1414 1955 1415 /* Grab the next descriptor number they're advertising, and increment 1956 1416 * the index we've seen. */ 1957 1957 - if (unlikely(__get_user(ring_head, 1958 1958 - &vq->avail->ring[last_avail_idx & (vq->num - 1)]))) { 1417 1417 + if (unlikely(vhost_get_user(vq, ring_head, 1418 1418 + &vq->avail->ring[last_avail_idx & (vq->num - 1)]))) { 1959 1419 vq_err(vq, "Failed to read head: idx %d address %p\n", 1960 1420 last_avail_idx, 1961 1421 &vq->avail->ring[last_avail_idx % vq->num]); ··· 1990 1450 i, vq->num, head); 1991 1451 return -EINVAL; 1992 1452 } 1993 1993 - ret = __copy_from_user(&desc, vq->desc + i, sizeof desc); 1453 1453 + ret = vhost_copy_from_user(vq, &desc, vq->desc + i, 1454 1454 + sizeof desc); 1994 1455 if (unlikely(ret)) { 1995 1456 vq_err(vq, "Failed to get descriptor: idx %d addr %p\n", 1996 1457 i, vq->desc + i); ··· 2002 1461 out_num, in_num, 2003 1462 log, log_num, &desc); 2004 1463 if (unlikely(ret < 0)) { 2005 2005 - vq_err(vq, "Failure detected " 2006 2006 - "in indirect descriptor at idx %d\n", i); 1464 1464 + if (ret != -EAGAIN) 1465 1465 + vq_err(vq, "Failure detected " 1466 1466 + "in indirect descriptor at idx %d\n", i); 2007 1467 return ret; 2008 1468 } 2009 1469 continue; 2010 1470 } 2011 1471 1472 1472 + if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) 1473 1473 + access = VHOST_ACCESS_WO; 1474 1474 + else 1475 1475 + access = VHOST_ACCESS_RO; 2012 1476 ret = translate_desc(vq, vhost64_to_cpu(vq, desc.addr), 2013 1477 vhost32_to_cpu(vq, desc.len), iov + iov_count, 2014 2014 - iov_size - iov_count); 1478 1478 + iov_size - iov_count, access); 2015 1479 if (unlikely(ret < 0)) { 2016 2016 - vq_err(vq, "Translation failure %d descriptor idx %d\n", 2017 2017 - ret, i); 1480 1480 + if (ret != -EAGAIN) 1481 1481 + vq_err(vq, "Translation failure %d descriptor idx %d\n", 1482 1482 + ret, i); 2018 1483 return ret; 2019 1484 } 2020 2020 - if (desc.flags & cpu_to_vhost16(vq, VRING_DESC_F_WRITE)) { 1485 1485 + if (access == VHOST_ACCESS_WO) { 2021 1486 /* If this is an input descriptor, 2022 1487 * increment that count. */ 2023 1488 *in_num += ret; ··· 2085 1538 start = vq->last_used_idx & (vq->num - 1); 2086 1539 used = vq->used->ring + start; 2087 1540 if (count == 1) { 2088 2088 - if (__put_user(heads[0].id, &used->id)) { 1541 1541 + if (vhost_put_user(vq, heads[0].id, &used->id)) { 2089 1542 vq_err(vq, "Failed to write used id"); 2090 1543 return -EFAULT; 2091 1544 } 2092 2092 - if (__put_user(heads[0].len, &used->len)) { 1545 1545 + if (vhost_put_user(vq, heads[0].len, &used->len)) { 2093 1546 vq_err(vq, "Failed to write used len"); 2094 1547 return -EFAULT; 2095 1548 } 2096 2096 - } else if (__copy_to_user(used, heads, count * sizeof *used)) { 1549 1549 + } else if (vhost_copy_to_user(vq, used, heads, count * sizeof *used)) { 2097 1550 vq_err(vq, "Failed to write used"); 2098 1551 return -EFAULT; 2099 1552 } ··· 2137 1590 2138 1591 /* Make sure buffer is written before we update index. */ 2139 1592 smp_wmb(); 2140 2140 - if (__put_user(cpu_to_vhost16(vq, vq->last_used_idx), &vq->used->idx)) { 1593 1593 + if (vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx), 1594 1594 + &vq->used->idx)) { 2141 1595 vq_err(vq, "Failed to increment used idx"); 2142 1596 return -EFAULT; 2143 1597 } ··· 2170 1622 2171 1623 if (!vhost_has_feature(vq, VIRTIO_RING_F_EVENT_IDX)) { 2172 1624 __virtio16 flags; 2173 2173 - if (__get_user(flags, &vq->avail->flags)) { 1625 1625 + if (vhost_get_user(vq, flags, &vq->avail->flags)) { 2174 1626 vq_err(vq, "Failed to get flags"); 2175 1627 return true; 2176 1628 } ··· 2184 1636 if (unlikely(!v)) 2185 1637 return true; 2186 1638 2187 2187 - if (__get_user(event, vhost_used_event(vq))) { 1639 1639 + if (vhost_get_user(vq, event, vhost_used_event(vq))) { 2188 1640 vq_err(vq, "Failed to get used event idx"); 2189 1641 return true; 2190 1642 } ··· 2226 1678 __virtio16 avail_idx; 2227 1679 int r; 2228 1680 2229 2229 - r = __get_user(avail_idx, &vq->avail->idx); 1681 1681 + r = vhost_get_user(vq, avail_idx, &vq->avail->idx); 2230 1682 if (r) 2231 1683 return false; 2232 1684 ··· 2261 1713 /* They could have slipped one in as we were doing that: make 2262 1714 * sure it's written, then check again. */ 2263 1715 smp_mb(); 2264 2264 - r = __get_user(avail_idx, &vq->avail->idx); 1716 1716 + r = vhost_get_user(vq, avail_idx, &vq->avail->idx); 2265 1717 if (r) { 2266 1718 vq_err(vq, "Failed to check avail idx at %p: %d\n", 2267 1719 &vq->avail->idx, r); ··· 2288 1740 } 2289 1741 } 2290 1742 EXPORT_SYMBOL_GPL(vhost_disable_notify); 1743 1743 + 1744 1744 + /* Create a new message. */ 1745 1745 + struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type) 1746 1746 + { 1747 1747 + struct vhost_msg_node *node = kmalloc(sizeof *node, GFP_KERNEL); 1748 1748 + if (!node) 1749 1749 + return NULL; 1750 1750 + node->vq = vq; 1751 1751 + node->msg.type = type; 1752 1752 + return node; 1753 1753 + } 1754 1754 + EXPORT_SYMBOL_GPL(vhost_new_msg); 1755 1755 + 1756 1756 + void vhost_enqueue_msg(struct vhost_dev *dev, struct list_head *head, 1757 1757 + struct vhost_msg_node *node) 1758 1758 + { 1759 1759 + spin_lock(&dev->iotlb_lock); 1760 1760 + list_add_tail(&node->node, head); 1761 1761 + spin_unlock(&dev->iotlb_lock); 1762 1762 + 1763 1763 + wake_up_interruptible_poll(&dev->wait, POLLIN | POLLRDNORM); 1764 1764 + } 1765 1765 + EXPORT_SYMBOL_GPL(vhost_enqueue_msg); 1766 1766 + 1767 1767 + struct vhost_msg_node *vhost_dequeue_msg(struct vhost_dev *dev, 1768 1768 + struct list_head *head) 1769 1769 + { 1770 1770 + struct vhost_msg_node *node = NULL; 1771 1771 + 1772 1772 + spin_lock(&dev->iotlb_lock); 1773 1773 + if (!list_empty(head)) { 1774 1774 + node = list_first_entry(head, struct vhost_msg_node, 1775 1775 + node); 1776 1776 + list_del(&node->node); 1777 1777 + } 1778 1778 + spin_unlock(&dev->iotlb_lock); 1779 1779 + 1780 1780 + return node; 1781 1781 + } 1782 1782 + EXPORT_SYMBOL_GPL(vhost_dequeue_msg); 1783 1783 + 2291 1784 2292 1785 static int __init vhost_init(void) 2293 1786 {

+57 -7

drivers/vhost/vhost.h

reviewed

··· 15 15 struct vhost_work; 16 16 typedef void (*vhost_work_fn_t)(struct vhost_work *work); 17 17 18 18 + #define VHOST_WORK_QUEUED 1 18 19 struct vhost_work { 19 19 - struct list_head node; 20 20 + struct llist_node node; 20 21 vhost_work_fn_t fn; 21 22 wait_queue_head_t done; 22 23 int flushing; 23 24 unsigned queue_seq; 24 25 unsigned done_seq; 26 26 + unsigned long flags; 25 27 }; 26 28 27 29 /* Poll a file (eventfd or socket) */ ··· 53 51 struct vhost_log { 54 52 u64 addr; 55 53 u64 len; 54 54 + }; 55 55 + 56 56 + #define START(node) ((node)->start) 57 57 + #define LAST(node) ((node)->last) 58 58 + 59 59 + struct vhost_umem_node { 60 60 + struct rb_node rb; 61 61 + struct list_head link; 62 62 + __u64 start; 63 63 + __u64 last; 64 64 + __u64 size; 65 65 + __u64 userspace_addr; 66 66 + __u32 perm; 67 67 + __u32 flags_padding; 68 68 + __u64 __subtree_last; 69 69 + }; 70 70 + 71 71 + struct vhost_umem { 72 72 + struct rb_root umem_tree; 73 73 + struct list_head umem_list; 74 74 + int numem; 56 75 }; 57 76 58 77 /* The virtqueue structure describes a queue attached to a device. */ ··· 121 98 u64 log_addr; 122 99 123 100 struct iovec iov[UIO_MAXIOV]; 101 101 + struct iovec iotlb_iov[64]; 124 102 struct iovec *indirect; 125 103 struct vring_used_elem *heads; 126 104 /* Protected by virtqueue mutex. */ 127 127 - struct vhost_memory *memory; 105 105 + struct vhost_umem *umem; 106 106 + struct vhost_umem *iotlb; 128 107 void *private_data; 129 108 u64 acked_features; 130 109 /* Log write descriptors */ ··· 143 118 u32 busyloop_timeout; 144 119 }; 145 120 121 121 + struct vhost_msg_node { 122 122 + struct vhost_msg msg; 123 123 + struct vhost_virtqueue *vq; 124 124 + struct list_head node; 125 125 + }; 126 126 + 146 127 struct vhost_dev { 147 147 - struct vhost_memory *memory; 148 128 struct mm_struct *mm; 149 129 struct mutex mutex; 150 130 struct vhost_virtqueue **vqs; 151 131 int nvqs; 152 132 struct file *log_file; 153 133 struct eventfd_ctx *log_ctx; 154 154 - spinlock_t work_lock; 155 155 - struct list_head work_list; 134 134 + struct llist_head work_list; 156 135 struct task_struct *worker; 136 136 + struct vhost_umem *umem; 137 137 + struct vhost_umem *iotlb; 138 138 + spinlock_t iotlb_lock; 139 139 + struct list_head read_list; 140 140 + struct list_head pending_list; 141 141 + wait_queue_head_t wait; 157 142 }; 158 143 159 144 void vhost_dev_init(struct vhost_dev *, struct vhost_virtqueue **vqs, int nvqs); 160 145 long vhost_dev_set_owner(struct vhost_dev *dev); 161 146 bool vhost_dev_has_owner(struct vhost_dev *dev); 162 147 long vhost_dev_check_owner(struct vhost_dev *); 163 163 - struct vhost_memory *vhost_dev_reset_owner_prepare(void); 164 164 - void vhost_dev_reset_owner(struct vhost_dev *, struct vhost_memory *); 148 148 + struct vhost_umem *vhost_dev_reset_owner_prepare(void); 149 149 + void vhost_dev_reset_owner(struct vhost_dev *, struct vhost_umem *); 165 150 void vhost_dev_cleanup(struct vhost_dev *, bool locked); 166 151 void vhost_dev_stop(struct vhost_dev *); 167 152 long vhost_dev_ioctl(struct vhost_dev *, unsigned int ioctl, void __user *argp); ··· 200 165 201 166 int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log, 202 167 unsigned int log_num, u64 len); 168 168 + int vq_iotlb_prefetch(struct vhost_virtqueue *vq); 169 169 + 170 170 + struct vhost_msg_node *vhost_new_msg(struct vhost_virtqueue *vq, int type); 171 171 + void vhost_enqueue_msg(struct vhost_dev *dev, 172 172 + struct list_head *head, 173 173 + struct vhost_msg_node *node); 174 174 + struct vhost_msg_node *vhost_dequeue_msg(struct vhost_dev *dev, 175 175 + struct list_head *head); 176 176 + unsigned int vhost_chr_poll(struct file *file, struct vhost_dev *dev, 177 177 + poll_table *wait); 178 178 + ssize_t vhost_chr_read_iter(struct vhost_dev *dev, struct iov_iter *to, 179 179 + int noblock); 180 180 + ssize_t vhost_chr_write_iter(struct vhost_dev *dev, 181 181 + struct iov_iter *from); 182 182 + int vhost_init_device_iotlb(struct vhost_dev *d, bool enabled); 203 183 204 184 #define vq_err(vq, fmt, ...) do { \ 205 185 pr_debug(pr_fmt(fmt), ##__VA_ARGS__); \

+719

drivers/vhost/vsock.c

reviewed

··· 1 1 + /* 2 2 + * vhost transport for vsock 3 3 + * 4 4 + * Copyright (C) 2013-2015 Red Hat, Inc. 5 5 + * Author: Asias He <asias@redhat.com> 6 6 + * Stefan Hajnoczi <stefanha@redhat.com> 7 7 + * 8 8 + * This work is licensed under the terms of the GNU GPL, version 2. 9 9 + */ 10 10 + #include <linux/miscdevice.h> 11 11 + #include <linux/atomic.h> 12 12 + #include <linux/module.h> 13 13 + #include <linux/mutex.h> 14 14 + #include <linux/vmalloc.h> 15 15 + #include <net/sock.h> 16 16 + #include <linux/virtio_vsock.h> 17 17 + #include <linux/vhost.h> 18 18 + 19 19 + #include <net/af_vsock.h> 20 20 + #include "vhost.h" 21 21 + 22 22 + #define VHOST_VSOCK_DEFAULT_HOST_CID 2 23 23 + 24 24 + enum { 25 25 + VHOST_VSOCK_FEATURES = VHOST_FEATURES, 26 26 + }; 27 27 + 28 28 + /* Used to track all the vhost_vsock instances on the system. */ 29 29 + static DEFINE_SPINLOCK(vhost_vsock_lock); 30 30 + static LIST_HEAD(vhost_vsock_list); 31 31 + 32 32 + struct vhost_vsock { 33 33 + struct vhost_dev dev; 34 34 + struct vhost_virtqueue vqs[2]; 35 35 + 36 36 + /* Link to global vhost_vsock_list, protected by vhost_vsock_lock */ 37 37 + struct list_head list; 38 38 + 39 39 + struct vhost_work send_pkt_work; 40 40 + spinlock_t send_pkt_list_lock; 41 41 + struct list_head send_pkt_list; /* host->guest pending packets */ 42 42 + 43 43 + atomic_t queued_replies; 44 44 + 45 45 + u32 guest_cid; 46 46 + }; 47 47 + 48 48 + static u32 vhost_transport_get_local_cid(void) 49 49 + { 50 50 + return VHOST_VSOCK_DEFAULT_HOST_CID; 51 51 + } 52 52 + 53 53 + static struct vhost_vsock *vhost_vsock_get(u32 guest_cid) 54 54 + { 55 55 + struct vhost_vsock *vsock; 56 56 + 57 57 + spin_lock_bh(&vhost_vsock_lock); 58 58 + list_for_each_entry(vsock, &vhost_vsock_list, list) { 59 59 + u32 other_cid = vsock->guest_cid; 60 60 + 61 61 + /* Skip instances that have no CID yet */ 62 62 + if (other_cid == 0) 63 63 + continue; 64 64 + 65 65 + if (other_cid == guest_cid) { 66 66 + spin_unlock_bh(&vhost_vsock_lock); 67 67 + return vsock; 68 68 + } 69 69 + } 70 70 + spin_unlock_bh(&vhost_vsock_lock); 71 71 + 72 72 + return NULL; 73 73 + } 74 74 + 75 75 + static void 76 76 + vhost_transport_do_send_pkt(struct vhost_vsock *vsock, 77 77 + struct vhost_virtqueue *vq) 78 78 + { 79 79 + struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX]; 80 80 + bool added = false; 81 81 + bool restart_tx = false; 82 82 + 83 83 + mutex_lock(&vq->mutex); 84 84 + 85 85 + if (!vq->private_data) 86 86 + goto out; 87 87 + 88 88 + /* Avoid further vmexits, we're already processing the virtqueue */ 89 89 + vhost_disable_notify(&vsock->dev, vq); 90 90 + 91 91 + for (;;) { 92 92 + struct virtio_vsock_pkt *pkt; 93 93 + struct iov_iter iov_iter; 94 94 + unsigned out, in; 95 95 + size_t nbytes; 96 96 + size_t len; 97 97 + int head; 98 98 + 99 99 + spin_lock_bh(&vsock->send_pkt_list_lock); 100 100 + if (list_empty(&vsock->send_pkt_list)) { 101 101 + spin_unlock_bh(&vsock->send_pkt_list_lock); 102 102 + vhost_enable_notify(&vsock->dev, vq); 103 103 + break; 104 104 + } 105 105 + 106 106 + pkt = list_first_entry(&vsock->send_pkt_list, 107 107 + struct virtio_vsock_pkt, list); 108 108 + list_del_init(&pkt->list); 109 109 + spin_unlock_bh(&vsock->send_pkt_list_lock); 110 110 + 111 111 + head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), 112 112 + &out, &in, NULL, NULL); 113 113 + if (head < 0) { 114 114 + spin_lock_bh(&vsock->send_pkt_list_lock); 115 115 + list_add(&pkt->list, &vsock->send_pkt_list); 116 116 + spin_unlock_bh(&vsock->send_pkt_list_lock); 117 117 + break; 118 118 + } 119 119 + 120 120 + if (head == vq->num) { 121 121 + spin_lock_bh(&vsock->send_pkt_list_lock); 122 122 + list_add(&pkt->list, &vsock->send_pkt_list); 123 123 + spin_unlock_bh(&vsock->send_pkt_list_lock); 124 124 + 125 125 + /* We cannot finish yet if more buffers snuck in while 126 126 + * re-enabling notify. 127 127 + */ 128 128 + if (unlikely(vhost_enable_notify(&vsock->dev, vq))) { 129 129 + vhost_disable_notify(&vsock->dev, vq); 130 130 + continue; 131 131 + } 132 132 + break; 133 133 + } 134 134 + 135 135 + if (out) { 136 136 + virtio_transport_free_pkt(pkt); 137 137 + vq_err(vq, "Expected 0 output buffers, got %u\n", out); 138 138 + break; 139 139 + } 140 140 + 141 141 + len = iov_length(&vq->iov[out], in); 142 142 + iov_iter_init(&iov_iter, READ, &vq->iov[out], in, len); 143 143 + 144 144 + nbytes = copy_to_iter(&pkt->hdr, sizeof(pkt->hdr), &iov_iter); 145 145 + if (nbytes != sizeof(pkt->hdr)) { 146 146 + virtio_transport_free_pkt(pkt); 147 147 + vq_err(vq, "Faulted on copying pkt hdr\n"); 148 148 + break; 149 149 + } 150 150 + 151 151 + nbytes = copy_to_iter(pkt->buf, pkt->len, &iov_iter); 152 152 + if (nbytes != pkt->len) { 153 153 + virtio_transport_free_pkt(pkt); 154 154 + vq_err(vq, "Faulted on copying pkt buf\n"); 155 155 + break; 156 156 + } 157 157 + 158 158 + vhost_add_used(vq, head, sizeof(pkt->hdr) + pkt->len); 159 159 + added = true; 160 160 + 161 161 + if (pkt->reply) { 162 162 + int val; 163 163 + 164 164 + val = atomic_dec_return(&vsock->queued_replies); 165 165 + 166 166 + /* Do we have resources to resume tx processing? */ 167 167 + if (val + 1 == tx_vq->num) 168 168 + restart_tx = true; 169 169 + } 170 170 + 171 171 + virtio_transport_free_pkt(pkt); 172 172 + } 173 173 + if (added) 174 174 + vhost_signal(&vsock->dev, vq); 175 175 + 176 176 + out: 177 177 + mutex_unlock(&vq->mutex); 178 178 + 179 179 + if (restart_tx) 180 180 + vhost_poll_queue(&tx_vq->poll); 181 181 + } 182 182 + 183 183 + static void vhost_transport_send_pkt_work(struct vhost_work *work) 184 184 + { 185 185 + struct vhost_virtqueue *vq; 186 186 + struct vhost_vsock *vsock; 187 187 + 188 188 + vsock = container_of(work, struct vhost_vsock, send_pkt_work); 189 189 + vq = &vsock->vqs[VSOCK_VQ_RX]; 190 190 + 191 191 + vhost_transport_do_send_pkt(vsock, vq); 192 192 + } 193 193 + 194 194 + static int 195 195 + vhost_transport_send_pkt(struct virtio_vsock_pkt *pkt) 196 196 + { 197 197 + struct vhost_vsock *vsock; 198 198 + struct vhost_virtqueue *vq; 199 199 + int len = pkt->len; 200 200 + 201 201 + /* Find the vhost_vsock according to guest context id */ 202 202 + vsock = vhost_vsock_get(le64_to_cpu(pkt->hdr.dst_cid)); 203 203 + if (!vsock) { 204 204 + virtio_transport_free_pkt(pkt); 205 205 + return -ENODEV; 206 206 + } 207 207 + 208 208 + vq = &vsock->vqs[VSOCK_VQ_RX]; 209 209 + 210 210 + if (pkt->reply) 211 211 + atomic_inc(&vsock->queued_replies); 212 212 + 213 213 + spin_lock_bh(&vsock->send_pkt_list_lock); 214 214 + list_add_tail(&pkt->list, &vsock->send_pkt_list); 215 215 + spin_unlock_bh(&vsock->send_pkt_list_lock); 216 216 + 217 217 + vhost_work_queue(&vsock->dev, &vsock->send_pkt_work); 218 218 + return len; 219 219 + } 220 220 + 221 221 + static struct virtio_vsock_pkt * 222 222 + vhost_vsock_alloc_pkt(struct vhost_virtqueue *vq, 223 223 + unsigned int out, unsigned int in) 224 224 + { 225 225 + struct virtio_vsock_pkt *pkt; 226 226 + struct iov_iter iov_iter; 227 227 + size_t nbytes; 228 228 + size_t len; 229 229 + 230 230 + if (in != 0) { 231 231 + vq_err(vq, "Expected 0 input buffers, got %u\n", in); 232 232 + return NULL; 233 233 + } 234 234 + 235 235 + pkt = kzalloc(sizeof(*pkt), GFP_KERNEL); 236 236 + if (!pkt) 237 237 + return NULL; 238 238 + 239 239 + len = iov_length(vq->iov, out); 240 240 + iov_iter_init(&iov_iter, WRITE, vq->iov, out, len); 241 241 + 242 242 + nbytes = copy_from_iter(&pkt->hdr, sizeof(pkt->hdr), &iov_iter); 243 243 + if (nbytes != sizeof(pkt->hdr)) { 244 244 + vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n", 245 245 + sizeof(pkt->hdr), nbytes); 246 246 + kfree(pkt); 247 247 + return NULL; 248 248 + } 249 249 + 250 250 + if (le16_to_cpu(pkt->hdr.type) == VIRTIO_VSOCK_TYPE_STREAM) 251 251 + pkt->len = le32_to_cpu(pkt->hdr.len); 252 252 + 253 253 + /* No payload */ 254 254 + if (!pkt->len) 255 255 + return pkt; 256 256 + 257 257 + /* The pkt is too big */ 258 258 + if (pkt->len > VIRTIO_VSOCK_MAX_PKT_BUF_SIZE) { 259 259 + kfree(pkt); 260 260 + return NULL; 261 261 + } 262 262 + 263 263 + pkt->buf = kmalloc(pkt->len, GFP_KERNEL); 264 264 + if (!pkt->buf) { 265 265 + kfree(pkt); 266 266 + return NULL; 267 267 + } 268 268 + 269 269 + nbytes = copy_from_iter(pkt->buf, pkt->len, &iov_iter); 270 270 + if (nbytes != pkt->len) { 271 271 + vq_err(vq, "Expected %u byte payload, got %zu bytes\n", 272 272 + pkt->len, nbytes); 273 273 + virtio_transport_free_pkt(pkt); 274 274 + return NULL; 275 275 + } 276 276 + 277 277 + return pkt; 278 278 + } 279 279 + 280 280 + /* Is there space left for replies to rx packets? */ 281 281 + static bool vhost_vsock_more_replies(struct vhost_vsock *vsock) 282 282 + { 283 283 + struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX]; 284 284 + int val; 285 285 + 286 286 + smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */ 287 287 + val = atomic_read(&vsock->queued_replies); 288 288 + 289 289 + return val < vq->num; 290 290 + } 291 291 + 292 292 + static void vhost_vsock_handle_tx_kick(struct vhost_work *work) 293 293 + { 294 294 + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 295 295 + poll.work); 296 296 + struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock, 297 297 + dev); 298 298 + struct virtio_vsock_pkt *pkt; 299 299 + int head; 300 300 + unsigned int out, in; 301 301 + bool added = false; 302 302 + 303 303 + mutex_lock(&vq->mutex); 304 304 + 305 305 + if (!vq->private_data) 306 306 + goto out; 307 307 + 308 308 + vhost_disable_notify(&vsock->dev, vq); 309 309 + for (;;) { 310 310 + if (!vhost_vsock_more_replies(vsock)) { 311 311 + /* Stop tx until the device processes already 312 312 + * pending replies. Leave tx virtqueue 313 313 + * callbacks disabled. 314 314 + */ 315 315 + goto no_more_replies; 316 316 + } 317 317 + 318 318 + head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov), 319 319 + &out, &in, NULL, NULL); 320 320 + if (head < 0) 321 321 + break; 322 322 + 323 323 + if (head == vq->num) { 324 324 + if (unlikely(vhost_enable_notify(&vsock->dev, vq))) { 325 325 + vhost_disable_notify(&vsock->dev, vq); 326 326 + continue; 327 327 + } 328 328 + break; 329 329 + } 330 330 + 331 331 + pkt = vhost_vsock_alloc_pkt(vq, out, in); 332 332 + if (!pkt) { 333 333 + vq_err(vq, "Faulted on pkt\n"); 334 334 + continue; 335 335 + } 336 336 + 337 337 + /* Only accept correctly addressed packets */ 338 338 + if (le64_to_cpu(pkt->hdr.src_cid) == vsock->guest_cid) 339 339 + virtio_transport_recv_pkt(pkt); 340 340 + else 341 341 + virtio_transport_free_pkt(pkt); 342 342 + 343 343 + vhost_add_used(vq, head, sizeof(pkt->hdr) + pkt->len); 344 344 + added = true; 345 345 + } 346 346 + 347 347 + no_more_replies: 348 348 + if (added) 349 349 + vhost_signal(&vsock->dev, vq); 350 350 + 351 351 + out: 352 352 + mutex_unlock(&vq->mutex); 353 353 + } 354 354 + 355 355 + static void vhost_vsock_handle_rx_kick(struct vhost_work *work) 356 356 + { 357 357 + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, 358 358 + poll.work); 359 359 + struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock, 360 360 + dev); 361 361 + 362 362 + vhost_transport_do_send_pkt(vsock, vq); 363 363 + } 364 364 + 365 365 + static int vhost_vsock_start(struct vhost_vsock *vsock) 366 366 + { 367 367 + size_t i; 368 368 + int ret; 369 369 + 370 370 + mutex_lock(&vsock->dev.mutex); 371 371 + 372 372 + ret = vhost_dev_check_owner(&vsock->dev); 373 373 + if (ret) 374 374 + goto err; 375 375 + 376 376 + for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 377 377 + struct vhost_virtqueue *vq = &vsock->vqs[i]; 378 378 + 379 379 + mutex_lock(&vq->mutex); 380 380 + 381 381 + if (!vhost_vq_access_ok(vq)) { 382 382 + ret = -EFAULT; 383 383 + mutex_unlock(&vq->mutex); 384 384 + goto err_vq; 385 385 + } 386 386 + 387 387 + if (!vq->private_data) { 388 388 + vq->private_data = vsock; 389 389 + vhost_vq_init_access(vq); 390 390 + } 391 391 + 392 392 + mutex_unlock(&vq->mutex); 393 393 + } 394 394 + 395 395 + mutex_unlock(&vsock->dev.mutex); 396 396 + return 0; 397 397 + 398 398 + err_vq: 399 399 + for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 400 400 + struct vhost_virtqueue *vq = &vsock->vqs[i]; 401 401 + 402 402 + mutex_lock(&vq->mutex); 403 403 + vq->private_data = NULL; 404 404 + mutex_unlock(&vq->mutex); 405 405 + } 406 406 + err: 407 407 + mutex_unlock(&vsock->dev.mutex); 408 408 + return ret; 409 409 + } 410 410 + 411 411 + static int vhost_vsock_stop(struct vhost_vsock *vsock) 412 412 + { 413 413 + size_t i; 414 414 + int ret; 415 415 + 416 416 + mutex_lock(&vsock->dev.mutex); 417 417 + 418 418 + ret = vhost_dev_check_owner(&vsock->dev); 419 419 + if (ret) 420 420 + goto err; 421 421 + 422 422 + for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 423 423 + struct vhost_virtqueue *vq = &vsock->vqs[i]; 424 424 + 425 425 + mutex_lock(&vq->mutex); 426 426 + vq->private_data = NULL; 427 427 + mutex_unlock(&vq->mutex); 428 428 + } 429 429 + 430 430 + err: 431 431 + mutex_unlock(&vsock->dev.mutex); 432 432 + return ret; 433 433 + } 434 434 + 435 435 + static void vhost_vsock_free(struct vhost_vsock *vsock) 436 436 + { 437 437 + kvfree(vsock); 438 438 + } 439 439 + 440 440 + static int vhost_vsock_dev_open(struct inode *inode, struct file *file) 441 441 + { 442 442 + struct vhost_virtqueue **vqs; 443 443 + struct vhost_vsock *vsock; 444 444 + int ret; 445 445 + 446 446 + /* This struct is large and allocation could fail, fall back to vmalloc 447 447 + * if there is no other way. 448 448 + */ 449 449 + vsock = kzalloc(sizeof(*vsock), GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); 450 450 + if (!vsock) { 451 451 + vsock = vmalloc(sizeof(*vsock)); 452 452 + if (!vsock) 453 453 + return -ENOMEM; 454 454 + } 455 455 + 456 456 + vqs = kmalloc_array(ARRAY_SIZE(vsock->vqs), sizeof(*vqs), GFP_KERNEL); 457 457 + if (!vqs) { 458 458 + ret = -ENOMEM; 459 459 + goto out; 460 460 + } 461 461 + 462 462 + atomic_set(&vsock->queued_replies, 0); 463 463 + 464 464 + vqs[VSOCK_VQ_TX] = &vsock->vqs[VSOCK_VQ_TX]; 465 465 + vqs[VSOCK_VQ_RX] = &vsock->vqs[VSOCK_VQ_RX]; 466 466 + vsock->vqs[VSOCK_VQ_TX].handle_kick = vhost_vsock_handle_tx_kick; 467 467 + vsock->vqs[VSOCK_VQ_RX].handle_kick = vhost_vsock_handle_rx_kick; 468 468 + 469 469 + vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs)); 470 470 + 471 471 + file->private_data = vsock; 472 472 + spin_lock_init(&vsock->send_pkt_list_lock); 473 473 + INIT_LIST_HEAD(&vsock->send_pkt_list); 474 474 + vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work); 475 475 + 476 476 + spin_lock_bh(&vhost_vsock_lock); 477 477 + list_add_tail(&vsock->list, &vhost_vsock_list); 478 478 + spin_unlock_bh(&vhost_vsock_lock); 479 479 + return 0; 480 480 + 481 481 + out: 482 482 + vhost_vsock_free(vsock); 483 483 + return ret; 484 484 + } 485 485 + 486 486 + static void vhost_vsock_flush(struct vhost_vsock *vsock) 487 487 + { 488 488 + int i; 489 489 + 490 490 + for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) 491 491 + if (vsock->vqs[i].handle_kick) 492 492 + vhost_poll_flush(&vsock->vqs[i].poll); 493 493 + vhost_work_flush(&vsock->dev, &vsock->send_pkt_work); 494 494 + } 495 495 + 496 496 + static void vhost_vsock_reset_orphans(struct sock *sk) 497 497 + { 498 498 + struct vsock_sock *vsk = vsock_sk(sk); 499 499 + 500 500 + /* vmci_transport.c doesn't take sk_lock here either. At least we're 501 501 + * under vsock_table_lock so the sock cannot disappear while we're 502 502 + * executing. 503 503 + */ 504 504 + 505 505 + if (!vhost_vsock_get(vsk->local_addr.svm_cid)) { 506 506 + sock_set_flag(sk, SOCK_DONE); 507 507 + vsk->peer_shutdown = SHUTDOWN_MASK; 508 508 + sk->sk_state = SS_UNCONNECTED; 509 509 + sk->sk_err = ECONNRESET; 510 510 + sk->sk_error_report(sk); 511 511 + } 512 512 + } 513 513 + 514 514 + static int vhost_vsock_dev_release(struct inode *inode, struct file *file) 515 515 + { 516 516 + struct vhost_vsock *vsock = file->private_data; 517 517 + 518 518 + spin_lock_bh(&vhost_vsock_lock); 519 519 + list_del(&vsock->list); 520 520 + spin_unlock_bh(&vhost_vsock_lock); 521 521 + 522 522 + /* Iterating over all connections for all CIDs to find orphans is 523 523 + * inefficient. Room for improvement here. */ 524 524 + vsock_for_each_connected_socket(vhost_vsock_reset_orphans); 525 525 + 526 526 + vhost_vsock_stop(vsock); 527 527 + vhost_vsock_flush(vsock); 528 528 + vhost_dev_stop(&vsock->dev); 529 529 + 530 530 + spin_lock_bh(&vsock->send_pkt_list_lock); 531 531 + while (!list_empty(&vsock->send_pkt_list)) { 532 532 + struct virtio_vsock_pkt *pkt; 533 533 + 534 534 + pkt = list_first_entry(&vsock->send_pkt_list, 535 535 + struct virtio_vsock_pkt, list); 536 536 + list_del_init(&pkt->list); 537 537 + virtio_transport_free_pkt(pkt); 538 538 + } 539 539 + spin_unlock_bh(&vsock->send_pkt_list_lock); 540 540 + 541 541 + vhost_dev_cleanup(&vsock->dev, false); 542 542 + kfree(vsock->dev.vqs); 543 543 + vhost_vsock_free(vsock); 544 544 + return 0; 545 545 + } 546 546 + 547 547 + static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid) 548 548 + { 549 549 + struct vhost_vsock *other; 550 550 + 551 551 + /* Refuse reserved CIDs */ 552 552 + if (guest_cid <= VMADDR_CID_HOST || 553 553 + guest_cid == U32_MAX) 554 554 + return -EINVAL; 555 555 + 556 556 + /* 64-bit CIDs are not yet supported */ 557 557 + if (guest_cid > U32_MAX) 558 558 + return -EINVAL; 559 559 + 560 560 + /* Refuse if CID is already in use */ 561 561 + other = vhost_vsock_get(guest_cid); 562 562 + if (other && other != vsock) 563 563 + return -EADDRINUSE; 564 564 + 565 565 + spin_lock_bh(&vhost_vsock_lock); 566 566 + vsock->guest_cid = guest_cid; 567 567 + spin_unlock_bh(&vhost_vsock_lock); 568 568 + 569 569 + return 0; 570 570 + } 571 571 + 572 572 + static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features) 573 573 + { 574 574 + struct vhost_virtqueue *vq; 575 575 + int i; 576 576 + 577 577 + if (features & ~VHOST_VSOCK_FEATURES) 578 578 + return -EOPNOTSUPP; 579 579 + 580 580 + mutex_lock(&vsock->dev.mutex); 581 581 + if ((features & (1 << VHOST_F_LOG_ALL)) && 582 582 + !vhost_log_access_ok(&vsock->dev)) { 583 583 + mutex_unlock(&vsock->dev.mutex); 584 584 + return -EFAULT; 585 585 + } 586 586 + 587 587 + for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) { 588 588 + vq = &vsock->vqs[i]; 589 589 + mutex_lock(&vq->mutex); 590 590 + vq->acked_features = features; 591 591 + mutex_unlock(&vq->mutex); 592 592 + } 593 593 + mutex_unlock(&vsock->dev.mutex); 594 594 + return 0; 595 595 + } 596 596 + 597 597 + static long vhost_vsock_dev_ioctl(struct file *f, unsigned int ioctl, 598 598 + unsigned long arg) 599 599 + { 600 600 + struct vhost_vsock *vsock = f->private_data; 601 601 + void __user *argp = (void __user *)arg; 602 602 + u64 guest_cid; 603 603 + u64 features; 604 604 + int start; 605 605 + int r; 606 606 + 607 607 + switch (ioctl) { 608 608 + case VHOST_VSOCK_SET_GUEST_CID: 609 609 + if (copy_from_user(&guest_cid, argp, sizeof(guest_cid))) 610 610 + return -EFAULT; 611 611 + return vhost_vsock_set_cid(vsock, guest_cid); 612 612 + case VHOST_VSOCK_SET_RUNNING: 613 613 + if (copy_from_user(&start, argp, sizeof(start))) 614 614 + return -EFAULT; 615 615 + if (start) 616 616 + return vhost_vsock_start(vsock); 617 617 + else 618 618 + return vhost_vsock_stop(vsock); 619 619 + case VHOST_GET_FEATURES: 620 620 + features = VHOST_VSOCK_FEATURES; 621 621 + if (copy_to_user(argp, &features, sizeof(features))) 622 622 + return -EFAULT; 623 623 + return 0; 624 624 + case VHOST_SET_FEATURES: 625 625 + if (copy_from_user(&features, argp, sizeof(features))) 626 626 + return -EFAULT; 627 627 + return vhost_vsock_set_features(vsock, features); 628 628 + default: 629 629 + mutex_lock(&vsock->dev.mutex); 630 630 + r = vhost_dev_ioctl(&vsock->dev, ioctl, argp); 631 631 + if (r == -ENOIOCTLCMD) 632 632 + r = vhost_vring_ioctl(&vsock->dev, ioctl, argp); 633 633 + else 634 634 + vhost_vsock_flush(vsock); 635 635 + mutex_unlock(&vsock->dev.mutex); 636 636 + return r; 637 637 + } 638 638 + } 639 639 + 640 640 + static const struct file_operations vhost_vsock_fops = { 641 641 + .owner = THIS_MODULE, 642 642 + .open = vhost_vsock_dev_open, 643 643 + .release = vhost_vsock_dev_release, 644 644 + .llseek = noop_llseek, 645 645 + .unlocked_ioctl = vhost_vsock_dev_ioctl, 646 646 + }; 647 647 + 648 648 + static struct miscdevice vhost_vsock_misc = { 649 649 + .minor = MISC_DYNAMIC_MINOR, 650 650 + .name = "vhost-vsock", 651 651 + .fops = &vhost_vsock_fops, 652 652 + }; 653 653 + 654 654 + static struct virtio_transport vhost_transport = { 655 655 + .transport = { 656 656 + .get_local_cid = vhost_transport_get_local_cid, 657 657 + 658 658 + .init = virtio_transport_do_socket_init, 659 659 + .destruct = virtio_transport_destruct, 660 660 + .release = virtio_transport_release, 661 661 + .connect = virtio_transport_connect, 662 662 + .shutdown = virtio_transport_shutdown, 663 663 + 664 664 + .dgram_enqueue = virtio_transport_dgram_enqueue, 665 665 + .dgram_dequeue = virtio_transport_dgram_dequeue, 666 666 + .dgram_bind = virtio_transport_dgram_bind, 667 667 + .dgram_allow = virtio_transport_dgram_allow, 668 668 + 669 669 + .stream_enqueue = virtio_transport_stream_enqueue, 670 670 + .stream_dequeue = virtio_transport_stream_dequeue, 671 671 + .stream_has_data = virtio_transport_stream_has_data, 672 672 + .stream_has_space = virtio_transport_stream_has_space, 673 673 + .stream_rcvhiwat = virtio_transport_stream_rcvhiwat, 674 674 + .stream_is_active = virtio_transport_stream_is_active, 675 675 + .stream_allow = virtio_transport_stream_allow, 676 676 + 677 677 + .notify_poll_in = virtio_transport_notify_poll_in, 678 678 + .notify_poll_out = virtio_transport_notify_poll_out, 679 679 + .notify_recv_init = virtio_transport_notify_recv_init, 680 680 + .notify_recv_pre_block = virtio_transport_notify_recv_pre_block, 681 681 + .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue, 682 682 + .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue, 683 683 + .notify_send_init = virtio_transport_notify_send_init, 684 684 + .notify_send_pre_block = virtio_transport_notify_send_pre_block, 685 685 + .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, 686 686 + .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, 687 687 + 688 688 + .set_buffer_size = virtio_transport_set_buffer_size, 689 689 + .set_min_buffer_size = virtio_transport_set_min_buffer_size, 690 690 + .set_max_buffer_size = virtio_transport_set_max_buffer_size, 691 691 + .get_buffer_size = virtio_transport_get_buffer_size, 692 692 + .get_min_buffer_size = virtio_transport_get_min_buffer_size, 693 693 + .get_max_buffer_size = virtio_transport_get_max_buffer_size, 694 694 + }, 695 695 + 696 696 + .send_pkt = vhost_transport_send_pkt, 697 697 + }; 698 698 + 699 699 + static int __init vhost_vsock_init(void) 700 700 + { 701 701 + int ret; 702 702 + 703 703 + ret = vsock_core_init(&vhost_transport.transport); 704 704 + if (ret < 0) 705 705 + return ret; 706 706 + return misc_register(&vhost_vsock_misc); 707 707 + }; 708 708 + 709 709 + static void __exit vhost_vsock_exit(void) 710 710 + { 711 711 + misc_deregister(&vhost_vsock_misc); 712 712 + vsock_core_exit(); 713 713 + }; 714 714 + 715 715 + module_init(vhost_vsock_init); 716 716 + module_exit(vhost_vsock_exit); 717 717 + MODULE_LICENSE("GPL v2"); 718 718 + MODULE_AUTHOR("Asias He"); 719 719 + MODULE_DESCRIPTION("vhost transport for vsock ");

+2

drivers/virtio/virtio_balloon.c

reviewed

··· 207 207 num = min(num, ARRAY_SIZE(vb->pfns)); 208 208 209 209 mutex_lock(&vb->balloon_lock); 210 210 + /* We can't release more pages than taken */ 211 211 + num = min(num, (size_t)vb->num_pages); 210 212 for (vb->num_pfns = 0; vb->num_pfns < num; 211 213 vb->num_pfns += VIRTIO_BALLOON_PAGES_PER_PAGE) { 212 214 page = balloon_page_dequeue(vb_dev_info);

+14 -1

drivers/virtio/virtio_ring.c

reviewed

··· 117 117 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) 118 118 119 119 /* 120 120 - * The interaction between virtio and a possible IOMMU is a mess. 120 120 + * Modern virtio devices have feature bits to specify whether they need a 121 121 + * quirk and bypass the IOMMU. If not there, just use the DMA API. 122 122 + * 123 123 + * If there, the interaction between virtio and DMA API is messy. 121 124 * 122 125 * On most systems with virtio, physical addresses match bus addresses, 123 126 * and it doesn't particularly matter whether we use the DMA API. ··· 136 133 * 137 134 * For the time being, we preserve historic behavior and bypass the DMA 138 135 * API. 136 136 + * 137 137 + * TODO: install a per-device DMA ops structure that does the right thing 138 138 + * taking into account all the above quirks, and use the DMA API 139 139 + * unconditionally on data path. 139 140 */ 140 141 141 142 static bool vring_use_dma_api(struct virtio_device *vdev) 142 143 { 144 144 + if (!virtio_has_iommu_quirk(vdev)) 145 145 + return true; 146 146 + 147 147 + /* Otherwise, we are left to guess. */ 143 148 /* 144 149 * In theory, it's possible to have a buggy QEMU-supposed 145 150 * emulated Q35 IOMMU and Xen enabled at the same time. On ··· 1109 1098 case VIRTIO_RING_F_EVENT_IDX: 1110 1099 break; 1111 1100 case VIRTIO_F_VERSION_1: 1101 1101 + break; 1102 1102 + case VIRTIO_F_IOMMU_PLATFORM: 1112 1103 break; 1113 1104 default: 1114 1105 /* We don't understand this bit. */

+13

include/linux/virtio_config.h

reviewed

··· 149 149 return __virtio_test_bit(vdev, fbit); 150 150 } 151 151 152 152 + /** 153 153 + * virtio_has_iommu_quirk - determine whether this device has the iommu quirk 154 154 + * @vdev: the device 155 155 + */ 156 156 + static inline bool virtio_has_iommu_quirk(const struct virtio_device *vdev) 157 157 + { 158 158 + /* 159 159 + * Note the reverse polarity of the quirk feature (compared to most 160 160 + * other features), this is for compatibility with legacy systems. 161 161 + */ 162 162 + return !virtio_has_feature(vdev, VIRTIO_F_IOMMU_PLATFORM); 163 163 + } 164 164 + 152 165 static inline 153 166 struct virtqueue *virtio_find_single_vq(struct virtio_device *vdev, 154 167 vq_callback_t *c, const char *n)

+154

include/linux/virtio_vsock.h

reviewed

··· 1 1 + #ifndef _LINUX_VIRTIO_VSOCK_H 2 2 + #define _LINUX_VIRTIO_VSOCK_H 3 3 + 4 4 + #include <uapi/linux/virtio_vsock.h> 5 5 + #include <linux/socket.h> 6 6 + #include <net/sock.h> 7 7 + #include <net/af_vsock.h> 8 8 + 9 9 + #define VIRTIO_VSOCK_DEFAULT_MIN_BUF_SIZE 128 10 10 + #define VIRTIO_VSOCK_DEFAULT_BUF_SIZE (1024 * 256) 11 11 + #define VIRTIO_VSOCK_DEFAULT_MAX_BUF_SIZE (1024 * 256) 12 12 + #define VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE (1024 * 4) 13 13 + #define VIRTIO_VSOCK_MAX_BUF_SIZE 0xFFFFFFFFUL 14 14 + #define VIRTIO_VSOCK_MAX_PKT_BUF_SIZE (1024 * 64) 15 15 + 16 16 + enum { 17 17 + VSOCK_VQ_RX = 0, /* for host to guest data */ 18 18 + VSOCK_VQ_TX = 1, /* for guest to host data */ 19 19 + VSOCK_VQ_EVENT = 2, 20 20 + VSOCK_VQ_MAX = 3, 21 21 + }; 22 22 + 23 23 + /* Per-socket state (accessed via vsk->trans) */ 24 24 + struct virtio_vsock_sock { 25 25 + struct vsock_sock *vsk; 26 26 + 27 27 + /* Protected by lock_sock(sk_vsock(trans->vsk)) */ 28 28 + u32 buf_size; 29 29 + u32 buf_size_min; 30 30 + u32 buf_size_max; 31 31 + 32 32 + spinlock_t tx_lock; 33 33 + spinlock_t rx_lock; 34 34 + 35 35 + /* Protected by tx_lock */ 36 36 + u32 tx_cnt; 37 37 + u32 buf_alloc; 38 38 + u32 peer_fwd_cnt; 39 39 + u32 peer_buf_alloc; 40 40 + 41 41 + /* Protected by rx_lock */ 42 42 + u32 fwd_cnt; 43 43 + u32 rx_bytes; 44 44 + struct list_head rx_queue; 45 45 + }; 46 46 + 47 47 + struct virtio_vsock_pkt { 48 48 + struct virtio_vsock_hdr hdr; 49 49 + struct work_struct work; 50 50 + struct list_head list; 51 51 + void *buf; 52 52 + u32 len; 53 53 + u32 off; 54 54 + bool reply; 55 55 + }; 56 56 + 57 57 + struct virtio_vsock_pkt_info { 58 58 + u32 remote_cid, remote_port; 59 59 + struct msghdr *msg; 60 60 + u32 pkt_len; 61 61 + u16 type; 62 62 + u16 op; 63 63 + u32 flags; 64 64 + bool reply; 65 65 + }; 66 66 + 67 67 + struct virtio_transport { 68 68 + /* This must be the first field */ 69 69 + struct vsock_transport transport; 70 70 + 71 71 + /* Takes ownership of the packet */ 72 72 + int (*send_pkt)(struct virtio_vsock_pkt *pkt); 73 73 + }; 74 74 + 75 75 + ssize_t 76 76 + virtio_transport_stream_dequeue(struct vsock_sock *vsk, 77 77 + struct msghdr *msg, 78 78 + size_t len, 79 79 + int type); 80 80 + int 81 81 + virtio_transport_dgram_dequeue(struct vsock_sock *vsk, 82 82 + struct msghdr *msg, 83 83 + size_t len, int flags); 84 84 + 85 85 + s64 virtio_transport_stream_has_data(struct vsock_sock *vsk); 86 86 + s64 virtio_transport_stream_has_space(struct vsock_sock *vsk); 87 87 + 88 88 + int virtio_transport_do_socket_init(struct vsock_sock *vsk, 89 89 + struct vsock_sock *psk); 90 90 + u64 virtio_transport_get_buffer_size(struct vsock_sock *vsk); 91 91 + u64 virtio_transport_get_min_buffer_size(struct vsock_sock *vsk); 92 92 + u64 virtio_transport_get_max_buffer_size(struct vsock_sock *vsk); 93 93 + void virtio_transport_set_buffer_size(struct vsock_sock *vsk, u64 val); 94 94 + void virtio_transport_set_min_buffer_size(struct vsock_sock *vsk, u64 val); 95 95 + void virtio_transport_set_max_buffer_size(struct vsock_sock *vs, u64 val); 96 96 + int 97 97 + virtio_transport_notify_poll_in(struct vsock_sock *vsk, 98 98 + size_t target, 99 99 + bool *data_ready_now); 100 100 + int 101 101 + virtio_transport_notify_poll_out(struct vsock_sock *vsk, 102 102 + size_t target, 103 103 + bool *space_available_now); 104 104 + 105 105 + int virtio_transport_notify_recv_init(struct vsock_sock *vsk, 106 106 + size_t target, struct vsock_transport_recv_notify_data *data); 107 107 + int virtio_transport_notify_recv_pre_block(struct vsock_sock *vsk, 108 108 + size_t target, struct vsock_transport_recv_notify_data *data); 109 109 + int virtio_transport_notify_recv_pre_dequeue(struct vsock_sock *vsk, 110 110 + size_t target, struct vsock_transport_recv_notify_data *data); 111 111 + int virtio_transport_notify_recv_post_dequeue(struct vsock_sock *vsk, 112 112 + size_t target, ssize_t copied, bool data_read, 113 113 + struct vsock_transport_recv_notify_data *data); 114 114 + int virtio_transport_notify_send_init(struct vsock_sock *vsk, 115 115 + struct vsock_transport_send_notify_data *data); 116 116 + int virtio_transport_notify_send_pre_block(struct vsock_sock *vsk, 117 117 + struct vsock_transport_send_notify_data *data); 118 118 + int virtio_transport_notify_send_pre_enqueue(struct vsock_sock *vsk, 119 119 + struct vsock_transport_send_notify_data *data); 120 120 + int virtio_transport_notify_send_post_enqueue(struct vsock_sock *vsk, 121 121 + ssize_t written, struct vsock_transport_send_notify_data *data); 122 122 + 123 123 + u64 virtio_transport_stream_rcvhiwat(struct vsock_sock *vsk); 124 124 + bool virtio_transport_stream_is_active(struct vsock_sock *vsk); 125 125 + bool virtio_transport_stream_allow(u32 cid, u32 port); 126 126 + int virtio_transport_dgram_bind(struct vsock_sock *vsk, 127 127 + struct sockaddr_vm *addr); 128 128 + bool virtio_transport_dgram_allow(u32 cid, u32 port); 129 129 + 130 130 + int virtio_transport_connect(struct vsock_sock *vsk); 131 131 + 132 132 + int virtio_transport_shutdown(struct vsock_sock *vsk, int mode); 133 133 + 134 134 + void virtio_transport_release(struct vsock_sock *vsk); 135 135 + 136 136 + ssize_t 137 137 + virtio_transport_stream_enqueue(struct vsock_sock *vsk, 138 138 + struct msghdr *msg, 139 139 + size_t len); 140 140 + int 141 141 + virtio_transport_dgram_enqueue(struct vsock_sock *vsk, 142 142 + struct sockaddr_vm *remote_addr, 143 143 + struct msghdr *msg, 144 144 + size_t len); 145 145 + 146 146 + void virtio_transport_destruct(struct vsock_sock *vsk); 147 147 + 148 148 + void virtio_transport_recv_pkt(struct virtio_vsock_pkt *pkt); 149 149 + void virtio_transport_free_pkt(struct virtio_vsock_pkt *pkt); 150 150 + void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct virtio_vsock_pkt *pkt); 151 151 + u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 wanted); 152 152 + void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit); 153 153 + 154 154 + #endif /* _LINUX_VIRTIO_VSOCK_H */

+6

include/net/af_vsock.h

reviewed

··· 63 63 struct list_head accept_queue; 64 64 bool rejected; 65 65 struct delayed_work dwork; 66 66 + struct delayed_work close_work; 67 67 + bool close_work_scheduled; 66 68 u32 peer_shutdown; 67 69 bool sent_request; 68 70 bool ignore_connecting_rst; ··· 167 165 } 168 166 void vsock_core_exit(void); 169 167 168 168 + /* The transport may downcast this to access transport-specific functions */ 169 169 + const struct vsock_transport *vsock_core_get_transport(void); 170 170 + 170 171 /**** UTILS ****/ 171 172 172 173 void vsock_release_pending(struct sock *pending); ··· 182 177 struct sock *vsock_find_bound_socket(struct sockaddr_vm *addr); 183 178 struct sock *vsock_find_connected_socket(struct sockaddr_vm *src, 184 179 struct sockaddr_vm *dst); 180 180 + void vsock_remove_sock(struct vsock_sock *vsk); 185 181 void vsock_for_each_connected_socket(void (*fn)(struct sock *sk)); 186 182 187 183 #endif /* __AF_VSOCK_H__ */

+144

include/trace/events/vsock_virtio_transport_common.h

reviewed

··· 1 1 + #undef TRACE_SYSTEM 2 2 + #define TRACE_SYSTEM vsock 3 3 + 4 4 + #if !defined(_TRACE_VSOCK_VIRTIO_TRANSPORT_COMMON_H) || \ 5 5 + defined(TRACE_HEADER_MULTI_READ) 6 6 + #define _TRACE_VSOCK_VIRTIO_TRANSPORT_COMMON_H 7 7 + 8 8 + #include <linux/tracepoint.h> 9 9 + 10 10 + TRACE_DEFINE_ENUM(VIRTIO_VSOCK_TYPE_STREAM); 11 11 + 12 12 + #define show_type(val) \ 13 13 + __print_symbolic(val, { VIRTIO_VSOCK_TYPE_STREAM, "STREAM" }) 14 14 + 15 15 + TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_INVALID); 16 16 + TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_REQUEST); 17 17 + TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_RESPONSE); 18 18 + TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_RST); 19 19 + TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_SHUTDOWN); 20 20 + TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_RW); 21 21 + TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_CREDIT_UPDATE); 22 22 + TRACE_DEFINE_ENUM(VIRTIO_VSOCK_OP_CREDIT_REQUEST); 23 23 + 24 24 + #define show_op(val) \ 25 25 + __print_symbolic(val, \ 26 26 + { VIRTIO_VSOCK_OP_INVALID, "INVALID" }, \ 27 27 + { VIRTIO_VSOCK_OP_REQUEST, "REQUEST" }, \ 28 28 + { VIRTIO_VSOCK_OP_RESPONSE, "RESPONSE" }, \ 29 29 + { VIRTIO_VSOCK_OP_RST, "RST" }, \ 30 30 + { VIRTIO_VSOCK_OP_SHUTDOWN, "SHUTDOWN" }, \ 31 31 + { VIRTIO_VSOCK_OP_RW, "RW" }, \ 32 32 + { VIRTIO_VSOCK_OP_CREDIT_UPDATE, "CREDIT_UPDATE" }, \ 33 33 + { VIRTIO_VSOCK_OP_CREDIT_REQUEST, "CREDIT_REQUEST" }) 34 34 + 35 35 + TRACE_EVENT(virtio_transport_alloc_pkt, 36 36 + TP_PROTO( 37 37 + __u32 src_cid, __u32 src_port, 38 38 + __u32 dst_cid, __u32 dst_port, 39 39 + __u32 len, 40 40 + __u16 type, 41 41 + __u16 op, 42 42 + __u32 flags 43 43 + ), 44 44 + TP_ARGS( 45 45 + src_cid, src_port, 46 46 + dst_cid, dst_port, 47 47 + len, 48 48 + type, 49 49 + op, 50 50 + flags 51 51 + ), 52 52 + TP_STRUCT__entry( 53 53 + __field(__u32, src_cid) 54 54 + __field(__u32, src_port) 55 55 + __field(__u32, dst_cid) 56 56 + __field(__u32, dst_port) 57 57 + __field(__u32, len) 58 58 + __field(__u16, type) 59 59 + __field(__u16, op) 60 60 + __field(__u32, flags) 61 61 + ), 62 62 + TP_fast_assign( 63 63 + __entry->src_cid = src_cid; 64 64 + __entry->src_port = src_port; 65 65 + __entry->dst_cid = dst_cid; 66 66 + __entry->dst_port = dst_port; 67 67 + __entry->len = len; 68 68 + __entry->type = type; 69 69 + __entry->op = op; 70 70 + __entry->flags = flags; 71 71 + ), 72 72 + TP_printk("%u:%u -> %u:%u len=%u type=%s op=%s flags=%#x", 73 73 + __entry->src_cid, __entry->src_port, 74 74 + __entry->dst_cid, __entry->dst_port, 75 75 + __entry->len, 76 76 + show_type(__entry->type), 77 77 + show_op(__entry->op), 78 78 + __entry->flags) 79 79 + ); 80 80 + 81 81 + TRACE_EVENT(virtio_transport_recv_pkt, 82 82 + TP_PROTO( 83 83 + __u32 src_cid, __u32 src_port, 84 84 + __u32 dst_cid, __u32 dst_port, 85 85 + __u32 len, 86 86 + __u16 type, 87 87 + __u16 op, 88 88 + __u32 flags, 89 89 + __u32 buf_alloc, 90 90 + __u32 fwd_cnt 91 91 + ), 92 92 + TP_ARGS( 93 93 + src_cid, src_port, 94 94 + dst_cid, dst_port, 95 95 + len, 96 96 + type, 97 97 + op, 98 98 + flags, 99 99 + buf_alloc, 100 100 + fwd_cnt 101 101 + ), 102 102 + TP_STRUCT__entry( 103 103 + __field(__u32, src_cid) 104 104 + __field(__u32, src_port) 105 105 + __field(__u32, dst_cid) 106 106 + __field(__u32, dst_port) 107 107 + __field(__u32, len) 108 108 + __field(__u16, type) 109 109 + __field(__u16, op) 110 110 + __field(__u32, flags) 111 111 + __field(__u32, buf_alloc) 112 112 + __field(__u32, fwd_cnt) 113 113 + ), 114 114 + TP_fast_assign( 115 115 + __entry->src_cid = src_cid; 116 116 + __entry->src_port = src_port; 117 117 + __entry->dst_cid = dst_cid; 118 118 + __entry->dst_port = dst_port; 119 119 + __entry->len = len; 120 120 + __entry->type = type; 121 121 + __entry->op = op; 122 122 + __entry->flags = flags; 123 123 + __entry->buf_alloc = buf_alloc; 124 124 + __entry->fwd_cnt = fwd_cnt; 125 125 + ), 126 126 + TP_printk("%u:%u -> %u:%u len=%u type=%s op=%s flags=%#x " 127 127 + "buf_alloc=%u fwd_cnt=%u", 128 128 + __entry->src_cid, __entry->src_port, 129 129 + __entry->dst_cid, __entry->dst_port, 130 130 + __entry->len, 131 131 + show_type(__entry->type), 132 132 + show_op(__entry->op), 133 133 + __entry->flags, 134 134 + __entry->buf_alloc, 135 135 + __entry->fwd_cnt) 136 136 + ); 137 137 + 138 138 + #endif /* _TRACE_VSOCK_VIRTIO_TRANSPORT_COMMON_H */ 139 139 + 140 140 + #undef TRACE_INCLUDE_FILE 141 141 + #define TRACE_INCLUDE_FILE vsock_virtio_transport_common 142 142 + 143 143 + /* This part must be outside protection */ 144 144 + #include <trace/define_trace.h>

+1

include/uapi/linux/Kbuild

reviewed

··· 454 454 header-y += virtio_rng.h 455 455 header-y += virtio_scsi.h 456 456 header-y += virtio_types.h 457 457 + header-y += virtio_vsock.h 457 458 header-y += vm_sockets.h 458 459 header-y += vt.h 459 460 header-y += vtpm_proxy.h

+33

include/uapi/linux/vhost.h

reviewed

··· 47 47 __u64 log_guest_addr; 48 48 }; 49 49 50 50 + /* no alignment requirement */ 51 51 + struct vhost_iotlb_msg { 52 52 + __u64 iova; 53 53 + __u64 size; 54 54 + __u64 uaddr; 55 55 + #define VHOST_ACCESS_RO 0x1 56 56 + #define VHOST_ACCESS_WO 0x2 57 57 + #define VHOST_ACCESS_RW 0x3 58 58 + __u8 perm; 59 59 + #define VHOST_IOTLB_MISS 1 60 60 + #define VHOST_IOTLB_UPDATE 2 61 61 + #define VHOST_IOTLB_INVALIDATE 3 62 62 + #define VHOST_IOTLB_ACCESS_FAIL 4 63 63 + __u8 type; 64 64 + }; 65 65 + 66 66 + #define VHOST_IOTLB_MSG 0x1 67 67 + 68 68 + struct vhost_msg { 69 69 + int type; 70 70 + union { 71 71 + struct vhost_iotlb_msg iotlb; 72 72 + __u8 padding[64]; 73 73 + }; 74 74 + }; 75 75 + 50 76 struct vhost_memory_region { 51 77 __u64 guest_phys_addr; 52 78 __u64 memory_size; /* bytes */ ··· 172 146 #define VHOST_F_LOG_ALL 26 173 147 /* vhost-net should add virtio_net_hdr for RX, and strip for TX packets. */ 174 148 #define VHOST_NET_F_VIRTIO_NET_HDR 27 149 149 + /* Vhost have device IOTLB */ 150 150 + #define VHOST_F_DEVICE_IOTLB 63 175 151 176 152 /* VHOST_SCSI specific definitions */ 177 153 ··· 202 174 /* Set and get the events missed flag */ 203 175 #define VHOST_SCSI_SET_EVENTS_MISSED _IOW(VHOST_VIRTIO, 0x43, __u32) 204 176 #define VHOST_SCSI_GET_EVENTS_MISSED _IOW(VHOST_VIRTIO, 0x44, __u32) 177 177 + 178 178 + /* VHOST_VSOCK specific defines */ 179 179 + 180 180 + #define VHOST_VSOCK_SET_GUEST_CID _IOW(VHOST_VIRTIO, 0x60, __u64) 181 181 + #define VHOST_VSOCK_SET_RUNNING _IOW(VHOST_VIRTIO, 0x61, int) 205 182 206 183 #endif

+9 -1

include/uapi/linux/virtio_config.h

reviewed

··· 49 49 * transport being used (eg. virtio_ring), the rest are per-device feature 50 50 * bits. */ 51 51 #define VIRTIO_TRANSPORT_F_START 28 52 52 - #define VIRTIO_TRANSPORT_F_END 33 52 52 + #define VIRTIO_TRANSPORT_F_END 34 53 53 54 54 #ifndef VIRTIO_CONFIG_NO_LEGACY 55 55 /* Do we get callbacks when the ring is completely used, even if we've ··· 63 63 /* v1.0 compliant. */ 64 64 #define VIRTIO_F_VERSION_1 32 65 65 66 66 + /* 67 67 + * If clear - device has the IOMMU bypass quirk feature. 68 68 + * If set - use platform tools to detect the IOMMU. 69 69 + * 70 70 + * Note the reverse polarity (compared to most other features), 71 71 + * this is for compatibility with legacy systems. 72 72 + */ 73 73 + #define VIRTIO_F_IOMMU_PLATFORM 33 66 74 #endif /* _UAPI_LINUX_VIRTIO_CONFIG_H */

+1

include/uapi/linux/virtio_ids.h

reviewed

··· 41 41 #define VIRTIO_ID_CAIF 12 /* Virtio caif */ 42 42 #define VIRTIO_ID_GPU 16 /* virtio GPU */ 43 43 #define VIRTIO_ID_INPUT 18 /* virtio input */ 44 44 + #define VIRTIO_ID_VSOCK 19 /* virtio vsock transport */ 44 45 45 46 #endif /* _LINUX_VIRTIO_IDS_H */

+94

include/uapi/linux/virtio_vsock.h

reviewed

··· 1 1 + /* 2 2 + * This header, excluding the #ifdef __KERNEL__ part, is BSD licensed so 3 3 + * anyone can use the definitions to implement compatible drivers/servers: 4 4 + * 5 5 + * 6 6 + * Redistribution and use in source and binary forms, with or without 7 7 + * modification, are permitted provided that the following conditions 8 8 + * are met: 9 9 + * 1. Redistributions of source code must retain the above copyright 10 10 + * notice, this list of conditions and the following disclaimer. 11 11 + * 2. Redistributions in binary form must reproduce the above copyright 12 12 + * notice, this list of conditions and the following disclaimer in the 13 13 + * documentation and/or other materials provided with the distribution. 14 14 + * 3. Neither the name of IBM nor the names of its contributors 15 15 + * may be used to endorse or promote products derived from this software 16 16 + * without specific prior written permission. 17 17 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' 18 18 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 19 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 20 + * ARE DISCLAIMED. IN NO EVENT SHALL IBM OR CONTRIBUTORS BE LIABLE 21 21 + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 22 + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 23 + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 24 + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 25 + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 26 + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 27 + * SUCH DAMAGE. 28 28 + * 29 29 + * Copyright (C) Red Hat, Inc., 2013-2015 30 30 + * Copyright (C) Asias He <asias@redhat.com>, 2013 31 31 + * Copyright (C) Stefan Hajnoczi <stefanha@redhat.com>, 2015 32 32 + */ 33 33 + 34 34 + #ifndef _UAPI_LINUX_VIRTIO_VSOCK_H 35 35 + #define _UAPI_LINUX_VIRTIO_VOSCK_H 36 36 + 37 37 + #include <linux/types.h> 38 38 + #include <linux/virtio_ids.h> 39 39 + #include <linux/virtio_config.h> 40 40 + 41 41 + struct virtio_vsock_config { 42 42 + __le64 guest_cid; 43 43 + } __attribute__((packed)); 44 44 + 45 45 + enum virtio_vsock_event_id { 46 46 + VIRTIO_VSOCK_EVENT_TRANSPORT_RESET = 0, 47 47 + }; 48 48 + 49 49 + struct virtio_vsock_event { 50 50 + __le32 id; 51 51 + } __attribute__((packed)); 52 52 + 53 53 + struct virtio_vsock_hdr { 54 54 + __le64 src_cid; 55 55 + __le64 dst_cid; 56 56 + __le32 src_port; 57 57 + __le32 dst_port; 58 58 + __le32 len; 59 59 + __le16 type; /* enum virtio_vsock_type */ 60 60 + __le16 op; /* enum virtio_vsock_op */ 61 61 + __le32 flags; 62 62 + __le32 buf_alloc; 63 63 + __le32 fwd_cnt; 64 64 + } __attribute__((packed)); 65 65 + 66 66 + enum virtio_vsock_type { 67 67 + VIRTIO_VSOCK_TYPE_STREAM = 1, 68 68 + }; 69 69 + 70 70 + enum virtio_vsock_op { 71 71 + VIRTIO_VSOCK_OP_INVALID = 0, 72 72 + 73 73 + /* Connect operations */ 74 74 + VIRTIO_VSOCK_OP_REQUEST = 1, 75 75 + VIRTIO_VSOCK_OP_RESPONSE = 2, 76 76 + VIRTIO_VSOCK_OP_RST = 3, 77 77 + VIRTIO_VSOCK_OP_SHUTDOWN = 4, 78 78 + 79 79 + /* To send payload */ 80 80 + VIRTIO_VSOCK_OP_RW = 5, 81 81 + 82 82 + /* Tell the peer our credit info */ 83 83 + VIRTIO_VSOCK_OP_CREDIT_UPDATE = 6, 84 84 + /* Request the peer to send the credit info to us */ 85 85 + VIRTIO_VSOCK_OP_CREDIT_REQUEST = 7, 86 86 + }; 87 87 + 88 88 + /* VIRTIO_VSOCK_OP_SHUTDOWN flags values */ 89 89 + enum virtio_vsock_shutdown { 90 90 + VIRTIO_VSOCK_SHUTDOWN_RCV = 1, 91 91 + VIRTIO_VSOCK_SHUTDOWN_SEND = 2, 92 92 + }; 93 93 + 94 94 + #endif /* _UAPI_LINUX_VIRTIO_VSOCK_H */

+20

net/vmw_vsock/Kconfig

reviewed

··· 26 26 27 27 To compile this driver as a module, choose M here: the module 28 28 will be called vmw_vsock_vmci_transport. If unsure, say N. 29 29 + 30 30 + config VIRTIO_VSOCKETS 31 31 + tristate "virtio transport for Virtual Sockets" 32 32 + depends on VSOCKETS && VIRTIO 33 33 + select VIRTIO_VSOCKETS_COMMON 34 34 + help 35 35 + This module implements a virtio transport for Virtual Sockets. 36 36 + 37 37 + Enable this transport if your Virtual Machine host supports Virtual 38 38 + Sockets over virtio. 39 39 + 40 40 + To compile this driver as a module, choose M here: the module will be 41 41 + called vmw_vsock_virtio_transport. If unsure, say N. 42 42 + 43 43 + config VIRTIO_VSOCKETS_COMMON 44 44 + tristate 45 45 + help 46 46 + This option is selected by any driver which needs to access 47 47 + the virtio_vsock. The module will be called 48 48 + vmw_vsock_virtio_transport_common.

+6

net/vmw_vsock/Makefile

reviewed

··· 1 1 obj-$(CONFIG_VSOCKETS) += vsock.o 2 2 obj-$(CONFIG_VMWARE_VMCI_VSOCKETS) += vmw_vsock_vmci_transport.o 3 3 + obj-$(CONFIG_VIRTIO_VSOCKETS) += vmw_vsock_virtio_transport.o 4 4 + obj-$(CONFIG_VIRTIO_VSOCKETS_COMMON) += vmw_vsock_virtio_transport_common.o 3 5 4 6 vsock-y += af_vsock.o vsock_addr.o 5 7 6 8 vmw_vsock_vmci_transport-y += vmci_transport.o vmci_transport_notify.o \ 7 9 vmci_transport_notify_qstate.o 10 10 + 11 11 + vmw_vsock_virtio_transport-y += virtio_transport.o 12 12 + 13 13 + vmw_vsock_virtio_transport_common-y += virtio_transport_common.o

+19 -6

net/vmw_vsock/af_vsock.c

reviewed

··· 344 344 return ret; 345 345 } 346 346 347 347 + void vsock_remove_sock(struct vsock_sock *vsk) 348 348 + { 349 349 + if (vsock_in_bound_table(vsk)) 350 350 + vsock_remove_bound(vsk); 351 351 + 352 352 + if (vsock_in_connected_table(vsk)) 353 353 + vsock_remove_connected(vsk); 354 354 + } 355 355 + EXPORT_SYMBOL_GPL(vsock_remove_sock); 356 356 + 347 357 void vsock_for_each_connected_socket(void (*fn)(struct sock *sk)) 348 358 { 349 359 int i; ··· 669 659 670 660 vsk = vsock_sk(sk); 671 661 pending = NULL; /* Compiler warning. */ 672 672 - 673 673 - if (vsock_in_bound_table(vsk)) 674 674 - vsock_remove_bound(vsk); 675 675 - 676 676 - if (vsock_in_connected_table(vsk)) 677 677 - vsock_remove_connected(vsk); 678 662 679 663 transport->release(vsk); 680 664 ··· 1998 1994 mutex_unlock(&vsock_register_mutex); 1999 1995 } 2000 1996 EXPORT_SYMBOL_GPL(vsock_core_exit); 1997 1997 + 1998 1998 + const struct vsock_transport *vsock_core_get_transport(void) 1999 1999 + { 2000 2000 + /* vsock_register_mutex not taken since only the transport uses this 2001 2001 + * function and only while registered. 2002 2002 + */ 2003 2003 + return transport; 2004 2004 + } 2005 2005 + EXPORT_SYMBOL_GPL(vsock_core_get_transport); 2001 2006 2002 2007 MODULE_AUTHOR("VMware, Inc."); 2003 2008 MODULE_DESCRIPTION("VMware Virtual Socket Family");

+624

net/vmw_vsock/virtio_transport.c

reviewed

··· 1 1 + /* 2 2 + * virtio transport for vsock 3 3 + * 4 4 + * Copyright (C) 2013-2015 Red Hat, Inc. 5 5 + * Author: Asias He <asias@redhat.com> 6 6 + * Stefan Hajnoczi <stefanha@redhat.com> 7 7 + * 8 8 + * Some of the code is take from Gerd Hoffmann <kraxel@redhat.com>'s 9 9 + * early virtio-vsock proof-of-concept bits. 10 10 + * 11 11 + * This work is licensed under the terms of the GNU GPL, version 2. 12 12 + */ 13 13 + #include <linux/spinlock.h> 14 14 + #include <linux/module.h> 15 15 + #include <linux/list.h> 16 16 + #include <linux/atomic.h> 17 17 + #include <linux/virtio.h> 18 18 + #include <linux/virtio_ids.h> 19 19 + #include <linux/virtio_config.h> 20 20 + #include <linux/virtio_vsock.h> 21 21 + #include <net/sock.h> 22 22 + #include <linux/mutex.h> 23 23 + #include <net/af_vsock.h> 24 24 + 25 25 + static struct workqueue_struct *virtio_vsock_workqueue; 26 26 + static struct virtio_vsock *the_virtio_vsock; 27 27 + static DEFINE_MUTEX(the_virtio_vsock_mutex); /* protects the_virtio_vsock */ 28 28 + 29 29 + struct virtio_vsock { 30 30 + struct virtio_device *vdev; 31 31 + struct virtqueue *vqs[VSOCK_VQ_MAX]; 32 32 + 33 33 + /* Virtqueue processing is deferred to a workqueue */ 34 34 + struct work_struct tx_work; 35 35 + struct work_struct rx_work; 36 36 + struct work_struct event_work; 37 37 + 38 38 + /* The following fields are protected by tx_lock. vqs[VSOCK_VQ_TX] 39 39 + * must be accessed with tx_lock held. 40 40 + */ 41 41 + struct mutex tx_lock; 42 42 + 43 43 + struct work_struct send_pkt_work; 44 44 + spinlock_t send_pkt_list_lock; 45 45 + struct list_head send_pkt_list; 46 46 + 47 47 + atomic_t queued_replies; 48 48 + 49 49 + /* The following fields are protected by rx_lock. vqs[VSOCK_VQ_RX] 50 50 + * must be accessed with rx_lock held. 51 51 + */ 52 52 + struct mutex rx_lock; 53 53 + int rx_buf_nr; 54 54 + int rx_buf_max_nr; 55 55 + 56 56 + /* The following fields are protected by event_lock. 57 57 + * vqs[VSOCK_VQ_EVENT] must be accessed with event_lock held. 58 58 + */ 59 59 + struct mutex event_lock; 60 60 + struct virtio_vsock_event event_list[8]; 61 61 + 62 62 + u32 guest_cid; 63 63 + }; 64 64 + 65 65 + static struct virtio_vsock *virtio_vsock_get(void) 66 66 + { 67 67 + return the_virtio_vsock; 68 68 + } 69 69 + 70 70 + static u32 virtio_transport_get_local_cid(void) 71 71 + { 72 72 + struct virtio_vsock *vsock = virtio_vsock_get(); 73 73 + 74 74 + return vsock->guest_cid; 75 75 + } 76 76 + 77 77 + static void 78 78 + virtio_transport_send_pkt_work(struct work_struct *work) 79 79 + { 80 80 + struct virtio_vsock *vsock = 81 81 + container_of(work, struct virtio_vsock, send_pkt_work); 82 82 + struct virtqueue *vq; 83 83 + bool added = false; 84 84 + bool restart_rx = false; 85 85 + 86 86 + mutex_lock(&vsock->tx_lock); 87 87 + 88 88 + vq = vsock->vqs[VSOCK_VQ_TX]; 89 89 + 90 90 + /* Avoid unnecessary interrupts while we're processing the ring */ 91 91 + virtqueue_disable_cb(vq); 92 92 + 93 93 + for (;;) { 94 94 + struct virtio_vsock_pkt *pkt; 95 95 + struct scatterlist hdr, buf, *sgs[2]; 96 96 + int ret, in_sg = 0, out_sg = 0; 97 97 + bool reply; 98 98 + 99 99 + spin_lock_bh(&vsock->send_pkt_list_lock); 100 100 + if (list_empty(&vsock->send_pkt_list)) { 101 101 + spin_unlock_bh(&vsock->send_pkt_list_lock); 102 102 + virtqueue_enable_cb(vq); 103 103 + break; 104 104 + } 105 105 + 106 106 + pkt = list_first_entry(&vsock->send_pkt_list, 107 107 + struct virtio_vsock_pkt, list); 108 108 + list_del_init(&pkt->list); 109 109 + spin_unlock_bh(&vsock->send_pkt_list_lock); 110 110 + 111 111 + reply = pkt->reply; 112 112 + 113 113 + sg_init_one(&hdr, &pkt->hdr, sizeof(pkt->hdr)); 114 114 + sgs[out_sg++] = &hdr; 115 115 + if (pkt->buf) { 116 116 + sg_init_one(&buf, pkt->buf, pkt->len); 117 117 + sgs[out_sg++] = &buf; 118 118 + } 119 119 + 120 120 + ret = virtqueue_add_sgs(vq, sgs, out_sg, in_sg, pkt, GFP_KERNEL); 121 121 + if (ret < 0) { 122 122 + spin_lock_bh(&vsock->send_pkt_list_lock); 123 123 + list_add(&pkt->list, &vsock->send_pkt_list); 124 124 + spin_unlock_bh(&vsock->send_pkt_list_lock); 125 125 + 126 126 + if (!virtqueue_enable_cb(vq) && ret == -ENOSPC) 127 127 + continue; /* retry now that we have more space */ 128 128 + break; 129 129 + } 130 130 + 131 131 + if (reply) { 132 132 + struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX]; 133 133 + int val; 134 134 + 135 135 + val = atomic_dec_return(&vsock->queued_replies); 136 136 + 137 137 + /* Do we now have resources to resume rx processing? */ 138 138 + if (val + 1 == virtqueue_get_vring_size(rx_vq)) 139 139 + restart_rx = true; 140 140 + } 141 141 + 142 142 + added = true; 143 143 + } 144 144 + 145 145 + if (added) 146 146 + virtqueue_kick(vq); 147 147 + 148 148 + mutex_unlock(&vsock->tx_lock); 149 149 + 150 150 + if (restart_rx) 151 151 + queue_work(virtio_vsock_workqueue, &vsock->rx_work); 152 152 + } 153 153 + 154 154 + static int 155 155 + virtio_transport_send_pkt(struct virtio_vsock_pkt *pkt) 156 156 + { 157 157 + struct virtio_vsock *vsock; 158 158 + int len = pkt->len; 159 159 + 160 160 + vsock = virtio_vsock_get(); 161 161 + if (!vsock) { 162 162 + virtio_transport_free_pkt(pkt); 163 163 + return -ENODEV; 164 164 + } 165 165 + 166 166 + if (pkt->reply) 167 167 + atomic_inc(&vsock->queued_replies); 168 168 + 169 169 + spin_lock_bh(&vsock->send_pkt_list_lock); 170 170 + list_add_tail(&pkt->list, &vsock->send_pkt_list); 171 171 + spin_unlock_bh(&vsock->send_pkt_list_lock); 172 172 + 173 173 + queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work); 174 174 + return len; 175 175 + } 176 176 + 177 177 + static void virtio_vsock_rx_fill(struct virtio_vsock *vsock) 178 178 + { 179 179 + int buf_len = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE; 180 180 + struct virtio_vsock_pkt *pkt; 181 181 + struct scatterlist hdr, buf, *sgs[2]; 182 182 + struct virtqueue *vq; 183 183 + int ret; 184 184 + 185 185 + vq = vsock->vqs[VSOCK_VQ_RX]; 186 186 + 187 187 + do { 188 188 + pkt = kzalloc(sizeof(*pkt), GFP_KERNEL); 189 189 + if (!pkt) 190 190 + break; 191 191 + 192 192 + pkt->buf = kmalloc(buf_len, GFP_KERNEL); 193 193 + if (!pkt->buf) { 194 194 + virtio_transport_free_pkt(pkt); 195 195 + break; 196 196 + } 197 197 + 198 198 + pkt->len = buf_len; 199 199 + 200 200 + sg_init_one(&hdr, &pkt->hdr, sizeof(pkt->hdr)); 201 201 + sgs[0] = &hdr; 202 202 + 203 203 + sg_init_one(&buf, pkt->buf, buf_len); 204 204 + sgs[1] = &buf; 205 205 + ret = virtqueue_add_sgs(vq, sgs, 0, 2, pkt, GFP_KERNEL); 206 206 + if (ret) { 207 207 + virtio_transport_free_pkt(pkt); 208 208 + break; 209 209 + } 210 210 + vsock->rx_buf_nr++; 211 211 + } while (vq->num_free); 212 212 + if (vsock->rx_buf_nr > vsock->rx_buf_max_nr) 213 213 + vsock->rx_buf_max_nr = vsock->rx_buf_nr; 214 214 + virtqueue_kick(vq); 215 215 + } 216 216 + 217 217 + static void virtio_transport_tx_work(struct work_struct *work) 218 218 + { 219 219 + struct virtio_vsock *vsock = 220 220 + container_of(work, struct virtio_vsock, tx_work); 221 221 + struct virtqueue *vq; 222 222 + bool added = false; 223 223 + 224 224 + vq = vsock->vqs[VSOCK_VQ_TX]; 225 225 + mutex_lock(&vsock->tx_lock); 226 226 + do { 227 227 + struct virtio_vsock_pkt *pkt; 228 228 + unsigned int len; 229 229 + 230 230 + virtqueue_disable_cb(vq); 231 231 + while ((pkt = virtqueue_get_buf(vq, &len)) != NULL) { 232 232 + virtio_transport_free_pkt(pkt); 233 233 + added = true; 234 234 + } 235 235 + } while (!virtqueue_enable_cb(vq)); 236 236 + mutex_unlock(&vsock->tx_lock); 237 237 + 238 238 + if (added) 239 239 + queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work); 240 240 + } 241 241 + 242 242 + /* Is there space left for replies to rx packets? */ 243 243 + static bool virtio_transport_more_replies(struct virtio_vsock *vsock) 244 244 + { 245 245 + struct virtqueue *vq = vsock->vqs[VSOCK_VQ_RX]; 246 246 + int val; 247 247 + 248 248 + smp_rmb(); /* paired with atomic_inc() and atomic_dec_return() */ 249 249 + val = atomic_read(&vsock->queued_replies); 250 250 + 251 251 + return val < virtqueue_get_vring_size(vq); 252 252 + } 253 253 + 254 254 + static void virtio_transport_rx_work(struct work_struct *work) 255 255 + { 256 256 + struct virtio_vsock *vsock = 257 257 + container_of(work, struct virtio_vsock, rx_work); 258 258 + struct virtqueue *vq; 259 259 + 260 260 + vq = vsock->vqs[VSOCK_VQ_RX]; 261 261 + 262 262 + mutex_lock(&vsock->rx_lock); 263 263 + 264 264 + do { 265 265 + virtqueue_disable_cb(vq); 266 266 + for (;;) { 267 267 + struct virtio_vsock_pkt *pkt; 268 268 + unsigned int len; 269 269 + 270 270 + if (!virtio_transport_more_replies(vsock)) { 271 271 + /* Stop rx until the device processes already 272 272 + * pending replies. Leave rx virtqueue 273 273 + * callbacks disabled. 274 274 + */ 275 275 + goto out; 276 276 + } 277 277 + 278 278 + pkt = virtqueue_get_buf(vq, &len); 279 279 + if (!pkt) { 280 280 + break; 281 281 + } 282 282 + 283 283 + vsock->rx_buf_nr--; 284 284 + 285 285 + /* Drop short/long packets */ 286 286 + if (unlikely(len < sizeof(pkt->hdr) || 287 287 + len > sizeof(pkt->hdr) + pkt->len)) { 288 288 + virtio_transport_free_pkt(pkt); 289 289 + continue; 290 290 + } 291 291 + 292 292 + pkt->len = len - sizeof(pkt->hdr); 293 293 + virtio_transport_recv_pkt(pkt); 294 294 + } 295 295 + } while (!virtqueue_enable_cb(vq)); 296 296 + 297 297 + out: 298 298 + if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2) 299 299 + virtio_vsock_rx_fill(vsock); 300 300 + mutex_unlock(&vsock->rx_lock); 301 301 + } 302 302 + 303 303 + /* event_lock must be held */ 304 304 + static int virtio_vsock_event_fill_one(struct virtio_vsock *vsock, 305 305 + struct virtio_vsock_event *event) 306 306 + { 307 307 + struct scatterlist sg; 308 308 + struct virtqueue *vq; 309 309 + 310 310 + vq = vsock->vqs[VSOCK_VQ_EVENT]; 311 311 + 312 312 + sg_init_one(&sg, event, sizeof(*event)); 313 313 + 314 314 + return virtqueue_add_inbuf(vq, &sg, 1, event, GFP_KERNEL); 315 315 + } 316 316 + 317 317 + /* event_lock must be held */ 318 318 + static void virtio_vsock_event_fill(struct virtio_vsock *vsock) 319 319 + { 320 320 + size_t i; 321 321 + 322 322 + for (i = 0; i < ARRAY_SIZE(vsock->event_list); i++) { 323 323 + struct virtio_vsock_event *event = &vsock->event_list[i]; 324 324 + 325 325 + virtio_vsock_event_fill_one(vsock, event); 326 326 + } 327 327 + 328 328 + virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]); 329 329 + } 330 330 + 331 331 + static void virtio_vsock_reset_sock(struct sock *sk) 332 332 + { 333 333 + lock_sock(sk); 334 334 + sk->sk_state = SS_UNCONNECTED; 335 335 + sk->sk_err = ECONNRESET; 336 336 + sk->sk_error_report(sk); 337 337 + release_sock(sk); 338 338 + } 339 339 + 340 340 + static void virtio_vsock_update_guest_cid(struct virtio_vsock *vsock) 341 341 + { 342 342 + struct virtio_device *vdev = vsock->vdev; 343 343 + u64 guest_cid; 344 344 + 345 345 + vdev->config->get(vdev, offsetof(struct virtio_vsock_config, guest_cid), 346 346 + &guest_cid, sizeof(guest_cid)); 347 347 + vsock->guest_cid = le64_to_cpu(guest_cid); 348 348 + } 349 349 + 350 350 + /* event_lock must be held */ 351 351 + static void virtio_vsock_event_handle(struct virtio_vsock *vsock, 352 352 + struct virtio_vsock_event *event) 353 353 + { 354 354 + switch (le32_to_cpu(event->id)) { 355 355 + case VIRTIO_VSOCK_EVENT_TRANSPORT_RESET: 356 356 + virtio_vsock_update_guest_cid(vsock); 357 357 + vsock_for_each_connected_socket(virtio_vsock_reset_sock); 358 358 + break; 359 359 + } 360 360 + } 361 361 + 362 362 + static void virtio_transport_event_work(struct work_struct *work) 363 363 + { 364 364 + struct virtio_vsock *vsock = 365 365 + container_of(work, struct virtio_vsock, event_work); 366 366 + struct virtqueue *vq; 367 367 + 368 368 + vq = vsock->vqs[VSOCK_VQ_EVENT]; 369 369 + 370 370 + mutex_lock(&vsock->event_lock); 371 371 + 372 372 + do { 373 373 + struct virtio_vsock_event *event; 374 374 + unsigned int len; 375 375 + 376 376 + virtqueue_disable_cb(vq); 377 377 + while ((event = virtqueue_get_buf(vq, &len)) != NULL) { 378 378 + if (len == sizeof(*event)) 379 379 + virtio_vsock_event_handle(vsock, event); 380 380 + 381 381 + virtio_vsock_event_fill_one(vsock, event); 382 382 + } 383 383 + } while (!virtqueue_enable_cb(vq)); 384 384 + 385 385 + virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]); 386 386 + 387 387 + mutex_unlock(&vsock->event_lock); 388 388 + } 389 389 + 390 390 + static void virtio_vsock_event_done(struct virtqueue *vq) 391 391 + { 392 392 + struct virtio_vsock *vsock = vq->vdev->priv; 393 393 + 394 394 + if (!vsock) 395 395 + return; 396 396 + queue_work(virtio_vsock_workqueue, &vsock->event_work); 397 397 + } 398 398 + 399 399 + static void virtio_vsock_tx_done(struct virtqueue *vq) 400 400 + { 401 401 + struct virtio_vsock *vsock = vq->vdev->priv; 402 402 + 403 403 + if (!vsock) 404 404 + return; 405 405 + queue_work(virtio_vsock_workqueue, &vsock->tx_work); 406 406 + } 407 407 + 408 408 + static void virtio_vsock_rx_done(struct virtqueue *vq) 409 409 + { 410 410 + struct virtio_vsock *vsock = vq->vdev->priv; 411 411 + 412 412 + if (!vsock) 413 413 + return; 414 414 + queue_work(virtio_vsock_workqueue, &vsock->rx_work); 415 415 + } 416 416 + 417 417 + static struct virtio_transport virtio_transport = { 418 418 + .transport = { 419 419 + .get_local_cid = virtio_transport_get_local_cid, 420 420 + 421 421 + .init = virtio_transport_do_socket_init, 422 422 + .destruct = virtio_transport_destruct, 423 423 + .release = virtio_transport_release, 424 424 + .connect = virtio_transport_connect, 425 425 + .shutdown = virtio_transport_shutdown, 426 426 + 427 427 + .dgram_bind = virtio_transport_dgram_bind, 428 428 + .dgram_dequeue = virtio_transport_dgram_dequeue, 429 429 + .dgram_enqueue = virtio_transport_dgram_enqueue, 430 430 + .dgram_allow = virtio_transport_dgram_allow, 431 431 + 432 432 + .stream_dequeue = virtio_transport_stream_dequeue, 433 433 + .stream_enqueue = virtio_transport_stream_enqueue, 434 434 + .stream_has_data = virtio_transport_stream_has_data, 435 435 + .stream_has_space = virtio_transport_stream_has_space, 436 436 + .stream_rcvhiwat = virtio_transport_stream_rcvhiwat, 437 437 + .stream_is_active = virtio_transport_stream_is_active, 438 438 + .stream_allow = virtio_transport_stream_allow, 439 439 + 440 440 + .notify_poll_in = virtio_transport_notify_poll_in, 441 441 + .notify_poll_out = virtio_transport_notify_poll_out, 442 442 + .notify_recv_init = virtio_transport_notify_recv_init, 443 443 + .notify_recv_pre_block = virtio_transport_notify_recv_pre_block, 444 444 + .notify_recv_pre_dequeue = virtio_transport_notify_recv_pre_dequeue, 445 445 + .notify_recv_post_dequeue = virtio_transport_notify_recv_post_dequeue, 446 446 + .notify_send_init = virtio_transport_notify_send_init, 447 447 + .notify_send_pre_block = virtio_transport_notify_send_pre_block, 448 448 + .notify_send_pre_enqueue = virtio_transport_notify_send_pre_enqueue, 449 449 + .notify_send_post_enqueue = virtio_transport_notify_send_post_enqueue, 450 450 + 451 451 + .set_buffer_size = virtio_transport_set_buffer_size, 452 452 + .set_min_buffer_size = virtio_transport_set_min_buffer_size, 453 453 + .set_max_buffer_size = virtio_transport_set_max_buffer_size, 454 454 + .get_buffer_size = virtio_transport_get_buffer_size, 455 455 + .get_min_buffer_size = virtio_transport_get_min_buffer_size, 456 456 + .get_max_buffer_size = virtio_transport_get_max_buffer_size, 457 457 + }, 458 458 + 459 459 + .send_pkt = virtio_transport_send_pkt, 460 460 + }; 461 461 + 462 462 + static int virtio_vsock_probe(struct virtio_device *vdev) 463 463 + { 464 464 + vq_callback_t *callbacks[] = { 465 465 + virtio_vsock_rx_done, 466 466 + virtio_vsock_tx_done, 467 467 + virtio_vsock_event_done, 468 468 + }; 469 469 + static const char * const names[] = { 470 470 + "rx", 471 471 + "tx", 472 472 + "event", 473 473 + }; 474 474 + struct virtio_vsock *vsock = NULL; 475 475 + int ret; 476 476 + 477 477 + ret = mutex_lock_interruptible(&the_virtio_vsock_mutex); 478 478 + if (ret) 479 479 + return ret; 480 480 + 481 481 + /* Only one virtio-vsock device per guest is supported */ 482 482 + if (the_virtio_vsock) { 483 483 + ret = -EBUSY; 484 484 + goto out; 485 485 + } 486 486 + 487 487 + vsock = kzalloc(sizeof(*vsock), GFP_KERNEL); 488 488 + if (!vsock) { 489 489 + ret = -ENOMEM; 490 490 + goto out; 491 491 + } 492 492 + 493 493 + vsock->vdev = vdev; 494 494 + 495 495 + ret = vsock->vdev->config->find_vqs(vsock->vdev, VSOCK_VQ_MAX, 496 496 + vsock->vqs, callbacks, names); 497 497 + if (ret < 0) 498 498 + goto out; 499 499 + 500 500 + virtio_vsock_update_guest_cid(vsock); 501 501 + 502 502 + ret = vsock_core_init(&virtio_transport.transport); 503 503 + if (ret < 0) 504 504 + goto out_vqs; 505 505 + 506 506 + vsock->rx_buf_nr = 0; 507 507 + vsock->rx_buf_max_nr = 0; 508 508 + atomic_set(&vsock->queued_replies, 0); 509 509 + 510 510 + vdev->priv = vsock; 511 511 + the_virtio_vsock = vsock; 512 512 + mutex_init(&vsock->tx_lock); 513 513 + mutex_init(&vsock->rx_lock); 514 514 + mutex_init(&vsock->event_lock); 515 515 + spin_lock_init(&vsock->send_pkt_list_lock); 516 516 + INIT_LIST_HEAD(&vsock->send_pkt_list); 517 517 + INIT_WORK(&vsock->rx_work, virtio_transport_rx_work); 518 518 + INIT_WORK(&vsock->tx_work, virtio_transport_tx_work); 519 519 + INIT_WORK(&vsock->event_work, virtio_transport_event_work); 520 520 + INIT_WORK(&vsock->send_pkt_work, virtio_transport_send_pkt_work); 521 521 + 522 522 + mutex_lock(&vsock->rx_lock); 523 523 + virtio_vsock_rx_fill(vsock); 524 524 + mutex_unlock(&vsock->rx_lock); 525 525 + 526 526 + mutex_lock(&vsock->event_lock); 527 527 + virtio_vsock_event_fill(vsock); 528 528 + mutex_unlock(&vsock->event_lock); 529 529 + 530 530 + mutex_unlock(&the_virtio_vsock_mutex); 531 531 + return 0; 532 532 + 533 533 + out_vqs: 534 534 + vsock->vdev->config->del_vqs(vsock->vdev); 535 535 + out: 536 536 + kfree(vsock); 537 537 + mutex_unlock(&the_virtio_vsock_mutex); 538 538 + return ret; 539 539 + } 540 540 + 541 541 + static void virtio_vsock_remove(struct virtio_device *vdev) 542 542 + { 543 543 + struct virtio_vsock *vsock = vdev->priv; 544 544 + struct virtio_vsock_pkt *pkt; 545 545 + 546 546 + flush_work(&vsock->rx_work); 547 547 + flush_work(&vsock->tx_work); 548 548 + flush_work(&vsock->event_work); 549 549 + flush_work(&vsock->send_pkt_work); 550 550 + 551 551 + vdev->config->reset(vdev); 552 552 + 553 553 + mutex_lock(&vsock->rx_lock); 554 554 + while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX]))) 555 555 + virtio_transport_free_pkt(pkt); 556 556 + mutex_unlock(&vsock->rx_lock); 557 557 + 558 558 + mutex_lock(&vsock->tx_lock); 559 559 + while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX]))) 560 560 + virtio_transport_free_pkt(pkt); 561 561 + mutex_unlock(&vsock->tx_lock); 562 562 + 563 563 + spin_lock_bh(&vsock->send_pkt_list_lock); 564 564 + while (!list_empty(&vsock->send_pkt_list)) { 565 565 + pkt = list_first_entry(&vsock->send_pkt_list, 566 566 + struct virtio_vsock_pkt, list); 567 567 + list_del(&pkt->list); 568 568 + virtio_transport_free_pkt(pkt); 569 569 + } 570 570 + spin_unlock_bh(&vsock->send_pkt_list_lock); 571 571 + 572 572 + mutex_lock(&the_virtio_vsock_mutex); 573 573 + the_virtio_vsock = NULL; 574 574 + vsock_core_exit(); 575 575 + mutex_unlock(&the_virtio_vsock_mutex); 576 576 + 577 577 + vdev->config->del_vqs(vdev); 578 578 + 579 579 + kfree(vsock); 580 580 + } 581 581 + 582 582 + static struct virtio_device_id id_table[] = { 583 583 + { VIRTIO_ID_VSOCK, VIRTIO_DEV_ANY_ID }, 584 584 + { 0 }, 585 585 + }; 586 586 + 587 587 + static unsigned int features[] = { 588 588 + }; 589 589 + 590 590 + static struct virtio_driver virtio_vsock_driver = { 591 591 + .feature_table = features, 592 592 + .feature_table_size = ARRAY_SIZE(features), 593 593 + .driver.name = KBUILD_MODNAME, 594 594 + .driver.owner = THIS_MODULE, 595 595 + .id_table = id_table, 596 596 + .probe = virtio_vsock_probe, 597 597 + .remove = virtio_vsock_remove, 598 598 + }; 599 599 + 600 600 + static int __init virtio_vsock_init(void) 601 601 + { 602 602 + int ret; 603 603 + 604 604 + virtio_vsock_workqueue = alloc_workqueue("virtio_vsock", 0, 0); 605 605 + if (!virtio_vsock_workqueue) 606 606 + return -ENOMEM; 607 607 + ret = register_virtio_driver(&virtio_vsock_driver); 608 608 + if (ret) 609 609 + destroy_workqueue(virtio_vsock_workqueue); 610 610 + return ret; 611 611 + } 612 612 + 613 613 + static void __exit virtio_vsock_exit(void) 614 614 + { 615 615 + unregister_virtio_driver(&virtio_vsock_driver); 616 616 + destroy_workqueue(virtio_vsock_workqueue); 617 617 + } 618 618 + 619 619 + module_init(virtio_vsock_init); 620 620 + module_exit(virtio_vsock_exit); 621 621 + MODULE_LICENSE("GPL v2"); 622 622 + MODULE_AUTHOR("Asias He"); 623 623 + MODULE_DESCRIPTION("virtio transport for vsock"); 624 624 + MODULE_DEVICE_TABLE(virtio, id_table);

+992

net/vmw_vsock/virtio_transport_common.c

reviewed

··· 1 1 + /* 2 2 + * common code for virtio vsock 3 3 + * 4 4 + * Copyright (C) 2013-2015 Red Hat, Inc. 5 5 + * Author: Asias He <asias@redhat.com> 6 6 + * Stefan Hajnoczi <stefanha@redhat.com> 7 7 + * 8 8 + * This work is licensed under the terms of the GNU GPL, version 2. 9 9 + */ 10 10 + #include <linux/spinlock.h> 11 11 + #include <linux/module.h> 12 12 + #include <linux/ctype.h> 13 13 + #include <linux/list.h> 14 14 + #include <linux/virtio.h> 15 15 + #include <linux/virtio_ids.h> 16 16 + #include <linux/virtio_config.h> 17 17 + #include <linux/virtio_vsock.h> 18 18 + 19 19 + #include <net/sock.h> 20 20 + #include <net/af_vsock.h> 21 21 + 22 22 + #define CREATE_TRACE_POINTS 23 23 + #include <trace/events/vsock_virtio_transport_common.h> 24 24 + 25 25 + /* How long to wait for graceful shutdown of a connection */ 26 26 + #define VSOCK_CLOSE_TIMEOUT (8 * HZ) 27 27 + 28 28 + static const struct virtio_transport *virtio_transport_get_ops(void) 29 29 + { 30 30 + const struct vsock_transport *t = vsock_core_get_transport(); 31 31 + 32 32 + return container_of(t, struct virtio_transport, transport); 33 33 + } 34 34 + 35 35 + struct virtio_vsock_pkt * 36 36 + virtio_transport_alloc_pkt(struct virtio_vsock_pkt_info *info, 37 37 + size_t len, 38 38 + u32 src_cid, 39 39 + u32 src_port, 40 40 + u32 dst_cid, 41 41 + u32 dst_port) 42 42 + { 43 43 + struct virtio_vsock_pkt *pkt; 44 44 + int err; 45 45 + 46 46 + pkt = kzalloc(sizeof(*pkt), GFP_KERNEL); 47 47 + if (!pkt) 48 48 + return NULL; 49 49 + 50 50 + pkt->hdr.type = cpu_to_le16(info->type); 51 51 + pkt->hdr.op = cpu_to_le16(info->op); 52 52 + pkt->hdr.src_cid = cpu_to_le64(src_cid); 53 53 + pkt->hdr.dst_cid = cpu_to_le64(dst_cid); 54 54 + pkt->hdr.src_port = cpu_to_le32(src_port); 55 55 + pkt->hdr.dst_port = cpu_to_le32(dst_port); 56 56 + pkt->hdr.flags = cpu_to_le32(info->flags); 57 57 + pkt->len = len; 58 58 + pkt->hdr.len = cpu_to_le32(len); 59 59 + pkt->reply = info->reply; 60 60 + 61 61 + if (info->msg && len > 0) { 62 62 + pkt->buf = kmalloc(len, GFP_KERNEL); 63 63 + if (!pkt->buf) 64 64 + goto out_pkt; 65 65 + err = memcpy_from_msg(pkt->buf, info->msg, len); 66 66 + if (err) 67 67 + goto out; 68 68 + } 69 69 + 70 70 + trace_virtio_transport_alloc_pkt(src_cid, src_port, 71 71 + dst_cid, dst_port, 72 72 + len, 73 73 + info->type, 74 74 + info->op, 75 75 + info->flags); 76 76 + 77 77 + return pkt; 78 78 + 79 79 + out: 80 80 + kfree(pkt->buf); 81 81 + out_pkt: 82 82 + kfree(pkt); 83 83 + return NULL; 84 84 + } 85 85 + EXPORT_SYMBOL_GPL(virtio_transport_alloc_pkt); 86 86 + 87 87 + static int virtio_transport_send_pkt_info(struct vsock_sock *vsk, 88 88 + struct virtio_vsock_pkt_info *info) 89 89 + { 90 90 + u32 src_cid, src_port, dst_cid, dst_port; 91 91 + struct virtio_vsock_sock *vvs; 92 92 + struct virtio_vsock_pkt *pkt; 93 93 + u32 pkt_len = info->pkt_len; 94 94 + 95 95 + src_cid = vm_sockets_get_local_cid(); 96 96 + src_port = vsk->local_addr.svm_port; 97 97 + if (!info->remote_cid) { 98 98 + dst_cid = vsk->remote_addr.svm_cid; 99 99 + dst_port = vsk->remote_addr.svm_port; 100 100 + } else { 101 101 + dst_cid = info->remote_cid; 102 102 + dst_port = info->remote_port; 103 103 + } 104 104 + 105 105 + vvs = vsk->trans; 106 106 + 107 107 + /* we can send less than pkt_len bytes */ 108 108 + if (pkt_len > VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE) 109 109 + pkt_len = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE; 110 110 + 111 111 + /* virtio_transport_get_credit might return less than pkt_len credit */ 112 112 + pkt_len = virtio_transport_get_credit(vvs, pkt_len); 113 113 + 114 114 + /* Do not send zero length OP_RW pkt */ 115 115 + if (pkt_len == 0 && info->op == VIRTIO_VSOCK_OP_RW) 116 116 + return pkt_len; 117 117 + 118 118 + pkt = virtio_transport_alloc_pkt(info, pkt_len, 119 119 + src_cid, src_port, 120 120 + dst_cid, dst_port); 121 121 + if (!pkt) { 122 122 + virtio_transport_put_credit(vvs, pkt_len); 123 123 + return -ENOMEM; 124 124 + } 125 125 + 126 126 + virtio_transport_inc_tx_pkt(vvs, pkt); 127 127 + 128 128 + return virtio_transport_get_ops()->send_pkt(pkt); 129 129 + } 130 130 + 131 131 + static void virtio_transport_inc_rx_pkt(struct virtio_vsock_sock *vvs, 132 132 + struct virtio_vsock_pkt *pkt) 133 133 + { 134 134 + vvs->rx_bytes += pkt->len; 135 135 + } 136 136 + 137 137 + static void virtio_transport_dec_rx_pkt(struct virtio_vsock_sock *vvs, 138 138 + struct virtio_vsock_pkt *pkt) 139 139 + { 140 140 + vvs->rx_bytes -= pkt->len; 141 141 + vvs->fwd_cnt += pkt->len; 142 142 + } 143 143 + 144 144 + void virtio_transport_inc_tx_pkt(struct virtio_vsock_sock *vvs, struct virtio_vsock_pkt *pkt) 145 145 + { 146 146 + spin_lock_bh(&vvs->tx_lock); 147 147 + pkt->hdr.fwd_cnt = cpu_to_le32(vvs->fwd_cnt); 148 148 + pkt->hdr.buf_alloc = cpu_to_le32(vvs->buf_alloc); 149 149 + spin_unlock_bh(&vvs->tx_lock); 150 150 + } 151 151 + EXPORT_SYMBOL_GPL(virtio_transport_inc_tx_pkt); 152 152 + 153 153 + u32 virtio_transport_get_credit(struct virtio_vsock_sock *vvs, u32 credit) 154 154 + { 155 155 + u32 ret; 156 156 + 157 157 + spin_lock_bh(&vvs->tx_lock); 158 158 + ret = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); 159 159 + if (ret > credit) 160 160 + ret = credit; 161 161 + vvs->tx_cnt += ret; 162 162 + spin_unlock_bh(&vvs->tx_lock); 163 163 + 164 164 + return ret; 165 165 + } 166 166 + EXPORT_SYMBOL_GPL(virtio_transport_get_credit); 167 167 + 168 168 + void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit) 169 169 + { 170 170 + spin_lock_bh(&vvs->tx_lock); 171 171 + vvs->tx_cnt -= credit; 172 172 + spin_unlock_bh(&vvs->tx_lock); 173 173 + } 174 174 + EXPORT_SYMBOL_GPL(virtio_transport_put_credit); 175 175 + 176 176 + static int virtio_transport_send_credit_update(struct vsock_sock *vsk, 177 177 + int type, 178 178 + struct virtio_vsock_hdr *hdr) 179 179 + { 180 180 + struct virtio_vsock_pkt_info info = { 181 181 + .op = VIRTIO_VSOCK_OP_CREDIT_UPDATE, 182 182 + .type = type, 183 183 + }; 184 184 + 185 185 + return virtio_transport_send_pkt_info(vsk, &info); 186 186 + } 187 187 + 188 188 + static ssize_t 189 189 + virtio_transport_stream_do_dequeue(struct vsock_sock *vsk, 190 190 + struct msghdr *msg, 191 191 + size_t len) 192 192 + { 193 193 + struct virtio_vsock_sock *vvs = vsk->trans; 194 194 + struct virtio_vsock_pkt *pkt; 195 195 + size_t bytes, total = 0; 196 196 + int err = -EFAULT; 197 197 + 198 198 + spin_lock_bh(&vvs->rx_lock); 199 199 + while (total < len && !list_empty(&vvs->rx_queue)) { 200 200 + pkt = list_first_entry(&vvs->rx_queue, 201 201 + struct virtio_vsock_pkt, list); 202 202 + 203 203 + bytes = len - total; 204 204 + if (bytes > pkt->len - pkt->off) 205 205 + bytes = pkt->len - pkt->off; 206 206 + 207 207 + /* sk_lock is held by caller so no one else can dequeue. 208 208 + * Unlock rx_lock since memcpy_to_msg() may sleep. 209 209 + */ 210 210 + spin_unlock_bh(&vvs->rx_lock); 211 211 + 212 212 + err = memcpy_to_msg(msg, pkt->buf + pkt->off, bytes); 213 213 + if (err) 214 214 + goto out; 215 215 + 216 216 + spin_lock_bh(&vvs->rx_lock); 217 217 + 218 218 + total += bytes; 219 219 + pkt->off += bytes; 220 220 + if (pkt->off == pkt->len) { 221 221 + virtio_transport_dec_rx_pkt(vvs, pkt); 222 222 + list_del(&pkt->list); 223 223 + virtio_transport_free_pkt(pkt); 224 224 + } 225 225 + } 226 226 + spin_unlock_bh(&vvs->rx_lock); 227 227 + 228 228 + /* Send a credit pkt to peer */ 229 229 + virtio_transport_send_credit_update(vsk, VIRTIO_VSOCK_TYPE_STREAM, 230 230 + NULL); 231 231 + 232 232 + return total; 233 233 + 234 234 + out: 235 235 + if (total) 236 236 + err = total; 237 237 + return err; 238 238 + } 239 239 + 240 240 + ssize_t 241 241 + virtio_transport_stream_dequeue(struct vsock_sock *vsk, 242 242 + struct msghdr *msg, 243 243 + size_t len, int flags) 244 244 + { 245 245 + if (flags & MSG_PEEK) 246 246 + return -EOPNOTSUPP; 247 247 + 248 248 + return virtio_transport_stream_do_dequeue(vsk, msg, len); 249 249 + } 250 250 + EXPORT_SYMBOL_GPL(virtio_transport_stream_dequeue); 251 251 + 252 252 + int 253 253 + virtio_transport_dgram_dequeue(struct vsock_sock *vsk, 254 254 + struct msghdr *msg, 255 255 + size_t len, int flags) 256 256 + { 257 257 + return -EOPNOTSUPP; 258 258 + } 259 259 + EXPORT_SYMBOL_GPL(virtio_transport_dgram_dequeue); 260 260 + 261 261 + s64 virtio_transport_stream_has_data(struct vsock_sock *vsk) 262 262 + { 263 263 + struct virtio_vsock_sock *vvs = vsk->trans; 264 264 + s64 bytes; 265 265 + 266 266 + spin_lock_bh(&vvs->rx_lock); 267 267 + bytes = vvs->rx_bytes; 268 268 + spin_unlock_bh(&vvs->rx_lock); 269 269 + 270 270 + return bytes; 271 271 + } 272 272 + EXPORT_SYMBOL_GPL(virtio_transport_stream_has_data); 273 273 + 274 274 + static s64 virtio_transport_has_space(struct vsock_sock *vsk) 275 275 + { 276 276 + struct virtio_vsock_sock *vvs = vsk->trans; 277 277 + s64 bytes; 278 278 + 279 279 + bytes = vvs->peer_buf_alloc - (vvs->tx_cnt - vvs->peer_fwd_cnt); 280 280 + if (bytes < 0) 281 281 + bytes = 0; 282 282 + 283 283 + return bytes; 284 284 + } 285 285 + 286 286 + s64 virtio_transport_stream_has_space(struct vsock_sock *vsk) 287 287 + { 288 288 + struct virtio_vsock_sock *vvs = vsk->trans; 289 289 + s64 bytes; 290 290 + 291 291 + spin_lock_bh(&vvs->tx_lock); 292 292 + bytes = virtio_transport_has_space(vsk); 293 293 + spin_unlock_bh(&vvs->tx_lock); 294 294 + 295 295 + return bytes; 296 296 + } 297 297 + EXPORT_SYMBOL_GPL(virtio_transport_stream_has_space); 298 298 + 299 299 + int virtio_transport_do_socket_init(struct vsock_sock *vsk, 300 300 + struct vsock_sock *psk) 301 301 + { 302 302 + struct virtio_vsock_sock *vvs; 303 303 + 304 304 + vvs = kzalloc(sizeof(*vvs), GFP_KERNEL); 305 305 + if (!vvs) 306 306 + return -ENOMEM; 307 307 + 308 308 + vsk->trans = vvs; 309 309 + vvs->vsk = vsk; 310 310 + if (psk) { 311 311 + struct virtio_vsock_sock *ptrans = psk->trans; 312 312 + 313 313 + vvs->buf_size = ptrans->buf_size; 314 314 + vvs->buf_size_min = ptrans->buf_size_min; 315 315 + vvs->buf_size_max = ptrans->buf_size_max; 316 316 + vvs->peer_buf_alloc = ptrans->peer_buf_alloc; 317 317 + } else { 318 318 + vvs->buf_size = VIRTIO_VSOCK_DEFAULT_BUF_SIZE; 319 319 + vvs->buf_size_min = VIRTIO_VSOCK_DEFAULT_MIN_BUF_SIZE; 320 320 + vvs->buf_size_max = VIRTIO_VSOCK_DEFAULT_MAX_BUF_SIZE; 321 321 + } 322 322 + 323 323 + vvs->buf_alloc = vvs->buf_size; 324 324 + 325 325 + spin_lock_init(&vvs->rx_lock); 326 326 + spin_lock_init(&vvs->tx_lock); 327 327 + INIT_LIST_HEAD(&vvs->rx_queue); 328 328 + 329 329 + return 0; 330 330 + } 331 331 + EXPORT_SYMBOL_GPL(virtio_transport_do_socket_init); 332 332 + 333 333 + u64 virtio_transport_get_buffer_size(struct vsock_sock *vsk) 334 334 + { 335 335 + struct virtio_vsock_sock *vvs = vsk->trans; 336 336 + 337 337 + return vvs->buf_size; 338 338 + } 339 339 + EXPORT_SYMBOL_GPL(virtio_transport_get_buffer_size); 340 340 + 341 341 + u64 virtio_transport_get_min_buffer_size(struct vsock_sock *vsk) 342 342 + { 343 343 + struct virtio_vsock_sock *vvs = vsk->trans; 344 344 + 345 345 + return vvs->buf_size_min; 346 346 + } 347 347 + EXPORT_SYMBOL_GPL(virtio_transport_get_min_buffer_size); 348 348 + 349 349 + u64 virtio_transport_get_max_buffer_size(struct vsock_sock *vsk) 350 350 + { 351 351 + struct virtio_vsock_sock *vvs = vsk->trans; 352 352 + 353 353 + return vvs->buf_size_max; 354 354 + } 355 355 + EXPORT_SYMBOL_GPL(virtio_transport_get_max_buffer_size); 356 356 + 357 357 + void virtio_transport_set_buffer_size(struct vsock_sock *vsk, u64 val) 358 358 + { 359 359 + struct virtio_vsock_sock *vvs = vsk->trans; 360 360 + 361 361 + if (val > VIRTIO_VSOCK_MAX_BUF_SIZE) 362 362 + val = VIRTIO_VSOCK_MAX_BUF_SIZE; 363 363 + if (val < vvs->buf_size_min) 364 364 + vvs->buf_size_min = val; 365 365 + if (val > vvs->buf_size_max) 366 366 + vvs->buf_size_max = val; 367 367 + vvs->buf_size = val; 368 368 + vvs->buf_alloc = val; 369 369 + } 370 370 + EXPORT_SYMBOL_GPL(virtio_transport_set_buffer_size); 371 371 + 372 372 + void virtio_transport_set_min_buffer_size(struct vsock_sock *vsk, u64 val) 373 373 + { 374 374 + struct virtio_vsock_sock *vvs = vsk->trans; 375 375 + 376 376 + if (val > VIRTIO_VSOCK_MAX_BUF_SIZE) 377 377 + val = VIRTIO_VSOCK_MAX_BUF_SIZE; 378 378 + if (val > vvs->buf_size) 379 379 + vvs->buf_size = val; 380 380 + vvs->buf_size_min = val; 381 381 + } 382 382 + EXPORT_SYMBOL_GPL(virtio_transport_set_min_buffer_size); 383 383 + 384 384 + void virtio_transport_set_max_buffer_size(struct vsock_sock *vsk, u64 val) 385 385 + { 386 386 + struct virtio_vsock_sock *vvs = vsk->trans; 387 387 + 388 388 + if (val > VIRTIO_VSOCK_MAX_BUF_SIZE) 389 389 + val = VIRTIO_VSOCK_MAX_BUF_SIZE; 390 390 + if (val < vvs->buf_size) 391 391 + vvs->buf_size = val; 392 392 + vvs->buf_size_max = val; 393 393 + } 394 394 + EXPORT_SYMBOL_GPL(virtio_transport_set_max_buffer_size); 395 395 + 396 396 + int 397 397 + virtio_transport_notify_poll_in(struct vsock_sock *vsk, 398 398 + size_t target, 399 399 + bool *data_ready_now) 400 400 + { 401 401 + if (vsock_stream_has_data(vsk)) 402 402 + *data_ready_now = true; 403 403 + else 404 404 + *data_ready_now = false; 405 405 + 406 406 + return 0; 407 407 + } 408 408 + EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_in); 409 409 + 410 410 + int 411 411 + virtio_transport_notify_poll_out(struct vsock_sock *vsk, 412 412 + size_t target, 413 413 + bool *space_avail_now) 414 414 + { 415 415 + s64 free_space; 416 416 + 417 417 + free_space = vsock_stream_has_space(vsk); 418 418 + if (free_space > 0) 419 419 + *space_avail_now = true; 420 420 + else if (free_space == 0) 421 421 + *space_avail_now = false; 422 422 + 423 423 + return 0; 424 424 + } 425 425 + EXPORT_SYMBOL_GPL(virtio_transport_notify_poll_out); 426 426 + 427 427 + int virtio_transport_notify_recv_init(struct vsock_sock *vsk, 428 428 + size_t target, struct vsock_transport_recv_notify_data *data) 429 429 + { 430 430 + return 0; 431 431 + } 432 432 + EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_init); 433 433 + 434 434 + int virtio_transport_notify_recv_pre_block(struct vsock_sock *vsk, 435 435 + size_t target, struct vsock_transport_recv_notify_data *data) 436 436 + { 437 437 + return 0; 438 438 + } 439 439 + EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_block); 440 440 + 441 441 + int virtio_transport_notify_recv_pre_dequeue(struct vsock_sock *vsk, 442 442 + size_t target, struct vsock_transport_recv_notify_data *data) 443 443 + { 444 444 + return 0; 445 445 + } 446 446 + EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_pre_dequeue); 447 447 + 448 448 + int virtio_transport_notify_recv_post_dequeue(struct vsock_sock *vsk, 449 449 + size_t target, ssize_t copied, bool data_read, 450 450 + struct vsock_transport_recv_notify_data *data) 451 451 + { 452 452 + return 0; 453 453 + } 454 454 + EXPORT_SYMBOL_GPL(virtio_transport_notify_recv_post_dequeue); 455 455 + 456 456 + int virtio_transport_notify_send_init(struct vsock_sock *vsk, 457 457 + struct vsock_transport_send_notify_data *data) 458 458 + { 459 459 + return 0; 460 460 + } 461 461 + EXPORT_SYMBOL_GPL(virtio_transport_notify_send_init); 462 462 + 463 463 + int virtio_transport_notify_send_pre_block(struct vsock_sock *vsk, 464 464 + struct vsock_transport_send_notify_data *data) 465 465 + { 466 466 + return 0; 467 467 + } 468 468 + EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_block); 469 469 + 470 470 + int virtio_transport_notify_send_pre_enqueue(struct vsock_sock *vsk, 471 471 + struct vsock_transport_send_notify_data *data) 472 472 + { 473 473 + return 0; 474 474 + } 475 475 + EXPORT_SYMBOL_GPL(virtio_transport_notify_send_pre_enqueue); 476 476 + 477 477 + int virtio_transport_notify_send_post_enqueue(struct vsock_sock *vsk, 478 478 + ssize_t written, struct vsock_transport_send_notify_data *data) 479 479 + { 480 480 + return 0; 481 481 + } 482 482 + EXPORT_SYMBOL_GPL(virtio_transport_notify_send_post_enqueue); 483 483 + 484 484 + u64 virtio_transport_stream_rcvhiwat(struct vsock_sock *vsk) 485 485 + { 486 486 + struct virtio_vsock_sock *vvs = vsk->trans; 487 487 + 488 488 + return vvs->buf_size; 489 489 + } 490 490 + EXPORT_SYMBOL_GPL(virtio_transport_stream_rcvhiwat); 491 491 + 492 492 + bool virtio_transport_stream_is_active(struct vsock_sock *vsk) 493 493 + { 494 494 + return true; 495 495 + } 496 496 + EXPORT_SYMBOL_GPL(virtio_transport_stream_is_active); 497 497 + 498 498 + bool virtio_transport_stream_allow(u32 cid, u32 port) 499 499 + { 500 500 + return true; 501 501 + } 502 502 + EXPORT_SYMBOL_GPL(virtio_transport_stream_allow); 503 503 + 504 504 + int virtio_transport_dgram_bind(struct vsock_sock *vsk, 505 505 + struct sockaddr_vm *addr) 506 506 + { 507 507 + return -EOPNOTSUPP; 508 508 + } 509 509 + EXPORT_SYMBOL_GPL(virtio_transport_dgram_bind); 510 510 + 511 511 + bool virtio_transport_dgram_allow(u32 cid, u32 port) 512 512 + { 513 513 + return false; 514 514 + } 515 515 + EXPORT_SYMBOL_GPL(virtio_transport_dgram_allow); 516 516 + 517 517 + int virtio_transport_connect(struct vsock_sock *vsk) 518 518 + { 519 519 + struct virtio_vsock_pkt_info info = { 520 520 + .op = VIRTIO_VSOCK_OP_REQUEST, 521 521 + .type = VIRTIO_VSOCK_TYPE_STREAM, 522 522 + }; 523 523 + 524 524 + return virtio_transport_send_pkt_info(vsk, &info); 525 525 + } 526 526 + EXPORT_SYMBOL_GPL(virtio_transport_connect); 527 527 + 528 528 + int virtio_transport_shutdown(struct vsock_sock *vsk, int mode) 529 529 + { 530 530 + struct virtio_vsock_pkt_info info = { 531 531 + .op = VIRTIO_VSOCK_OP_SHUTDOWN, 532 532 + .type = VIRTIO_VSOCK_TYPE_STREAM, 533 533 + .flags = (mode & RCV_SHUTDOWN ? 534 534 + VIRTIO_VSOCK_SHUTDOWN_RCV : 0) | 535 535 + (mode & SEND_SHUTDOWN ? 536 536 + VIRTIO_VSOCK_SHUTDOWN_SEND : 0), 537 537 + }; 538 538 + 539 539 + return virtio_transport_send_pkt_info(vsk, &info); 540 540 + } 541 541 + EXPORT_SYMBOL_GPL(virtio_transport_shutdown); 542 542 + 543 543 + int 544 544 + virtio_transport_dgram_enqueue(struct vsock_sock *vsk, 545 545 + struct sockaddr_vm *remote_addr, 546 546 + struct msghdr *msg, 547 547 + size_t dgram_len) 548 548 + { 549 549 + return -EOPNOTSUPP; 550 550 + } 551 551 + EXPORT_SYMBOL_GPL(virtio_transport_dgram_enqueue); 552 552 + 553 553 + ssize_t 554 554 + virtio_transport_stream_enqueue(struct vsock_sock *vsk, 555 555 + struct msghdr *msg, 556 556 + size_t len) 557 557 + { 558 558 + struct virtio_vsock_pkt_info info = { 559 559 + .op = VIRTIO_VSOCK_OP_RW, 560 560 + .type = VIRTIO_VSOCK_TYPE_STREAM, 561 561 + .msg = msg, 562 562 + .pkt_len = len, 563 563 + }; 564 564 + 565 565 + return virtio_transport_send_pkt_info(vsk, &info); 566 566 + } 567 567 + EXPORT_SYMBOL_GPL(virtio_transport_stream_enqueue); 568 568 + 569 569 + void virtio_transport_destruct(struct vsock_sock *vsk) 570 570 + { 571 571 + struct virtio_vsock_sock *vvs = vsk->trans; 572 572 + 573 573 + kfree(vvs); 574 574 + } 575 575 + EXPORT_SYMBOL_GPL(virtio_transport_destruct); 576 576 + 577 577 + static int virtio_transport_reset(struct vsock_sock *vsk, 578 578 + struct virtio_vsock_pkt *pkt) 579 579 + { 580 580 + struct virtio_vsock_pkt_info info = { 581 581 + .op = VIRTIO_VSOCK_OP_RST, 582 582 + .type = VIRTIO_VSOCK_TYPE_STREAM, 583 583 + .reply = !!pkt, 584 584 + }; 585 585 + 586 586 + /* Send RST only if the original pkt is not a RST pkt */ 587 587 + if (pkt && le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 588 588 + return 0; 589 589 + 590 590 + return virtio_transport_send_pkt_info(vsk, &info); 591 591 + } 592 592 + 593 593 + /* Normally packets are associated with a socket. There may be no socket if an 594 594 + * attempt was made to connect to a socket that does not exist. 595 595 + */ 596 596 + static int virtio_transport_reset_no_sock(struct virtio_vsock_pkt *pkt) 597 597 + { 598 598 + struct virtio_vsock_pkt_info info = { 599 599 + .op = VIRTIO_VSOCK_OP_RST, 600 600 + .type = le16_to_cpu(pkt->hdr.type), 601 601 + .reply = true, 602 602 + }; 603 603 + 604 604 + /* Send RST only if the original pkt is not a RST pkt */ 605 605 + if (le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 606 606 + return 0; 607 607 + 608 608 + pkt = virtio_transport_alloc_pkt(&info, 0, 609 609 + le32_to_cpu(pkt->hdr.dst_cid), 610 610 + le32_to_cpu(pkt->hdr.dst_port), 611 611 + le32_to_cpu(pkt->hdr.src_cid), 612 612 + le32_to_cpu(pkt->hdr.src_port)); 613 613 + if (!pkt) 614 614 + return -ENOMEM; 615 615 + 616 616 + return virtio_transport_get_ops()->send_pkt(pkt); 617 617 + } 618 618 + 619 619 + static void virtio_transport_wait_close(struct sock *sk, long timeout) 620 620 + { 621 621 + if (timeout) { 622 622 + DEFINE_WAIT(wait); 623 623 + 624 624 + do { 625 625 + prepare_to_wait(sk_sleep(sk), &wait, 626 626 + TASK_INTERRUPTIBLE); 627 627 + if (sk_wait_event(sk, &timeout, 628 628 + sock_flag(sk, SOCK_DONE))) 629 629 + break; 630 630 + } while (!signal_pending(current) && timeout); 631 631 + 632 632 + finish_wait(sk_sleep(sk), &wait); 633 633 + } 634 634 + } 635 635 + 636 636 + static void virtio_transport_do_close(struct vsock_sock *vsk, 637 637 + bool cancel_timeout) 638 638 + { 639 639 + struct sock *sk = sk_vsock(vsk); 640 640 + 641 641 + sock_set_flag(sk, SOCK_DONE); 642 642 + vsk->peer_shutdown = SHUTDOWN_MASK; 643 643 + if (vsock_stream_has_data(vsk) <= 0) 644 644 + sk->sk_state = SS_DISCONNECTING; 645 645 + sk->sk_state_change(sk); 646 646 + 647 647 + if (vsk->close_work_scheduled && 648 648 + (!cancel_timeout || cancel_delayed_work(&vsk->close_work))) { 649 649 + vsk->close_work_scheduled = false; 650 650 + 651 651 + vsock_remove_sock(vsk); 652 652 + 653 653 + /* Release refcnt obtained when we scheduled the timeout */ 654 654 + sock_put(sk); 655 655 + } 656 656 + } 657 657 + 658 658 + static void virtio_transport_close_timeout(struct work_struct *work) 659 659 + { 660 660 + struct vsock_sock *vsk = 661 661 + container_of(work, struct vsock_sock, close_work.work); 662 662 + struct sock *sk = sk_vsock(vsk); 663 663 + 664 664 + sock_hold(sk); 665 665 + lock_sock(sk); 666 666 + 667 667 + if (!sock_flag(sk, SOCK_DONE)) { 668 668 + (void)virtio_transport_reset(vsk, NULL); 669 669 + 670 670 + virtio_transport_do_close(vsk, false); 671 671 + } 672 672 + 673 673 + vsk->close_work_scheduled = false; 674 674 + 675 675 + release_sock(sk); 676 676 + sock_put(sk); 677 677 + } 678 678 + 679 679 + /* User context, vsk->sk is locked */ 680 680 + static bool virtio_transport_close(struct vsock_sock *vsk) 681 681 + { 682 682 + struct sock *sk = &vsk->sk; 683 683 + 684 684 + if (!(sk->sk_state == SS_CONNECTED || 685 685 + sk->sk_state == SS_DISCONNECTING)) 686 686 + return true; 687 687 + 688 688 + /* Already received SHUTDOWN from peer, reply with RST */ 689 689 + if ((vsk->peer_shutdown & SHUTDOWN_MASK) == SHUTDOWN_MASK) { 690 690 + (void)virtio_transport_reset(vsk, NULL); 691 691 + return true; 692 692 + } 693 693 + 694 694 + if ((sk->sk_shutdown & SHUTDOWN_MASK) != SHUTDOWN_MASK) 695 695 + (void)virtio_transport_shutdown(vsk, SHUTDOWN_MASK); 696 696 + 697 697 + if (sock_flag(sk, SOCK_LINGER) && !(current->flags & PF_EXITING)) 698 698 + virtio_transport_wait_close(sk, sk->sk_lingertime); 699 699 + 700 700 + if (sock_flag(sk, SOCK_DONE)) { 701 701 + return true; 702 702 + } 703 703 + 704 704 + sock_hold(sk); 705 705 + INIT_DELAYED_WORK(&vsk->close_work, 706 706 + virtio_transport_close_timeout); 707 707 + vsk->close_work_scheduled = true; 708 708 + schedule_delayed_work(&vsk->close_work, VSOCK_CLOSE_TIMEOUT); 709 709 + return false; 710 710 + } 711 711 + 712 712 + void virtio_transport_release(struct vsock_sock *vsk) 713 713 + { 714 714 + struct sock *sk = &vsk->sk; 715 715 + bool remove_sock = true; 716 716 + 717 717 + lock_sock(sk); 718 718 + if (sk->sk_type == SOCK_STREAM) 719 719 + remove_sock = virtio_transport_close(vsk); 720 720 + release_sock(sk); 721 721 + 722 722 + if (remove_sock) 723 723 + vsock_remove_sock(vsk); 724 724 + } 725 725 + EXPORT_SYMBOL_GPL(virtio_transport_release); 726 726 + 727 727 + static int 728 728 + virtio_transport_recv_connecting(struct sock *sk, 729 729 + struct virtio_vsock_pkt *pkt) 730 730 + { 731 731 + struct vsock_sock *vsk = vsock_sk(sk); 732 732 + int err; 733 733 + int skerr; 734 734 + 735 735 + switch (le16_to_cpu(pkt->hdr.op)) { 736 736 + case VIRTIO_VSOCK_OP_RESPONSE: 737 737 + sk->sk_state = SS_CONNECTED; 738 738 + sk->sk_socket->state = SS_CONNECTED; 739 739 + vsock_insert_connected(vsk); 740 740 + sk->sk_state_change(sk); 741 741 + break; 742 742 + case VIRTIO_VSOCK_OP_INVALID: 743 743 + break; 744 744 + case VIRTIO_VSOCK_OP_RST: 745 745 + skerr = ECONNRESET; 746 746 + err = 0; 747 747 + goto destroy; 748 748 + default: 749 749 + skerr = EPROTO; 750 750 + err = -EINVAL; 751 751 + goto destroy; 752 752 + } 753 753 + return 0; 754 754 + 755 755 + destroy: 756 756 + virtio_transport_reset(vsk, pkt); 757 757 + sk->sk_state = SS_UNCONNECTED; 758 758 + sk->sk_err = skerr; 759 759 + sk->sk_error_report(sk); 760 760 + return err; 761 761 + } 762 762 + 763 763 + static int 764 764 + virtio_transport_recv_connected(struct sock *sk, 765 765 + struct virtio_vsock_pkt *pkt) 766 766 + { 767 767 + struct vsock_sock *vsk = vsock_sk(sk); 768 768 + struct virtio_vsock_sock *vvs = vsk->trans; 769 769 + int err = 0; 770 770 + 771 771 + switch (le16_to_cpu(pkt->hdr.op)) { 772 772 + case VIRTIO_VSOCK_OP_RW: 773 773 + pkt->len = le32_to_cpu(pkt->hdr.len); 774 774 + pkt->off = 0; 775 775 + 776 776 + spin_lock_bh(&vvs->rx_lock); 777 777 + virtio_transport_inc_rx_pkt(vvs, pkt); 778 778 + list_add_tail(&pkt->list, &vvs->rx_queue); 779 779 + spin_unlock_bh(&vvs->rx_lock); 780 780 + 781 781 + sk->sk_data_ready(sk); 782 782 + return err; 783 783 + case VIRTIO_VSOCK_OP_CREDIT_UPDATE: 784 784 + sk->sk_write_space(sk); 785 785 + break; 786 786 + case VIRTIO_VSOCK_OP_SHUTDOWN: 787 787 + if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SHUTDOWN_RCV) 788 788 + vsk->peer_shutdown |= RCV_SHUTDOWN; 789 789 + if (le32_to_cpu(pkt->hdr.flags) & VIRTIO_VSOCK_SHUTDOWN_SEND) 790 790 + vsk->peer_shutdown |= SEND_SHUTDOWN; 791 791 + if (vsk->peer_shutdown == SHUTDOWN_MASK && 792 792 + vsock_stream_has_data(vsk) <= 0) 793 793 + sk->sk_state = SS_DISCONNECTING; 794 794 + if (le32_to_cpu(pkt->hdr.flags)) 795 795 + sk->sk_state_change(sk); 796 796 + break; 797 797 + case VIRTIO_VSOCK_OP_RST: 798 798 + virtio_transport_do_close(vsk, true); 799 799 + break; 800 800 + default: 801 801 + err = -EINVAL; 802 802 + break; 803 803 + } 804 804 + 805 805 + virtio_transport_free_pkt(pkt); 806 806 + return err; 807 807 + } 808 808 + 809 809 + static void 810 810 + virtio_transport_recv_disconnecting(struct sock *sk, 811 811 + struct virtio_vsock_pkt *pkt) 812 812 + { 813 813 + struct vsock_sock *vsk = vsock_sk(sk); 814 814 + 815 815 + if (le16_to_cpu(pkt->hdr.op) == VIRTIO_VSOCK_OP_RST) 816 816 + virtio_transport_do_close(vsk, true); 817 817 + } 818 818 + 819 819 + static int 820 820 + virtio_transport_send_response(struct vsock_sock *vsk, 821 821 + struct virtio_vsock_pkt *pkt) 822 822 + { 823 823 + struct virtio_vsock_pkt_info info = { 824 824 + .op = VIRTIO_VSOCK_OP_RESPONSE, 825 825 + .type = VIRTIO_VSOCK_TYPE_STREAM, 826 826 + .remote_cid = le32_to_cpu(pkt->hdr.src_cid), 827 827 + .remote_port = le32_to_cpu(pkt->hdr.src_port), 828 828 + .reply = true, 829 829 + }; 830 830 + 831 831 + return virtio_transport_send_pkt_info(vsk, &info); 832 832 + } 833 833 + 834 834 + /* Handle server socket */ 835 835 + static int 836 836 + virtio_transport_recv_listen(struct sock *sk, struct virtio_vsock_pkt *pkt) 837 837 + { 838 838 + struct vsock_sock *vsk = vsock_sk(sk); 839 839 + struct vsock_sock *vchild; 840 840 + struct sock *child; 841 841 + 842 842 + if (le16_to_cpu(pkt->hdr.op) != VIRTIO_VSOCK_OP_REQUEST) { 843 843 + virtio_transport_reset(vsk, pkt); 844 844 + return -EINVAL; 845 845 + } 846 846 + 847 847 + if (sk_acceptq_is_full(sk)) { 848 848 + virtio_transport_reset(vsk, pkt); 849 849 + return -ENOMEM; 850 850 + } 851 851 + 852 852 + child = __vsock_create(sock_net(sk), NULL, sk, GFP_KERNEL, 853 853 + sk->sk_type, 0); 854 854 + if (!child) { 855 855 + virtio_transport_reset(vsk, pkt); 856 856 + return -ENOMEM; 857 857 + } 858 858 + 859 859 + sk->sk_ack_backlog++; 860 860 + 861 861 + lock_sock_nested(child, SINGLE_DEPTH_NESTING); 862 862 + 863 863 + child->sk_state = SS_CONNECTED; 864 864 + 865 865 + vchild = vsock_sk(child); 866 866 + vsock_addr_init(&vchild->local_addr, le32_to_cpu(pkt->hdr.dst_cid), 867 867 + le32_to_cpu(pkt->hdr.dst_port)); 868 868 + vsock_addr_init(&vchild->remote_addr, le32_to_cpu(pkt->hdr.src_cid), 869 869 + le32_to_cpu(pkt->hdr.src_port)); 870 870 + 871 871 + vsock_insert_connected(vchild); 872 872 + vsock_enqueue_accept(sk, child); 873 873 + virtio_transport_send_response(vchild, pkt); 874 874 + 875 875 + release_sock(child); 876 876 + 877 877 + sk->sk_data_ready(sk); 878 878 + return 0; 879 879 + } 880 880 + 881 881 + static bool virtio_transport_space_update(struct sock *sk, 882 882 + struct virtio_vsock_pkt *pkt) 883 883 + { 884 884 + struct vsock_sock *vsk = vsock_sk(sk); 885 885 + struct virtio_vsock_sock *vvs = vsk->trans; 886 886 + bool space_available; 887 887 + 888 888 + /* buf_alloc and fwd_cnt is always included in the hdr */ 889 889 + spin_lock_bh(&vvs->tx_lock); 890 890 + vvs->peer_buf_alloc = le32_to_cpu(pkt->hdr.buf_alloc); 891 891 + vvs->peer_fwd_cnt = le32_to_cpu(pkt->hdr.fwd_cnt); 892 892 + space_available = virtio_transport_has_space(vsk); 893 893 + spin_unlock_bh(&vvs->tx_lock); 894 894 + return space_available; 895 895 + } 896 896 + 897 897 + /* We are under the virtio-vsock's vsock->rx_lock or vhost-vsock's vq->mutex 898 898 + * lock. 899 899 + */ 900 900 + void virtio_transport_recv_pkt(struct virtio_vsock_pkt *pkt) 901 901 + { 902 902 + struct sockaddr_vm src, dst; 903 903 + struct vsock_sock *vsk; 904 904 + struct sock *sk; 905 905 + bool space_available; 906 906 + 907 907 + vsock_addr_init(&src, le32_to_cpu(pkt->hdr.src_cid), 908 908 + le32_to_cpu(pkt->hdr.src_port)); 909 909 + vsock_addr_init(&dst, le32_to_cpu(pkt->hdr.dst_cid), 910 910 + le32_to_cpu(pkt->hdr.dst_port)); 911 911 + 912 912 + trace_virtio_transport_recv_pkt(src.svm_cid, src.svm_port, 913 913 + dst.svm_cid, dst.svm_port, 914 914 + le32_to_cpu(pkt->hdr.len), 915 915 + le16_to_cpu(pkt->hdr.type), 916 916 + le16_to_cpu(pkt->hdr.op), 917 917 + le32_to_cpu(pkt->hdr.flags), 918 918 + le32_to_cpu(pkt->hdr.buf_alloc), 919 919 + le32_to_cpu(pkt->hdr.fwd_cnt)); 920 920 + 921 921 + if (le16_to_cpu(pkt->hdr.type) != VIRTIO_VSOCK_TYPE_STREAM) { 922 922 + (void)virtio_transport_reset_no_sock(pkt); 923 923 + goto free_pkt; 924 924 + } 925 925 + 926 926 + /* The socket must be in connected or bound table 927 927 + * otherwise send reset back 928 928 + */ 929 929 + sk = vsock_find_connected_socket(&src, &dst); 930 930 + if (!sk) { 931 931 + sk = vsock_find_bound_socket(&dst); 932 932 + if (!sk) { 933 933 + (void)virtio_transport_reset_no_sock(pkt); 934 934 + goto free_pkt; 935 935 + } 936 936 + } 937 937 + 938 938 + vsk = vsock_sk(sk); 939 939 + 940 940 + space_available = virtio_transport_space_update(sk, pkt); 941 941 + 942 942 + lock_sock(sk); 943 943 + 944 944 + /* Update CID in case it has changed after a transport reset event */ 945 945 + vsk->local_addr.svm_cid = dst.svm_cid; 946 946 + 947 947 + if (space_available) 948 948 + sk->sk_write_space(sk); 949 949 + 950 950 + switch (sk->sk_state) { 951 951 + case VSOCK_SS_LISTEN: 952 952 + virtio_transport_recv_listen(sk, pkt); 953 953 + virtio_transport_free_pkt(pkt); 954 954 + break; 955 955 + case SS_CONNECTING: 956 956 + virtio_transport_recv_connecting(sk, pkt); 957 957 + virtio_transport_free_pkt(pkt); 958 958 + break; 959 959 + case SS_CONNECTED: 960 960 + virtio_transport_recv_connected(sk, pkt); 961 961 + break; 962 962 + case SS_DISCONNECTING: 963 963 + virtio_transport_recv_disconnecting(sk, pkt); 964 964 + virtio_transport_free_pkt(pkt); 965 965 + break; 966 966 + default: 967 967 + virtio_transport_free_pkt(pkt); 968 968 + break; 969 969 + } 970 970 + release_sock(sk); 971 971 + 972 972 + /* Release refcnt obtained when we fetched this socket out of the 973 973 + * bound or connected list. 974 974 + */ 975 975 + sock_put(sk); 976 976 + return; 977 977 + 978 978 + free_pkt: 979 979 + virtio_transport_free_pkt(pkt); 980 980 + } 981 981 + EXPORT_SYMBOL_GPL(virtio_transport_recv_pkt); 982 982 + 983 983 + void virtio_transport_free_pkt(struct virtio_vsock_pkt *pkt) 984 984 + { 985 985 + kfree(pkt->buf); 986 986 + kfree(pkt); 987 987 + } 988 988 + EXPORT_SYMBOL_GPL(virtio_transport_free_pkt); 989 989 + 990 990 + MODULE_LICENSE("GPL v2"); 991 991 + MODULE_AUTHOR("Asias He"); 992 992 + MODULE_DESCRIPTION("common code for virtio vsock");

+2

net/vmw_vsock/vmci_transport.c

reviewed

··· 1644 1644 1645 1645 static void vmci_transport_release(struct vsock_sock *vsk) 1646 1646 { 1647 1647 + vsock_remove_sock(vsk); 1648 1648 + 1647 1649 if (!vmci_handle_is_invalid(vmci_trans(vsk)->dg_handle)) { 1648 1650 vmci_datagram_destroy_handle(vmci_trans(vsk)->dg_handle); 1649 1651 vmci_trans(vsk)->dg_handle = VMCI_INVALID_HANDLE;