Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost

+1 -1

drivers/net/virtio_net.c

··· 1331 1331 } 1332 1332 } 1333 1333 1334 - if (rq->vq->num_free > virtqueue_get_vring_size(rq->vq) / 2) { 1334 + if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) { 1335 1335 if (!try_fill_recv(vi, rq, GFP_ATOMIC)) 1336 1336 schedule_delayed_work(&vi->refill, 0); 1337 1337 }

+9 -4

drivers/vhost/test.c

··· 22 22 * Using this limit prevents one virtqueue from starving others. */ 23 23 #define VHOST_TEST_WEIGHT 0x80000 24 24 25 + /* Max number of packets transferred before requeueing the job. 26 + * Using this limit prevents one virtqueue from starving others with 27 + * pkts. 28 + */ 29 + #define VHOST_TEST_PKT_WEIGHT 256 30 + 25 31 enum { 26 32 VHOST_TEST_VQ = 0, 27 33 VHOST_TEST_VQ_MAX = 1, ··· 86 80 } 87 81 vhost_add_used_and_signal(&n->dev, vq, head, 0); 88 82 total_len += len; 89 - if (unlikely(total_len >= VHOST_TEST_WEIGHT)) { 90 - vhost_poll_queue(&vq->poll); 83 + if (unlikely(vhost_exceeds_weight(vq, 0, total_len))) 91 84 break; 92 - } 93 85 } 94 86 95 87 mutex_unlock(&vq->mutex); ··· 119 115 dev = &n->dev; 120 116 vqs[VHOST_TEST_VQ] = &n->vqs[VHOST_TEST_VQ]; 121 117 n->vqs[VHOST_TEST_VQ].handle_kick = handle_vq_kick; 122 - vhost_dev_init(dev, vqs, VHOST_TEST_VQ_MAX); 118 + vhost_dev_init(dev, vqs, VHOST_TEST_VQ_MAX, UIO_MAXIOV, 119 + VHOST_TEST_PKT_WEIGHT, VHOST_TEST_WEIGHT); 123 120 124 121 f->private_data = n; 125 122

+5 -515

drivers/vhost/vhost.c

··· 203 203 int vhost_poll_start(struct vhost_poll *poll, struct file *file) 204 204 { 205 205 __poll_t mask; 206 - int ret = 0; 207 206 208 207 if (poll->wqh) 209 208 return 0; ··· 212 213 vhost_poll_wakeup(&poll->wait, 0, 0, poll_to_key(mask)); 213 214 if (mask & EPOLLERR) { 214 215 vhost_poll_stop(poll); 215 - ret = -EINVAL; 216 + return -EINVAL; 216 217 } 217 218 218 - return ret; 219 + return 0; 219 220 } 220 221 EXPORT_SYMBOL_GPL(vhost_poll_start); 221 222 ··· 297 298 __vhost_vq_meta_reset(d->vqs[i]); 298 299 } 299 300 300 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 301 - static void vhost_map_unprefetch(struct vhost_map *map) 302 - { 303 - kfree(map->pages); 304 - map->pages = NULL; 305 - map->npages = 0; 306 - map->addr = NULL; 307 - } 308 - 309 - static void vhost_uninit_vq_maps(struct vhost_virtqueue *vq) 310 - { 311 - struct vhost_map *map[VHOST_NUM_ADDRS]; 312 - int i; 313 - 314 - spin_lock(&vq->mmu_lock); 315 - for (i = 0; i < VHOST_NUM_ADDRS; i++) { 316 - map[i] = rcu_dereference_protected(vq->maps[i], 317 - lockdep_is_held(&vq->mmu_lock)); 318 - if (map[i]) 319 - rcu_assign_pointer(vq->maps[i], NULL); 320 - } 321 - spin_unlock(&vq->mmu_lock); 322 - 323 - synchronize_rcu(); 324 - 325 - for (i = 0; i < VHOST_NUM_ADDRS; i++) 326 - if (map[i]) 327 - vhost_map_unprefetch(map[i]); 328 - 329 - } 330 - 331 - static void vhost_reset_vq_maps(struct vhost_virtqueue *vq) 332 - { 333 - int i; 334 - 335 - vhost_uninit_vq_maps(vq); 336 - for (i = 0; i < VHOST_NUM_ADDRS; i++) 337 - vq->uaddrs[i].size = 0; 338 - } 339 - 340 - static bool vhost_map_range_overlap(struct vhost_uaddr *uaddr, 341 - unsigned long start, 342 - unsigned long end) 343 - { 344 - if (unlikely(!uaddr->size)) 345 - return false; 346 - 347 - return !(end < uaddr->uaddr || start > uaddr->uaddr - 1 + uaddr->size); 348 - } 349 - 350 - static void vhost_invalidate_vq_start(struct vhost_virtqueue *vq, 351 - int index, 352 - unsigned long start, 353 - unsigned long end) 354 - { 355 - struct vhost_uaddr *uaddr = &vq->uaddrs[index]; 356 - struct vhost_map *map; 357 - int i; 358 - 359 - if (!vhost_map_range_overlap(uaddr, start, end)) 360 - return; 361 - 362 - spin_lock(&vq->mmu_lock); 363 - ++vq->invalidate_count; 364 - 365 - map = rcu_dereference_protected(vq->maps[index], 366 - lockdep_is_held(&vq->mmu_lock)); 367 - if (map) { 368 - if (uaddr->write) { 369 - for (i = 0; i < map->npages; i++) 370 - set_page_dirty(map->pages[i]); 371 - } 372 - rcu_assign_pointer(vq->maps[index], NULL); 373 - } 374 - spin_unlock(&vq->mmu_lock); 375 - 376 - if (map) { 377 - synchronize_rcu(); 378 - vhost_map_unprefetch(map); 379 - } 380 - } 381 - 382 - static void vhost_invalidate_vq_end(struct vhost_virtqueue *vq, 383 - int index, 384 - unsigned long start, 385 - unsigned long end) 386 - { 387 - if (!vhost_map_range_overlap(&vq->uaddrs[index], start, end)) 388 - return; 389 - 390 - spin_lock(&vq->mmu_lock); 391 - --vq->invalidate_count; 392 - spin_unlock(&vq->mmu_lock); 393 - } 394 - 395 - static int vhost_invalidate_range_start(struct mmu_notifier *mn, 396 - const struct mmu_notifier_range *range) 397 - { 398 - struct vhost_dev *dev = container_of(mn, struct vhost_dev, 399 - mmu_notifier); 400 - int i, j; 401 - 402 - if (!mmu_notifier_range_blockable(range)) 403 - return -EAGAIN; 404 - 405 - for (i = 0; i < dev->nvqs; i++) { 406 - struct vhost_virtqueue *vq = dev->vqs[i]; 407 - 408 - for (j = 0; j < VHOST_NUM_ADDRS; j++) 409 - vhost_invalidate_vq_start(vq, j, 410 - range->start, 411 - range->end); 412 - } 413 - 414 - return 0; 415 - } 416 - 417 - static void vhost_invalidate_range_end(struct mmu_notifier *mn, 418 - const struct mmu_notifier_range *range) 419 - { 420 - struct vhost_dev *dev = container_of(mn, struct vhost_dev, 421 - mmu_notifier); 422 - int i, j; 423 - 424 - for (i = 0; i < dev->nvqs; i++) { 425 - struct vhost_virtqueue *vq = dev->vqs[i]; 426 - 427 - for (j = 0; j < VHOST_NUM_ADDRS; j++) 428 - vhost_invalidate_vq_end(vq, j, 429 - range->start, 430 - range->end); 431 - } 432 - } 433 - 434 - static const struct mmu_notifier_ops vhost_mmu_notifier_ops = { 435 - .invalidate_range_start = vhost_invalidate_range_start, 436 - .invalidate_range_end = vhost_invalidate_range_end, 437 - }; 438 - 439 - static void vhost_init_maps(struct vhost_dev *dev) 440 - { 441 - struct vhost_virtqueue *vq; 442 - int i, j; 443 - 444 - dev->mmu_notifier.ops = &vhost_mmu_notifier_ops; 445 - 446 - for (i = 0; i < dev->nvqs; ++i) { 447 - vq = dev->vqs[i]; 448 - for (j = 0; j < VHOST_NUM_ADDRS; j++) 449 - RCU_INIT_POINTER(vq->maps[j], NULL); 450 - } 451 - } 452 - #endif 453 - 454 301 static void vhost_vq_reset(struct vhost_dev *dev, 455 302 struct vhost_virtqueue *vq) 456 303 { ··· 325 480 vq->busyloop_timeout = 0; 326 481 vq->umem = NULL; 327 482 vq->iotlb = NULL; 328 - vq->invalidate_count = 0; 329 483 __vhost_vq_meta_reset(vq); 330 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 331 - vhost_reset_vq_maps(vq); 332 - #endif 333 484 } 334 485 335 486 static int vhost_worker(void *data) ··· 475 634 INIT_LIST_HEAD(&dev->read_list); 476 635 INIT_LIST_HEAD(&dev->pending_list); 477 636 spin_lock_init(&dev->iotlb_lock); 478 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 479 - vhost_init_maps(dev); 480 - #endif 637 + 481 638 482 639 for (i = 0; i < dev->nvqs; ++i) { 483 640 vq = dev->vqs[i]; ··· 484 645 vq->heads = NULL; 485 646 vq->dev = dev; 486 647 mutex_init(&vq->mutex); 487 - spin_lock_init(&vq->mmu_lock); 488 648 vhost_vq_reset(dev, vq); 489 649 if (vq->handle_kick) 490 650 vhost_poll_init(&vq->poll, vq->handle_kick, ··· 563 725 if (err) 564 726 goto err_cgroup; 565 727 566 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 567 - err = mmu_notifier_register(&dev->mmu_notifier, dev->mm); 568 - if (err) 569 - goto err_mmu_notifier; 570 - #endif 571 - 572 728 return 0; 573 - 574 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 575 - err_mmu_notifier: 576 - vhost_dev_free_iovecs(dev); 577 - #endif 578 729 err_cgroup: 579 730 kthread_stop(worker); 580 731 dev->worker = NULL; ··· 654 827 spin_unlock(&dev->iotlb_lock); 655 828 } 656 829 657 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 658 - static void vhost_setup_uaddr(struct vhost_virtqueue *vq, 659 - int index, unsigned long uaddr, 660 - size_t size, bool write) 661 - { 662 - struct vhost_uaddr *addr = &vq->uaddrs[index]; 663 - 664 - addr->uaddr = uaddr; 665 - addr->size = size; 666 - addr->write = write; 667 - } 668 - 669 - static void vhost_setup_vq_uaddr(struct vhost_virtqueue *vq) 670 - { 671 - vhost_setup_uaddr(vq, VHOST_ADDR_DESC, 672 - (unsigned long)vq->desc, 673 - vhost_get_desc_size(vq, vq->num), 674 - false); 675 - vhost_setup_uaddr(vq, VHOST_ADDR_AVAIL, 676 - (unsigned long)vq->avail, 677 - vhost_get_avail_size(vq, vq->num), 678 - false); 679 - vhost_setup_uaddr(vq, VHOST_ADDR_USED, 680 - (unsigned long)vq->used, 681 - vhost_get_used_size(vq, vq->num), 682 - true); 683 - } 684 - 685 - static int vhost_map_prefetch(struct vhost_virtqueue *vq, 686 - int index) 687 - { 688 - struct vhost_map *map; 689 - struct vhost_uaddr *uaddr = &vq->uaddrs[index]; 690 - struct page **pages; 691 - int npages = DIV_ROUND_UP(uaddr->size, PAGE_SIZE); 692 - int npinned; 693 - void *vaddr, *v; 694 - int err; 695 - int i; 696 - 697 - spin_lock(&vq->mmu_lock); 698 - 699 - err = -EFAULT; 700 - if (vq->invalidate_count) 701 - goto err; 702 - 703 - err = -ENOMEM; 704 - map = kmalloc(sizeof(*map), GFP_ATOMIC); 705 - if (!map) 706 - goto err; 707 - 708 - pages = kmalloc_array(npages, sizeof(struct page *), GFP_ATOMIC); 709 - if (!pages) 710 - goto err_pages; 711 - 712 - err = EFAULT; 713 - npinned = __get_user_pages_fast(uaddr->uaddr, npages, 714 - uaddr->write, pages); 715 - if (npinned > 0) 716 - release_pages(pages, npinned); 717 - if (npinned != npages) 718 - goto err_gup; 719 - 720 - for (i = 0; i < npinned; i++) 721 - if (PageHighMem(pages[i])) 722 - goto err_gup; 723 - 724 - vaddr = v = page_address(pages[0]); 725 - 726 - /* For simplicity, fallback to userspace address if VA is not 727 - * contigious. 728 - */ 729 - for (i = 1; i < npinned; i++) { 730 - v += PAGE_SIZE; 731 - if (v != page_address(pages[i])) 732 - goto err_gup; 733 - } 734 - 735 - map->addr = vaddr + (uaddr->uaddr & (PAGE_SIZE - 1)); 736 - map->npages = npages; 737 - map->pages = pages; 738 - 739 - rcu_assign_pointer(vq->maps[index], map); 740 - /* No need for a synchronize_rcu(). This function should be 741 - * called by dev->worker so we are serialized with all 742 - * readers. 743 - */ 744 - spin_unlock(&vq->mmu_lock); 745 - 746 - return 0; 747 - 748 - err_gup: 749 - kfree(pages); 750 - err_pages: 751 - kfree(map); 752 - err: 753 - spin_unlock(&vq->mmu_lock); 754 - return err; 755 - } 756 - #endif 757 - 758 830 void vhost_dev_cleanup(struct vhost_dev *dev) 759 831 { 760 832 int i; ··· 683 957 kthread_stop(dev->worker); 684 958 dev->worker = NULL; 685 959 } 686 - if (dev->mm) { 687 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 688 - mmu_notifier_unregister(&dev->mmu_notifier, dev->mm); 689 - #endif 960 + if (dev->mm) 690 961 mmput(dev->mm); 691 - } 692 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 693 - for (i = 0; i < dev->nvqs; i++) 694 - vhost_uninit_vq_maps(dev->vqs[i]); 695 - #endif 696 962 dev->mm = NULL; 697 963 } 698 964 EXPORT_SYMBOL_GPL(vhost_dev_cleanup); ··· 913 1195 914 1196 static inline int vhost_put_avail_event(struct vhost_virtqueue *vq) 915 1197 { 916 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 917 - struct vhost_map *map; 918 - struct vring_used *used; 919 - 920 - if (!vq->iotlb) { 921 - rcu_read_lock(); 922 - 923 - map = rcu_dereference(vq->maps[VHOST_ADDR_USED]); 924 - if (likely(map)) { 925 - used = map->addr; 926 - *((__virtio16 *)&used->ring[vq->num]) = 927 - cpu_to_vhost16(vq, vq->avail_idx); 928 - rcu_read_unlock(); 929 - return 0; 930 - } 931 - 932 - rcu_read_unlock(); 933 - } 934 - #endif 935 - 936 1198 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->avail_idx), 937 1199 vhost_avail_event(vq)); 938 1200 } ··· 921 1223 struct vring_used_elem *head, int idx, 922 1224 int count) 923 1225 { 924 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 925 - struct vhost_map *map; 926 - struct vring_used *used; 927 - size_t size; 928 - 929 - if (!vq->iotlb) { 930 - rcu_read_lock(); 931 - 932 - map = rcu_dereference(vq->maps[VHOST_ADDR_USED]); 933 - if (likely(map)) { 934 - used = map->addr; 935 - size = count * sizeof(*head); 936 - memcpy(used->ring + idx, head, size); 937 - rcu_read_unlock(); 938 - return 0; 939 - } 940 - 941 - rcu_read_unlock(); 942 - } 943 - #endif 944 - 945 1226 return vhost_copy_to_user(vq, vq->used->ring + idx, head, 946 1227 count * sizeof(*head)); 947 1228 } ··· 928 1251 static inline int vhost_put_used_flags(struct vhost_virtqueue *vq) 929 1252 930 1253 { 931 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 932 - struct vhost_map *map; 933 - struct vring_used *used; 934 - 935 - if (!vq->iotlb) { 936 - rcu_read_lock(); 937 - 938 - map = rcu_dereference(vq->maps[VHOST_ADDR_USED]); 939 - if (likely(map)) { 940 - used = map->addr; 941 - used->flags = cpu_to_vhost16(vq, vq->used_flags); 942 - rcu_read_unlock(); 943 - return 0; 944 - } 945 - 946 - rcu_read_unlock(); 947 - } 948 - #endif 949 - 950 1254 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->used_flags), 951 1255 &vq->used->flags); 952 1256 } ··· 935 1277 static inline int vhost_put_used_idx(struct vhost_virtqueue *vq) 936 1278 937 1279 { 938 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 939 - struct vhost_map *map; 940 - struct vring_used *used; 941 - 942 - if (!vq->iotlb) { 943 - rcu_read_lock(); 944 - 945 - map = rcu_dereference(vq->maps[VHOST_ADDR_USED]); 946 - if (likely(map)) { 947 - used = map->addr; 948 - used->idx = cpu_to_vhost16(vq, vq->last_used_idx); 949 - rcu_read_unlock(); 950 - return 0; 951 - } 952 - 953 - rcu_read_unlock(); 954 - } 955 - #endif 956 - 957 1280 return vhost_put_user(vq, cpu_to_vhost16(vq, vq->last_used_idx), 958 1281 &vq->used->idx); 959 1282 } ··· 980 1341 static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq, 981 1342 __virtio16 *idx) 982 1343 { 983 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 984 - struct vhost_map *map; 985 - struct vring_avail *avail; 986 - 987 - if (!vq->iotlb) { 988 - rcu_read_lock(); 989 - 990 - map = rcu_dereference(vq->maps[VHOST_ADDR_AVAIL]); 991 - if (likely(map)) { 992 - avail = map->addr; 993 - *idx = avail->idx; 994 - rcu_read_unlock(); 995 - return 0; 996 - } 997 - 998 - rcu_read_unlock(); 999 - } 1000 - #endif 1001 - 1002 1344 return vhost_get_avail(vq, *idx, &vq->avail->idx); 1003 1345 } 1004 1346 1005 1347 static inline int vhost_get_avail_head(struct vhost_virtqueue *vq, 1006 1348 __virtio16 *head, int idx) 1007 1349 { 1008 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 1009 - struct vhost_map *map; 1010 - struct vring_avail *avail; 1011 - 1012 - if (!vq->iotlb) { 1013 - rcu_read_lock(); 1014 - 1015 - map = rcu_dereference(vq->maps[VHOST_ADDR_AVAIL]); 1016 - if (likely(map)) { 1017 - avail = map->addr; 1018 - *head = avail->ring[idx & (vq->num - 1)]; 1019 - rcu_read_unlock(); 1020 - return 0; 1021 - } 1022 - 1023 - rcu_read_unlock(); 1024 - } 1025 - #endif 1026 - 1027 1350 return vhost_get_avail(vq, *head, 1028 1351 &vq->avail->ring[idx & (vq->num - 1)]); 1029 1352 } ··· 993 1392 static inline int vhost_get_avail_flags(struct vhost_virtqueue *vq, 994 1393 __virtio16 *flags) 995 1394 { 996 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 997 - struct vhost_map *map; 998 - struct vring_avail *avail; 999 - 1000 - if (!vq->iotlb) { 1001 - rcu_read_lock(); 1002 - 1003 - map = rcu_dereference(vq->maps[VHOST_ADDR_AVAIL]); 1004 - if (likely(map)) { 1005 - avail = map->addr; 1006 - *flags = avail->flags; 1007 - rcu_read_unlock(); 1008 - return 0; 1009 - } 1010 - 1011 - rcu_read_unlock(); 1012 - } 1013 - #endif 1014 - 1015 1395 return vhost_get_avail(vq, *flags, &vq->avail->flags); 1016 1396 } 1017 1397 1018 1398 static inline int vhost_get_used_event(struct vhost_virtqueue *vq, 1019 1399 __virtio16 *event) 1020 1400 { 1021 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 1022 - struct vhost_map *map; 1023 - struct vring_avail *avail; 1024 - 1025 - if (!vq->iotlb) { 1026 - rcu_read_lock(); 1027 - map = rcu_dereference(vq->maps[VHOST_ADDR_AVAIL]); 1028 - if (likely(map)) { 1029 - avail = map->addr; 1030 - *event = (__virtio16)avail->ring[vq->num]; 1031 - rcu_read_unlock(); 1032 - return 0; 1033 - } 1034 - rcu_read_unlock(); 1035 - } 1036 - #endif 1037 - 1038 1401 return vhost_get_avail(vq, *event, vhost_used_event(vq)); 1039 1402 } 1040 1403 1041 1404 static inline int vhost_get_used_idx(struct vhost_virtqueue *vq, 1042 1405 __virtio16 *idx) 1043 1406 { 1044 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 1045 - struct vhost_map *map; 1046 - struct vring_used *used; 1047 - 1048 - if (!vq->iotlb) { 1049 - rcu_read_lock(); 1050 - 1051 - map = rcu_dereference(vq->maps[VHOST_ADDR_USED]); 1052 - if (likely(map)) { 1053 - used = map->addr; 1054 - *idx = used->idx; 1055 - rcu_read_unlock(); 1056 - return 0; 1057 - } 1058 - 1059 - rcu_read_unlock(); 1060 - } 1061 - #endif 1062 - 1063 1407 return vhost_get_used(vq, *idx, &vq->used->idx); 1064 1408 } 1065 1409 1066 1410 static inline int vhost_get_desc(struct vhost_virtqueue *vq, 1067 1411 struct vring_desc *desc, int idx) 1068 1412 { 1069 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 1070 - struct vhost_map *map; 1071 - struct vring_desc *d; 1072 - 1073 - if (!vq->iotlb) { 1074 - rcu_read_lock(); 1075 - 1076 - map = rcu_dereference(vq->maps[VHOST_ADDR_DESC]); 1077 - if (likely(map)) { 1078 - d = map->addr; 1079 - *desc = *(d + idx); 1080 - rcu_read_unlock(); 1081 - return 0; 1082 - } 1083 - 1084 - rcu_read_unlock(); 1085 - } 1086 - #endif 1087 - 1088 1413 return vhost_copy_from_user(vq, desc, vq->desc + idx, sizeof(*desc)); 1089 1414 } 1090 1415 ··· 1351 1824 return true; 1352 1825 } 1353 1826 1354 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 1355 - static void vhost_vq_map_prefetch(struct vhost_virtqueue *vq) 1356 - { 1357 - struct vhost_map __rcu *map; 1358 - int i; 1359 - 1360 - for (i = 0; i < VHOST_NUM_ADDRS; i++) { 1361 - rcu_read_lock(); 1362 - map = rcu_dereference(vq->maps[i]); 1363 - rcu_read_unlock(); 1364 - if (unlikely(!map)) 1365 - vhost_map_prefetch(vq, i); 1366 - } 1367 - } 1368 - #endif 1369 - 1370 1827 int vq_meta_prefetch(struct vhost_virtqueue *vq) 1371 1828 { 1372 1829 unsigned int num = vq->num; 1373 1830 1374 - if (!vq->iotlb) { 1375 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 1376 - vhost_vq_map_prefetch(vq); 1377 - #endif 1831 + if (!vq->iotlb) 1378 1832 return 1; 1379 - } 1380 1833 1381 1834 return iotlb_access_ok(vq, VHOST_ACCESS_RO, (u64)(uintptr_t)vq->desc, 1382 1835 vhost_get_desc_size(vq, num), VHOST_ADDR_DESC) && ··· 1567 2060 1568 2061 mutex_lock(&vq->mutex); 1569 2062 1570 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 1571 - /* Unregister MMU notifer to allow invalidation callback 1572 - * can access vq->uaddrs[] without holding a lock. 1573 - */ 1574 - if (d->mm) 1575 - mmu_notifier_unregister(&d->mmu_notifier, d->mm); 1576 - 1577 - vhost_uninit_vq_maps(vq); 1578 - #endif 1579 - 1580 2063 switch (ioctl) { 1581 2064 case VHOST_SET_VRING_NUM: 1582 2065 r = vhost_vring_set_num(d, vq, argp); ··· 1577 2080 default: 1578 2081 BUG(); 1579 2082 } 1580 - 1581 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 1582 - vhost_setup_vq_uaddr(vq); 1583 - 1584 - if (d->mm) 1585 - mmu_notifier_register(&d->mmu_notifier, d->mm); 1586 - #endif 1587 2083 1588 2084 mutex_unlock(&vq->mutex); 1589 2085

-41

drivers/vhost/vhost.h

··· 12 12 #include <linux/virtio_config.h> 13 13 #include <linux/virtio_ring.h> 14 14 #include <linux/atomic.h> 15 - #include <linux/pagemap.h> 16 - #include <linux/mmu_notifier.h> 17 - #include <asm/cacheflush.h> 18 15 19 16 struct vhost_work; 20 17 typedef void (*vhost_work_fn_t)(struct vhost_work *work); ··· 80 83 VHOST_NUM_ADDRS = 3, 81 84 }; 82 85 83 - struct vhost_map { 84 - int npages; 85 - void *addr; 86 - struct page **pages; 87 - }; 88 - 89 - struct vhost_uaddr { 90 - unsigned long uaddr; 91 - size_t size; 92 - bool write; 93 - }; 94 - 95 - #if defined(CONFIG_MMU_NOTIFIER) && ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE == 0 96 - #define VHOST_ARCH_CAN_ACCEL_UACCESS 0 97 - #else 98 - #define VHOST_ARCH_CAN_ACCEL_UACCESS 0 99 - #endif 100 - 101 86 /* The virtqueue structure describes a queue attached to a device. */ 102 87 struct vhost_virtqueue { 103 88 struct vhost_dev *dev; ··· 90 111 struct vring_desc __user *desc; 91 112 struct vring_avail __user *avail; 92 113 struct vring_used __user *used; 93 - 94 - #if VHOST_ARCH_CAN_ACCEL_UACCESS 95 - /* Read by memory accessors, modified by meta data 96 - * prefetching, MMU notifier and vring ioctl(). 97 - * Synchonrized through mmu_lock (writers) and RCU (writers 98 - * and readers). 99 - */ 100 - struct vhost_map __rcu *maps[VHOST_NUM_ADDRS]; 101 - /* Read by MMU notifier, modified by vring ioctl(), 102 - * synchronized through MMU notifier 103 - * registering/unregistering. 104 - */ 105 - struct vhost_uaddr uaddrs[VHOST_NUM_ADDRS]; 106 - #endif 107 114 const struct vhost_umem_node *meta_iotlb[VHOST_NUM_ADDRS]; 108 - 109 115 struct file *kick; 110 116 struct eventfd_ctx *call_ctx; 111 117 struct eventfd_ctx *error_ctx; ··· 145 181 bool user_be; 146 182 #endif 147 183 u32 busyloop_timeout; 148 - spinlock_t mmu_lock; 149 - int invalidate_count; 150 184 }; 151 185 152 186 struct vhost_msg_node { ··· 158 196 159 197 struct vhost_dev { 160 198 struct mm_struct *mm; 161 - #ifdef CONFIG_MMU_NOTIFIER 162 - struct mmu_notifier mmu_notifier; 163 - #endif 164 199 struct mutex mutex; 165 200 struct vhost_virtqueue **vqs; 166 201 int nvqs;

+2 -1

mm/balloon_compaction.c

··· 124 124 struct page *balloon_page_alloc(void) 125 125 { 126 126 struct page *page = alloc_page(balloon_mapping_gfp_mask() | 127 - __GFP_NOMEMALLOC | __GFP_NORETRY); 127 + __GFP_NOMEMALLOC | __GFP_NORETRY | 128 + __GFP_NOWARN); 128 129 return page; 129 130 } 130 131 EXPORT_SYMBOL_GPL(balloon_page_alloc);

Configure Feed

Configure Feed