Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
vhost: move smp_rmb() into vhost_get_avail_idx()
All callers of vhost_get_avail_idx() use smp_rmb() to
order the available ring entry read and avail_idx read.
Make vhost_get_avail_idx() call smp_rmb() itself whenever the avail_idx
is accessed. This way, the callers don't need to worry about the memory
barrier. As a side benefit, we also validate the index on all paths now,
which will hopefully help prevent/catch earlier future bugs.
Note that current code is inconsistent in how the errors are handled.
They are treated as an empty ring in some places, but as non-empty
ring in other places. This patch doesn't attempt to change the existing
behaviour.
No functional change intended.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Reviewed-by: Gavin Shan <gshan@redhat.com>
Acked-by: Will Deacon <will@kernel.org>
Message-Id: <20240429232748.642356-1-gshan@redhat.com>
+42
-63
+42
-63
drivers/vhost/vhost.c
+42
-63
drivers/vhost/vhost.c
···
1346
1346
mutex_unlock(&d->vqs[i]->mutex);
1347
1347
}
1348
1348
1349
-
static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq,
1350
-
__virtio16 *idx)
1349
+
static inline int vhost_get_avail_idx(struct vhost_virtqueue *vq)
1351
1350
{
1352
-
return vhost_get_avail(vq, *idx, &vq->avail->idx);
1351
+
__virtio16 idx;
1352
+
int r;
1353
+
1354
+
r = vhost_get_avail(vq, idx, &vq->avail->idx);
1355
+
if (unlikely(r < 0)) {
1356
+
vq_err(vq, "Failed to access available index at %p (%d)\n",
1357
+
&vq->avail->idx, r);
1358
+
return r;
1359
+
}
1360
+
1361
+
/* Check it isn't doing very strange thing with available indexes */
1362
+
vq->avail_idx = vhost16_to_cpu(vq, idx);
1363
+
if (unlikely((u16)(vq->avail_idx - vq->last_avail_idx) > vq->num)) {
1364
+
vq_err(vq, "Invalid available index change from %u to %u",
1365
+
vq->last_avail_idx, vq->avail_idx);
1366
+
return -EINVAL;
1367
+
}
1368
+
1369
+
/* We're done if there is nothing new */
1370
+
if (vq->avail_idx == vq->last_avail_idx)
1371
+
return 0;
1372
+
1373
+
/*
1374
+
* We updated vq->avail_idx so we need a memory barrier between
1375
+
* the index read above and the caller reading avail ring entries.
1376
+
*/
1377
+
smp_rmb();
1378
+
return 1;
1353
1379
}
1354
1380
1355
1381
static inline int vhost_get_avail_head(struct vhost_virtqueue *vq,
···
2580
2554
{
2581
2555
struct vring_desc desc;
2582
2556
unsigned int i, head, found = 0;
2583
-
u16 last_avail_idx;
2584
-
__virtio16 avail_idx;
2557
+
u16 last_avail_idx = vq->last_avail_idx;
2585
2558
__virtio16 ring_head;
2586
2559
int ret, access;
2587
2560
2588
-
/* Check it isn't doing very strange things with descriptor numbers. */
2589
-
last_avail_idx = vq->last_avail_idx;
2590
-
2591
2561
if (vq->avail_idx == vq->last_avail_idx) {
2592
-
if (unlikely(vhost_get_avail_idx(vq, &avail_idx))) {
2593
-
vq_err(vq, "Failed to access avail idx at %p\n",
2594
-
&vq->avail->idx);
2595
-
return -EFAULT;
2596
-
}
2597
-
vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
2562
+
ret = vhost_get_avail_idx(vq);
2563
+
if (unlikely(ret < 0))
2564
+
return ret;
2598
2565
2599
-
if (unlikely((u16)(vq->avail_idx - last_avail_idx) > vq->num)) {
2600
-
vq_err(vq, "Guest moved avail index from %u to %u",
2601
-
last_avail_idx, vq->avail_idx);
2602
-
return -EFAULT;
2603
-
}
2604
-
2605
-
/* If there's nothing new since last we looked, return
2606
-
* invalid.
2607
-
*/
2608
-
if (vq->avail_idx == last_avail_idx)
2566
+
if (!ret)
2609
2567
return vq->num;
2610
-
2611
-
/* Only get avail ring entries after they have been
2612
-
* exposed by guest.
2613
-
*/
2614
-
smp_rmb();
2615
2568
}
2616
2569
2617
2570
/* Grab the next descriptor number they're advertising, and increment
···
2851
2846
/* return true if we're sure that avaiable ring is empty */
2852
2847
bool vhost_vq_avail_empty(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2853
2848
{
2854
-
__virtio16 avail_idx;
2855
2849
int r;
2856
2850
2857
2851
if (vq->avail_idx != vq->last_avail_idx)
2858
2852
return false;
2859
2853
2860
-
r = vhost_get_avail_idx(vq, &avail_idx);
2861
-
if (unlikely(r))
2862
-
return false;
2854
+
r = vhost_get_avail_idx(vq);
2863
2855
2864
-
vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
2865
-
if (vq->avail_idx != vq->last_avail_idx) {
2866
-
/* Since we have updated avail_idx, the following
2867
-
* call to vhost_get_vq_desc() will read available
2868
-
* ring entries. Make sure that read happens after
2869
-
* the avail_idx read.
2870
-
*/
2871
-
smp_rmb();
2872
-
return false;
2873
-
}
2874
-
2875
-
return true;
2856
+
/* Note: we treat error as non-empty here */
2857
+
return r == 0;
2876
2858
}
2877
2859
EXPORT_SYMBOL_GPL(vhost_vq_avail_empty);
2878
2860
2879
2861
/* OK, now we need to know about added descriptors. */
2880
2862
bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
2881
2863
{
2882
-
__virtio16 avail_idx;
2883
2864
int r;
2884
2865
2885
2866
if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
···
2889
2898
/* They could have slipped one in as we were doing that: make
2890
2899
* sure it's written, then check again. */
2891
2900
smp_mb();
2892
-
r = vhost_get_avail_idx(vq, &avail_idx);
2893
-
if (r) {
2894
-
vq_err(vq, "Failed to check avail idx at %p: %d\n",
2895
-
&vq->avail->idx, r);
2901
+
2902
+
r = vhost_get_avail_idx(vq);
2903
+
/* Note: we treat error as empty here */
2904
+
if (unlikely(r < 0))
2896
2905
return false;
2897
-
}
2898
2906
2899
-
vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
2900
-
if (vq->avail_idx != vq->last_avail_idx) {
2901
-
/* Since we have updated avail_idx, the following
2902
-
* call to vhost_get_vq_desc() will read available
2903
-
* ring entries. Make sure that read happens after
2904
-
* the avail_idx read.
2905
-
*/
2906
-
smp_rmb();
2907
-
return true;
2908
-
}
2909
-
2910
-
return false;
2907
+
return r;
2911
2908
}
2912
2909
EXPORT_SYMBOL_GPL(vhost_enable_notify);
2913
2910