Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
Pull virtio updates from Michael Tsirkin:
"Several new features here:
- virtio-balloon supports new stats
- vdpa supports setting mac address
- vdpa/mlx5 suspend/resume as well as MKEY ops are now faster
- virtio_fs supports new sysfs entries for queue info
- virtio/vsock performance has been improved
And fixes, cleanups all over the place"
* tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost: (34 commits)
vsock/virtio: avoid queuing packets when intermediate queue is empty
vsock/virtio: refactor virtio_transport_send_pkt_work
fw_cfg: Constify struct kobj_type
vdpa/mlx5: Postpone MR deletion
vdpa/mlx5: Introduce init/destroy for MR resources
vdpa/mlx5: Rename mr_mtx -> lock
vdpa/mlx5: Extract mr members in own resource struct
vdpa/mlx5: Rename function
vdpa/mlx5: Delete direct MKEYs in parallel
vdpa/mlx5: Create direct MKEYs in parallel
MAINTAINERS: add virtio-vsock driver in the VIRTIO CORE section
virtio_fs: add sysfs entries for queue information
virtio_fs: introduce virtio_fs_put_locked helper
vdpa: Remove unused declarations
vdpa/mlx5: Parallelize VQ suspend/resume for CVQ MQ command
vdpa/mlx5: Small improvement for change_num_qps()
vdpa/mlx5: Keep notifiers during suspend but ignore
vdpa/mlx5: Parallelize device resume
vdpa/mlx5: Parallelize device suspend
vdpa/mlx5: Use async API for vq modify commands
...
+1098
-300
+1
MAINTAINERS
+1
MAINTAINERS
+1
-1
drivers/firmware/qemu_fw_cfg.c
+1
-1
drivers/firmware/qemu_fw_cfg.c
···
452
452
}
453
453
454
454
/* kobj_type: ties together all properties required to register an entry */
455
-
static struct kobj_type fw_cfg_sysfs_entry_ktype = {
455
+
static const struct kobj_type fw_cfg_sysfs_entry_ktype = {
456
456
.default_groups = fw_cfg_sysfs_entry_groups,
457
457
.sysfs_ops = &fw_cfg_sysfs_attr_ops,
458
458
.release = fw_cfg_sysfs_release_entry,
+16
-5
drivers/net/ethernet/mellanox/mlx5/core/cmd.c
+16
-5
drivers/net/ethernet/mellanox/mlx5/core/cmd.c
···
1887
1887
1888
1888
throttle_op = mlx5_cmd_is_throttle_opcode(opcode);
1889
1889
if (throttle_op) {
1890
-
/* atomic context may not sleep */
1891
-
if (callback)
1892
-
return -EINVAL;
1893
-
down(&dev->cmd.vars.throttle_sem);
1890
+
if (callback) {
1891
+
if (down_trylock(&dev->cmd.vars.throttle_sem))
1892
+
return -EBUSY;
1893
+
} else {
1894
+
down(&dev->cmd.vars.throttle_sem);
1895
+
}
1894
1896
}
1895
1897
1896
1898
pages_queue = is_manage_pages(in);
···
2098
2096
{
2099
2097
struct mlx5_async_work *work = _work;
2100
2098
struct mlx5_async_ctx *ctx;
2099
+
struct mlx5_core_dev *dev;
2100
+
u16 opcode;
2101
2101
2102
2102
ctx = work->ctx;
2103
-
status = cmd_status_err(ctx->dev, status, work->opcode, work->op_mod, work->out);
2103
+
dev = ctx->dev;
2104
+
opcode = work->opcode;
2105
+
status = cmd_status_err(dev, status, work->opcode, work->op_mod, work->out);
2104
2106
work->user_callback(status, work);
2107
+
/* Can't access "work" from this point on. It could have been freed in
2108
+
* the callback.
2109
+
*/
2110
+
if (mlx5_cmd_is_throttle_opcode(opcode))
2111
+
up(&dev->cmd.vars.throttle_sem);
2105
2112
if (atomic_dec_and_test(&ctx->num_inflight))
2106
2113
complete(&ctx->inflight_done);
2107
2114
}
+9
drivers/nvdimm/nd_virtio.c
+9
drivers/nvdimm/nd_virtio.c
···
44
44
unsigned long flags;
45
45
int err, err1;
46
46
47
+
/*
48
+
* Don't bother to submit the request to the device if the device is
49
+
* not activated.
50
+
*/
51
+
if (vdev->config->get_status(vdev) & VIRTIO_CONFIG_S_NEEDS_RESET) {
52
+
dev_info(&vdev->dev, "virtio pmem device needs a reset\n");
53
+
return -EIO;
54
+
}
55
+
47
56
might_sleep();
48
57
req_data = kmalloc(sizeof(*req_data), GFP_KERNEL);
49
58
if (!req_data)
-3
drivers/vdpa/ifcvf/ifcvf_base.h
-3
drivers/vdpa/ifcvf/ifcvf_base.h
···
112
112
const void *src, int length);
113
113
u8 ifcvf_get_status(struct ifcvf_hw *hw);
114
114
void ifcvf_set_status(struct ifcvf_hw *hw, u8 status);
115
-
void io_write64_twopart(u64 val, u32 *lo, u32 *hi);
116
115
void ifcvf_reset(struct ifcvf_hw *hw);
117
116
u64 ifcvf_get_dev_features(struct ifcvf_hw *hw);
118
117
u64 ifcvf_get_hw_features(struct ifcvf_hw *hw);
119
118
int ifcvf_verify_min_features(struct ifcvf_hw *hw, u64 features);
120
119
u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid);
121
120
int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num);
122
-
struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw);
123
-
int ifcvf_probed_virtio_net(struct ifcvf_hw *hw);
124
121
u32 ifcvf_get_config_size(struct ifcvf_hw *hw);
125
122
u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector);
126
123
u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector);
+42
-5
drivers/vdpa/mlx5/core/mlx5_vdpa.h
+42
-5
drivers/vdpa/mlx5/core/mlx5_vdpa.h
···
83
83
MLX5_VDPA_NUM_AS = 2
84
84
};
85
85
86
+
struct mlx5_vdpa_mr_resources {
87
+
struct mlx5_vdpa_mr *mr[MLX5_VDPA_NUM_AS];
88
+
unsigned int group2asid[MLX5_VDPA_NUMVQ_GROUPS];
89
+
90
+
/* Pre-deletion mr list */
91
+
struct list_head mr_list_head;
92
+
93
+
/* Deferred mr list */
94
+
struct list_head mr_gc_list_head;
95
+
struct workqueue_struct *wq_gc;
96
+
struct delayed_work gc_dwork_ent;
97
+
98
+
struct mutex lock;
99
+
100
+
atomic_t shutdown;
101
+
};
102
+
86
103
struct mlx5_vdpa_dev {
87
104
struct vdpa_device vdev;
88
105
struct mlx5_core_dev *mdev;
89
106
struct mlx5_vdpa_resources res;
107
+
struct mlx5_vdpa_mr_resources mres;
90
108
91
109
u64 mlx_features;
92
110
u64 actual_features;
···
113
95
u16 max_idx;
114
96
u32 generation;
115
97
116
-
struct mlx5_vdpa_mr *mr[MLX5_VDPA_NUM_AS];
117
-
struct list_head mr_list_head;
118
-
/* serialize mr access */
119
-
struct mutex mr_mtx;
120
98
struct mlx5_control_vq cvq;
121
99
struct workqueue_struct *wq;
122
-
unsigned int group2asid[MLX5_VDPA_NUMVQ_GROUPS];
123
100
bool suspended;
101
+
102
+
struct mlx5_async_ctx async_ctx;
103
+
};
104
+
105
+
struct mlx5_vdpa_async_cmd {
106
+
int err;
107
+
struct mlx5_async_work cb_work;
108
+
struct completion cmd_done;
109
+
110
+
void *in;
111
+
size_t inlen;
112
+
113
+
void *out;
114
+
size_t outlen;
124
115
};
125
116
126
117
int mlx5_vdpa_create_tis(struct mlx5_vdpa_dev *mvdev, void *in, u32 *tisn);
···
148
121
int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, u32 mkey);
149
122
struct mlx5_vdpa_mr *mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,
150
123
struct vhost_iotlb *iotlb);
124
+
int mlx5_vdpa_init_mr_resources(struct mlx5_vdpa_dev *mvdev);
151
125
void mlx5_vdpa_destroy_mr_resources(struct mlx5_vdpa_dev *mvdev);
126
+
void mlx5_vdpa_clean_mrs(struct mlx5_vdpa_dev *mvdev);
152
127
void mlx5_vdpa_get_mr(struct mlx5_vdpa_dev *mvdev,
153
128
struct mlx5_vdpa_mr *mr);
154
129
void mlx5_vdpa_put_mr(struct mlx5_vdpa_dev *mvdev,
···
163
134
unsigned int asid);
164
135
int mlx5_vdpa_create_dma_mr(struct mlx5_vdpa_dev *mvdev);
165
136
int mlx5_vdpa_reset_mr(struct mlx5_vdpa_dev *mvdev, unsigned int asid);
137
+
int mlx5_vdpa_exec_async_cmds(struct mlx5_vdpa_dev *mvdev,
138
+
struct mlx5_vdpa_async_cmd *cmds,
139
+
int num_cmds);
140
+
141
+
#define mlx5_vdpa_err(__dev, format, ...) \
142
+
dev_err((__dev)->mdev->device, "%s:%d:(pid %d) error: " format, __func__, __LINE__, \
143
+
current->pid, ##__VA_ARGS__)
144
+
166
145
167
146
#define mlx5_vdpa_warn(__dev, format, ...) \
168
147
dev_warn((__dev)->mdev->device, "%s:%d:(pid %d) warning: " format, __func__, __LINE__, \
+250
-41
drivers/vdpa/mlx5/core/mr.c
+250
-41
drivers/vdpa/mlx5/core/mr.c
···
49
49
}
50
50
}
51
51
52
-
static int create_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
53
-
{
54
-
int inlen;
55
-
void *mkc;
56
-
void *in;
57
-
int err;
52
+
struct mlx5_create_mkey_mem {
53
+
u8 out[MLX5_ST_SZ_BYTES(create_mkey_out)];
54
+
u8 in[MLX5_ST_SZ_BYTES(create_mkey_in)];
55
+
__be64 mtt[];
56
+
};
58
57
59
-
inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + roundup(MLX5_ST_SZ_BYTES(mtt) * mr->nsg, 16);
60
-
in = kvzalloc(inlen, GFP_KERNEL);
61
-
if (!in)
62
-
return -ENOMEM;
58
+
struct mlx5_destroy_mkey_mem {
59
+
u8 out[MLX5_ST_SZ_BYTES(destroy_mkey_out)];
60
+
u8 in[MLX5_ST_SZ_BYTES(destroy_mkey_in)];
61
+
};
62
+
63
+
static void fill_create_direct_mr(struct mlx5_vdpa_dev *mvdev,
64
+
struct mlx5_vdpa_direct_mr *mr,
65
+
struct mlx5_create_mkey_mem *mem)
66
+
{
67
+
void *in = &mem->in;
68
+
void *mkc;
63
69
64
70
MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
65
71
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
···
82
76
MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
83
77
get_octo_len(mr->end - mr->start, mr->log_size));
84
78
populate_mtts(mr, MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt));
85
-
err = mlx5_vdpa_create_mkey(mvdev, &mr->mr, in, inlen);
86
-
kvfree(in);
87
-
if (err) {
88
-
mlx5_vdpa_warn(mvdev, "Failed to create direct MR\n");
89
-
return err;
90
-
}
91
79
92
-
return 0;
80
+
MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
81
+
MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
82
+
}
83
+
84
+
static void create_direct_mr_end(struct mlx5_vdpa_dev *mvdev,
85
+
struct mlx5_vdpa_direct_mr *mr,
86
+
struct mlx5_create_mkey_mem *mem)
87
+
{
88
+
u32 mkey_index = MLX5_GET(create_mkey_out, mem->out, mkey_index);
89
+
90
+
mr->mr = mlx5_idx_to_mkey(mkey_index);
91
+
}
92
+
93
+
static void fill_destroy_direct_mr(struct mlx5_vdpa_dev *mvdev,
94
+
struct mlx5_vdpa_direct_mr *mr,
95
+
struct mlx5_destroy_mkey_mem *mem)
96
+
{
97
+
void *in = &mem->in;
98
+
99
+
MLX5_SET(destroy_mkey_in, in, uid, mvdev->res.uid);
100
+
MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY);
101
+
MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mr->mr));
93
102
}
94
103
95
104
static void destroy_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
96
105
{
106
+
if (!mr->mr)
107
+
return;
108
+
97
109
mlx5_vdpa_destroy_mkey(mvdev, mr->mr);
98
110
}
99
111
···
201
177
static int klm_byte_size(int nklms)
202
178
{
203
179
return 16 * ALIGN(nklms, 4);
180
+
}
181
+
182
+
#define MLX5_VDPA_MTT_ALIGN 16
183
+
184
+
static int create_direct_keys(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
185
+
{
186
+
struct mlx5_vdpa_async_cmd *cmds;
187
+
struct mlx5_vdpa_direct_mr *dmr;
188
+
int err = 0;
189
+
int i = 0;
190
+
191
+
cmds = kvcalloc(mr->num_directs, sizeof(*cmds), GFP_KERNEL);
192
+
if (!cmds)
193
+
return -ENOMEM;
194
+
195
+
list_for_each_entry(dmr, &mr->head, list) {
196
+
struct mlx5_create_mkey_mem *cmd_mem;
197
+
int mttlen, mttcount;
198
+
199
+
mttlen = roundup(MLX5_ST_SZ_BYTES(mtt) * dmr->nsg, MLX5_VDPA_MTT_ALIGN);
200
+
mttcount = mttlen / sizeof(cmd_mem->mtt[0]);
201
+
cmd_mem = kvcalloc(1, struct_size(cmd_mem, mtt, mttcount), GFP_KERNEL);
202
+
if (!cmd_mem) {
203
+
err = -ENOMEM;
204
+
goto done;
205
+
}
206
+
207
+
cmds[i].out = cmd_mem->out;
208
+
cmds[i].outlen = sizeof(cmd_mem->out);
209
+
cmds[i].in = cmd_mem->in;
210
+
cmds[i].inlen = struct_size(cmd_mem, mtt, mttcount);
211
+
212
+
fill_create_direct_mr(mvdev, dmr, cmd_mem);
213
+
214
+
i++;
215
+
}
216
+
217
+
err = mlx5_vdpa_exec_async_cmds(mvdev, cmds, mr->num_directs);
218
+
if (err) {
219
+
220
+
mlx5_vdpa_err(mvdev, "error issuing MTT mkey creation for direct mrs: %d\n", err);
221
+
goto done;
222
+
}
223
+
224
+
i = 0;
225
+
list_for_each_entry(dmr, &mr->head, list) {
226
+
struct mlx5_vdpa_async_cmd *cmd = &cmds[i++];
227
+
struct mlx5_create_mkey_mem *cmd_mem;
228
+
229
+
cmd_mem = container_of(cmd->out, struct mlx5_create_mkey_mem, out);
230
+
231
+
if (!cmd->err) {
232
+
create_direct_mr_end(mvdev, dmr, cmd_mem);
233
+
} else {
234
+
err = err ? err : cmd->err;
235
+
mlx5_vdpa_err(mvdev, "error creating MTT mkey [0x%llx, 0x%llx]: %d\n",
236
+
dmr->start, dmr->end, cmd->err);
237
+
}
238
+
}
239
+
240
+
done:
241
+
for (i = i-1; i >= 0; i--) {
242
+
struct mlx5_create_mkey_mem *cmd_mem;
243
+
244
+
cmd_mem = container_of(cmds[i].out, struct mlx5_create_mkey_mem, out);
245
+
kvfree(cmd_mem);
246
+
}
247
+
248
+
kvfree(cmds);
249
+
return err;
250
+
}
251
+
252
+
DEFINE_FREE(free_cmds, struct mlx5_vdpa_async_cmd *, kvfree(_T))
253
+
DEFINE_FREE(free_cmd_mem, struct mlx5_destroy_mkey_mem *, kvfree(_T))
254
+
255
+
static int destroy_direct_keys(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
256
+
{
257
+
struct mlx5_destroy_mkey_mem *cmd_mem __free(free_cmd_mem) = NULL;
258
+
struct mlx5_vdpa_async_cmd *cmds __free(free_cmds) = NULL;
259
+
struct mlx5_vdpa_direct_mr *dmr;
260
+
int err = 0;
261
+
int i = 0;
262
+
263
+
cmds = kvcalloc(mr->num_directs, sizeof(*cmds), GFP_KERNEL);
264
+
cmd_mem = kvcalloc(mr->num_directs, sizeof(*cmd_mem), GFP_KERNEL);
265
+
if (!cmds || !cmd_mem)
266
+
return -ENOMEM;
267
+
268
+
list_for_each_entry(dmr, &mr->head, list) {
269
+
cmds[i].out = cmd_mem[i].out;
270
+
cmds[i].outlen = sizeof(cmd_mem[i].out);
271
+
cmds[i].in = cmd_mem[i].in;
272
+
cmds[i].inlen = sizeof(cmd_mem[i].in);
273
+
fill_destroy_direct_mr(mvdev, dmr, &cmd_mem[i]);
274
+
i++;
275
+
}
276
+
277
+
err = mlx5_vdpa_exec_async_cmds(mvdev, cmds, mr->num_directs);
278
+
if (err) {
279
+
280
+
mlx5_vdpa_err(mvdev, "error issuing MTT mkey deletion for direct mrs: %d\n", err);
281
+
return err;
282
+
}
283
+
284
+
i = 0;
285
+
list_for_each_entry(dmr, &mr->head, list) {
286
+
struct mlx5_vdpa_async_cmd *cmd = &cmds[i++];
287
+
288
+
dmr->mr = 0;
289
+
if (cmd->err) {
290
+
err = err ? err : cmd->err;
291
+
mlx5_vdpa_err(mvdev, "error deleting MTT mkey [0x%llx, 0x%llx]: %d\n",
292
+
dmr->start, dmr->end, cmd->err);
293
+
}
294
+
}
295
+
296
+
return err;
204
297
}
205
298
206
299
static int create_indirect_key(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *mr)
···
420
279
goto err_map;
421
280
}
422
281
423
-
err = create_direct_mr(mvdev, mr);
424
-
if (err)
425
-
goto err_direct;
426
-
427
282
return 0;
428
283
429
-
err_direct:
430
-
dma_unmap_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0);
431
284
err_map:
432
285
sg_free_table(&mr->sg_head);
433
286
return err;
···
536
401
if (err)
537
402
goto err_chain;
538
403
404
+
err = create_direct_keys(mvdev, mr);
405
+
if (err)
406
+
goto err_chain;
407
+
539
408
/* Create the memory key that defines the guests's address space. This
540
409
* memory key refers to the direct keys that contain the MTT
541
410
* translations
···
628
489
struct mlx5_vdpa_direct_mr *n;
629
490
630
491
destroy_indirect_key(mvdev, mr);
492
+
destroy_direct_keys(mvdev, mr);
631
493
list_for_each_entry_safe_reverse(dmr, n, &mr->head, list) {
632
494
list_del_init(&dmr->list);
633
495
unmap_direct_mr(mvdev, dmr);
···
653
513
kfree(mr);
654
514
}
655
515
516
+
/* There can be multiple .set_map() operations in quick succession.
517
+
* This large delay is a simple way to prevent the MR cleanup from blocking
518
+
* .set_map() MR creation in this scenario.
519
+
*/
520
+
#define MLX5_VDPA_MR_GC_TRIGGER_MS 2000
521
+
522
+
static void mlx5_vdpa_mr_gc_handler(struct work_struct *work)
523
+
{
524
+
struct mlx5_vdpa_mr_resources *mres;
525
+
struct mlx5_vdpa_mr *mr, *tmp;
526
+
struct mlx5_vdpa_dev *mvdev;
527
+
528
+
mres = container_of(work, struct mlx5_vdpa_mr_resources, gc_dwork_ent.work);
529
+
530
+
if (atomic_read(&mres->shutdown)) {
531
+
mutex_lock(&mres->lock);
532
+
} else if (!mutex_trylock(&mres->lock)) {
533
+
queue_delayed_work(mres->wq_gc, &mres->gc_dwork_ent,
534
+
msecs_to_jiffies(MLX5_VDPA_MR_GC_TRIGGER_MS));
535
+
return;
536
+
}
537
+
538
+
mvdev = container_of(mres, struct mlx5_vdpa_dev, mres);
539
+
540
+
list_for_each_entry_safe(mr, tmp, &mres->mr_gc_list_head, mr_list) {
541
+
_mlx5_vdpa_destroy_mr(mvdev, mr);
542
+
}
543
+
544
+
mutex_unlock(&mres->lock);
545
+
}
546
+
656
547
static void _mlx5_vdpa_put_mr(struct mlx5_vdpa_dev *mvdev,
657
548
struct mlx5_vdpa_mr *mr)
658
549
{
550
+
struct mlx5_vdpa_mr_resources *mres = &mvdev->mres;
551
+
659
552
if (!mr)
660
553
return;
661
554
662
-
if (refcount_dec_and_test(&mr->refcount))
663
-
_mlx5_vdpa_destroy_mr(mvdev, mr);
555
+
if (refcount_dec_and_test(&mr->refcount)) {
556
+
list_move_tail(&mr->mr_list, &mres->mr_gc_list_head);
557
+
queue_delayed_work(mres->wq_gc, &mres->gc_dwork_ent,
558
+
msecs_to_jiffies(MLX5_VDPA_MR_GC_TRIGGER_MS));
559
+
}
664
560
}
665
561
666
562
void mlx5_vdpa_put_mr(struct mlx5_vdpa_dev *mvdev,
667
563
struct mlx5_vdpa_mr *mr)
668
564
{
669
-
mutex_lock(&mvdev->mr_mtx);
565
+
mutex_lock(&mvdev->mres.lock);
670
566
_mlx5_vdpa_put_mr(mvdev, mr);
671
-
mutex_unlock(&mvdev->mr_mtx);
567
+
mutex_unlock(&mvdev->mres.lock);
672
568
}
673
569
674
570
static void _mlx5_vdpa_get_mr(struct mlx5_vdpa_dev *mvdev,
···
719
543
void mlx5_vdpa_get_mr(struct mlx5_vdpa_dev *mvdev,
720
544
struct mlx5_vdpa_mr *mr)
721
545
{
722
-
mutex_lock(&mvdev->mr_mtx);
546
+
mutex_lock(&mvdev->mres.lock);
723
547
_mlx5_vdpa_get_mr(mvdev, mr);
724
-
mutex_unlock(&mvdev->mr_mtx);
548
+
mutex_unlock(&mvdev->mres.lock);
725
549
}
726
550
727
551
void mlx5_vdpa_update_mr(struct mlx5_vdpa_dev *mvdev,
728
552
struct mlx5_vdpa_mr *new_mr,
729
553
unsigned int asid)
730
554
{
731
-
struct mlx5_vdpa_mr *old_mr = mvdev->mr[asid];
555
+
struct mlx5_vdpa_mr *old_mr = mvdev->mres.mr[asid];
732
556
733
-
mutex_lock(&mvdev->mr_mtx);
557
+
mutex_lock(&mvdev->mres.lock);
734
558
735
559
_mlx5_vdpa_put_mr(mvdev, old_mr);
736
-
mvdev->mr[asid] = new_mr;
560
+
mvdev->mres.mr[asid] = new_mr;
737
561
738
-
mutex_unlock(&mvdev->mr_mtx);
562
+
mutex_unlock(&mvdev->mres.lock);
739
563
}
740
564
741
565
static void mlx5_vdpa_show_mr_leaks(struct mlx5_vdpa_dev *mvdev)
742
566
{
743
567
struct mlx5_vdpa_mr *mr;
744
568
745
-
mutex_lock(&mvdev->mr_mtx);
569
+
mutex_lock(&mvdev->mres.lock);
746
570
747
-
list_for_each_entry(mr, &mvdev->mr_list_head, mr_list) {
571
+
list_for_each_entry(mr, &mvdev->mres.mr_list_head, mr_list) {
748
572
749
573
mlx5_vdpa_warn(mvdev, "mkey still alive after resource delete: "
750
574
"mr: %p, mkey: 0x%x, refcount: %u\n",
751
575
mr, mr->mkey, refcount_read(&mr->refcount));
752
576
}
753
577
754
-
mutex_unlock(&mvdev->mr_mtx);
578
+
mutex_unlock(&mvdev->mres.lock);
755
579
756
580
}
757
581
758
-
void mlx5_vdpa_destroy_mr_resources(struct mlx5_vdpa_dev *mvdev)
582
+
void mlx5_vdpa_clean_mrs(struct mlx5_vdpa_dev *mvdev)
759
583
{
584
+
if (!mvdev->res.valid)
585
+
return;
586
+
760
587
for (int i = 0; i < MLX5_VDPA_NUM_AS; i++)
761
588
mlx5_vdpa_update_mr(mvdev, NULL, i);
762
589
···
792
613
if (err)
793
614
goto err_iotlb;
794
615
795
-
list_add_tail(&mr->mr_list, &mvdev->mr_list_head);
616
+
list_add_tail(&mr->mr_list, &mvdev->mres.mr_list_head);
796
617
797
618
return 0;
798
619
···
818
639
if (!mr)
819
640
return ERR_PTR(-ENOMEM);
820
641
821
-
mutex_lock(&mvdev->mr_mtx);
642
+
mutex_lock(&mvdev->mres.lock);
822
643
err = _mlx5_vdpa_create_mr(mvdev, mr, iotlb);
823
-
mutex_unlock(&mvdev->mr_mtx);
644
+
mutex_unlock(&mvdev->mres.lock);
824
645
825
646
if (err)
826
647
goto out_err;
···
840
661
{
841
662
int err;
842
663
843
-
if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] != asid)
664
+
if (mvdev->mres.group2asid[MLX5_VDPA_CVQ_GROUP] != asid)
844
665
return 0;
845
666
846
667
spin_lock(&mvdev->cvq.iommu_lock);
···
881
702
}
882
703
883
704
return 0;
705
+
}
706
+
707
+
int mlx5_vdpa_init_mr_resources(struct mlx5_vdpa_dev *mvdev)
708
+
{
709
+
struct mlx5_vdpa_mr_resources *mres = &mvdev->mres;
710
+
711
+
mres->wq_gc = create_singlethread_workqueue("mlx5_vdpa_mr_gc");
712
+
if (!mres->wq_gc)
713
+
return -ENOMEM;
714
+
715
+
INIT_DELAYED_WORK(&mres->gc_dwork_ent, mlx5_vdpa_mr_gc_handler);
716
+
717
+
mutex_init(&mres->lock);
718
+
719
+
INIT_LIST_HEAD(&mres->mr_list_head);
720
+
INIT_LIST_HEAD(&mres->mr_gc_list_head);
721
+
722
+
return 0;
723
+
}
724
+
725
+
void mlx5_vdpa_destroy_mr_resources(struct mlx5_vdpa_dev *mvdev)
726
+
{
727
+
struct mlx5_vdpa_mr_resources *mres = &mvdev->mres;
728
+
729
+
atomic_set(&mres->shutdown, 1);
730
+
731
+
flush_delayed_work(&mres->gc_dwork_ent);
732
+
destroy_workqueue(mres->wq_gc);
733
+
mres->wq_gc = NULL;
734
+
mutex_destroy(&mres->lock);
884
735
}
+73
-3
drivers/vdpa/mlx5/core/resources.c
+73
-3
drivers/vdpa/mlx5/core/resources.c
···
256
256
mlx5_vdpa_warn(mvdev, "resources already allocated\n");
257
257
return -EINVAL;
258
258
}
259
-
mutex_init(&mvdev->mr_mtx);
260
259
res->uar = mlx5_get_uars_page(mdev);
261
260
if (IS_ERR(res->uar)) {
262
261
err = PTR_ERR(res->uar);
···
300
301
err_uctx:
301
302
mlx5_put_uars_page(mdev, res->uar);
302
303
err_uars:
303
-
mutex_destroy(&mvdev->mr_mtx);
304
304
return err;
305
305
}
306
306
···
316
318
dealloc_pd(mvdev, res->pdn, res->uid);
317
319
destroy_uctx(mvdev, res->uid);
318
320
mlx5_put_uars_page(mvdev->mdev, res->uar);
319
-
mutex_destroy(&mvdev->mr_mtx);
320
321
res->valid = false;
322
+
}
323
+
324
+
static void virtqueue_cmd_callback(int status, struct mlx5_async_work *context)
325
+
{
326
+
struct mlx5_vdpa_async_cmd *cmd =
327
+
container_of(context, struct mlx5_vdpa_async_cmd, cb_work);
328
+
329
+
cmd->err = mlx5_cmd_check(context->ctx->dev, status, cmd->in, cmd->out);
330
+
complete(&cmd->cmd_done);
331
+
}
332
+
333
+
static int issue_async_cmd(struct mlx5_vdpa_dev *mvdev,
334
+
struct mlx5_vdpa_async_cmd *cmds,
335
+
int issued,
336
+
int *completed)
337
+
338
+
{
339
+
struct mlx5_vdpa_async_cmd *cmd = &cmds[issued];
340
+
int err;
341
+
342
+
retry:
343
+
err = mlx5_cmd_exec_cb(&mvdev->async_ctx,
344
+
cmd->in, cmd->inlen,
345
+
cmd->out, cmd->outlen,
346
+
virtqueue_cmd_callback,
347
+
&cmd->cb_work);
348
+
if (err == -EBUSY) {
349
+
if (*completed < issued) {
350
+
/* Throttled by own commands: wait for oldest completion. */
351
+
wait_for_completion(&cmds[*completed].cmd_done);
352
+
(*completed)++;
353
+
354
+
goto retry;
355
+
} else {
356
+
/* Throttled by external commands: switch to sync api. */
357
+
err = mlx5_cmd_exec(mvdev->mdev,
358
+
cmd->in, cmd->inlen,
359
+
cmd->out, cmd->outlen);
360
+
if (!err)
361
+
(*completed)++;
362
+
}
363
+
}
364
+
365
+
return err;
366
+
}
367
+
368
+
int mlx5_vdpa_exec_async_cmds(struct mlx5_vdpa_dev *mvdev,
369
+
struct mlx5_vdpa_async_cmd *cmds,
370
+
int num_cmds)
371
+
{
372
+
int completed = 0;
373
+
int issued = 0;
374
+
int err = 0;
375
+
376
+
for (int i = 0; i < num_cmds; i++)
377
+
init_completion(&cmds[i].cmd_done);
378
+
379
+
while (issued < num_cmds) {
380
+
381
+
err = issue_async_cmd(mvdev, cmds, issued, &completed);
382
+
if (err) {
383
+
mlx5_vdpa_err(mvdev, "error issuing command %d of %d: %d\n",
384
+
issued, num_cmds, err);
385
+
break;
386
+
}
387
+
388
+
issued++;
389
+
}
390
+
391
+
while (completed < issued)
392
+
wait_for_completion(&cmds[completed++].cmd_done);
393
+
394
+
return err;
321
395
}
+307
-170
drivers/vdpa/mlx5/net/mlx5_vnet.c
+307
-170
drivers/vdpa/mlx5/net/mlx5_vnet.c
···
941
941
MLX5_SET64(virtio_q, vq_ctx, used_addr, mvq->device_addr);
942
942
MLX5_SET64(virtio_q, vq_ctx, available_addr, mvq->driver_addr);
943
943
944
-
vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
944
+
vq_mr = mvdev->mres.mr[mvdev->mres.group2asid[MLX5_VDPA_DATAVQ_GROUP]];
945
945
if (vq_mr)
946
946
MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, vq_mr->mkey);
947
947
948
-
vq_desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
948
+
vq_desc_mr = mvdev->mres.mr[mvdev->mres.group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
949
949
if (vq_desc_mr &&
950
950
MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported))
951
951
MLX5_SET(virtio_q, vq_ctx, desc_group_mkey, vq_desc_mr->mkey);
···
953
953
/* If there is no mr update, make sure that the existing ones are set
954
954
* modify to ready.
955
955
*/
956
-
vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
956
+
vq_mr = mvdev->mres.mr[mvdev->mres.group2asid[MLX5_VDPA_DATAVQ_GROUP]];
957
957
if (vq_mr)
958
958
mvq->modified_fields |= MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY;
959
959
960
-
vq_desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
960
+
vq_desc_mr = mvdev->mres.mr[mvdev->mres.group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
961
961
if (vq_desc_mr)
962
962
mvq->modified_fields |= MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY;
963
963
}
···
1184
1184
u16 used_index;
1185
1185
};
1186
1186
1187
-
static int query_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq,
1188
-
struct mlx5_virtq_attr *attr)
1187
+
struct mlx5_virtqueue_query_mem {
1188
+
u8 in[MLX5_ST_SZ_BYTES(query_virtio_net_q_in)];
1189
+
u8 out[MLX5_ST_SZ_BYTES(query_virtio_net_q_out)];
1190
+
};
1191
+
1192
+
struct mlx5_virtqueue_modify_mem {
1193
+
u8 in[MLX5_ST_SZ_BYTES(modify_virtio_net_q_in)];
1194
+
u8 out[MLX5_ST_SZ_BYTES(modify_virtio_net_q_out)];
1195
+
};
1196
+
1197
+
static void fill_query_virtqueue_cmd(struct mlx5_vdpa_net *ndev,
1198
+
struct mlx5_vdpa_virtqueue *mvq,
1199
+
struct mlx5_virtqueue_query_mem *cmd)
1189
1200
{
1190
-
int outlen = MLX5_ST_SZ_BYTES(query_virtio_net_q_out);
1191
-
u32 in[MLX5_ST_SZ_DW(query_virtio_net_q_in)] = {};
1192
-
void *out;
1193
-
void *obj_context;
1194
-
void *cmd_hdr;
1195
-
int err;
1196
-
1197
-
out = kzalloc(outlen, GFP_KERNEL);
1198
-
if (!out)
1199
-
return -ENOMEM;
1200
-
1201
-
cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, in, general_obj_in_cmd_hdr);
1201
+
void *cmd_hdr = MLX5_ADDR_OF(query_virtio_net_q_in, cmd->in, general_obj_in_cmd_hdr);
1202
1202
1203
1203
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
1204
1204
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q);
1205
1205
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id);
1206
1206
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid);
1207
-
err = mlx5_cmd_exec(ndev->mvdev.mdev, in, sizeof(in), out, outlen);
1208
-
if (err)
1209
-
goto err_cmd;
1207
+
}
1210
1208
1211
-
obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, out, obj_context);
1209
+
static void query_virtqueue_end(struct mlx5_vdpa_net *ndev,
1210
+
struct mlx5_virtqueue_query_mem *cmd,
1211
+
struct mlx5_virtq_attr *attr)
1212
+
{
1213
+
void *obj_context = MLX5_ADDR_OF(query_virtio_net_q_out, cmd->out, obj_context);
1214
+
1212
1215
memset(attr, 0, sizeof(*attr));
1213
1216
attr->state = MLX5_GET(virtio_net_q_object, obj_context, state);
1214
1217
attr->available_index = MLX5_GET(virtio_net_q_object, obj_context, hw_available_index);
1215
1218
attr->used_index = MLX5_GET(virtio_net_q_object, obj_context, hw_used_index);
1216
-
kfree(out);
1217
-
return 0;
1219
+
}
1218
1220
1219
-
err_cmd:
1220
-
kfree(out);
1221
+
static int query_virtqueues(struct mlx5_vdpa_net *ndev,
1222
+
int start_vq,
1223
+
int num_vqs,
1224
+
struct mlx5_virtq_attr *attrs)
1225
+
{
1226
+
struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
1227
+
struct mlx5_virtqueue_query_mem *cmd_mem;
1228
+
struct mlx5_vdpa_async_cmd *cmds;
1229
+
int err = 0;
1230
+
1231
+
WARN(start_vq + num_vqs > mvdev->max_vqs, "query vq range invalid [%d, %d), max_vqs: %u\n",
1232
+
start_vq, start_vq + num_vqs, mvdev->max_vqs);
1233
+
1234
+
cmds = kvcalloc(num_vqs, sizeof(*cmds), GFP_KERNEL);
1235
+
cmd_mem = kvcalloc(num_vqs, sizeof(*cmd_mem), GFP_KERNEL);
1236
+
if (!cmds || !cmd_mem) {
1237
+
err = -ENOMEM;
1238
+
goto done;
1239
+
}
1240
+
1241
+
for (int i = 0; i < num_vqs; i++) {
1242
+
cmds[i].in = &cmd_mem[i].in;
1243
+
cmds[i].inlen = sizeof(cmd_mem[i].in);
1244
+
cmds[i].out = &cmd_mem[i].out;
1245
+
cmds[i].outlen = sizeof(cmd_mem[i].out);
1246
+
fill_query_virtqueue_cmd(ndev, &ndev->vqs[start_vq + i], &cmd_mem[i]);
1247
+
}
1248
+
1249
+
err = mlx5_vdpa_exec_async_cmds(&ndev->mvdev, cmds, num_vqs);
1250
+
if (err) {
1251
+
mlx5_vdpa_err(mvdev, "error issuing query cmd for vq range [%d, %d): %d\n",
1252
+
start_vq, start_vq + num_vqs, err);
1253
+
goto done;
1254
+
}
1255
+
1256
+
for (int i = 0; i < num_vqs; i++) {
1257
+
struct mlx5_vdpa_async_cmd *cmd = &cmds[i];
1258
+
int vq_idx = start_vq + i;
1259
+
1260
+
if (cmd->err) {
1261
+
mlx5_vdpa_err(mvdev, "query vq %d failed, err: %d\n", vq_idx, err);
1262
+
if (!err)
1263
+
err = cmd->err;
1264
+
continue;
1265
+
}
1266
+
1267
+
query_virtqueue_end(ndev, &cmd_mem[i], &attrs[i]);
1268
+
}
1269
+
1270
+
done:
1271
+
kvfree(cmd_mem);
1272
+
kvfree(cmds);
1221
1273
return err;
1222
1274
}
1223
1275
···
1303
1251
return true;
1304
1252
}
1305
1253
1306
-
static int modify_virtqueue(struct mlx5_vdpa_net *ndev,
1307
-
struct mlx5_vdpa_virtqueue *mvq,
1308
-
int state)
1254
+
static void fill_modify_virtqueue_cmd(struct mlx5_vdpa_net *ndev,
1255
+
struct mlx5_vdpa_virtqueue *mvq,
1256
+
int state,
1257
+
struct mlx5_virtqueue_modify_mem *cmd)
1309
1258
{
1310
-
int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in);
1311
-
u32 out[MLX5_ST_SZ_DW(modify_virtio_net_q_out)] = {};
1312
1259
struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
1313
1260
struct mlx5_vdpa_mr *desc_mr = NULL;
1314
1261
struct mlx5_vdpa_mr *vq_mr = NULL;
1315
-
bool state_change = false;
1316
1262
void *obj_context;
1317
1263
void *cmd_hdr;
1318
1264
void *vq_ctx;
1319
-
void *in;
1320
-
int err;
1321
1265
1322
-
if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_NONE)
1323
-
return 0;
1324
-
1325
-
if (!modifiable_virtqueue_fields(mvq))
1326
-
return -EINVAL;
1327
-
1328
-
in = kzalloc(inlen, GFP_KERNEL);
1329
-
if (!in)
1330
-
return -ENOMEM;
1331
-
1332
-
cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, in, general_obj_in_cmd_hdr);
1266
+
cmd_hdr = MLX5_ADDR_OF(modify_virtio_net_q_in, cmd->in, general_obj_in_cmd_hdr);
1333
1267
1334
1268
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
1335
1269
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_type, MLX5_OBJ_TYPE_VIRTIO_NET_Q);
1336
1270
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, obj_id, mvq->virtq_id);
1337
1271
MLX5_SET(general_obj_in_cmd_hdr, cmd_hdr, uid, ndev->mvdev.res.uid);
1338
1272
1339
-
obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, in, obj_context);
1273
+
obj_context = MLX5_ADDR_OF(modify_virtio_net_q_in, cmd->in, obj_context);
1340
1274
vq_ctx = MLX5_ADDR_OF(virtio_net_q_object, obj_context, virtio_q_context);
1341
1275
1342
-
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_STATE) {
1343
-
if (!is_valid_state_change(mvq->fw_state, state, is_resumable(ndev))) {
1344
-
err = -EINVAL;
1345
-
goto done;
1346
-
}
1347
-
1276
+
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_STATE)
1348
1277
MLX5_SET(virtio_net_q_object, obj_context, state, state);
1349
-
state_change = true;
1350
-
}
1351
1278
1352
1279
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_ADDRS) {
1353
1280
MLX5_SET64(virtio_q, vq_ctx, desc_addr, mvq->desc_addr);
···
1354
1323
}
1355
1324
1356
1325
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY) {
1357
-
vq_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP]];
1326
+
vq_mr = mvdev->mres.mr[mvdev->mres.group2asid[MLX5_VDPA_DATAVQ_GROUP]];
1358
1327
1359
1328
if (vq_mr)
1360
1329
MLX5_SET(virtio_q, vq_ctx, virtio_q_mkey, vq_mr->mkey);
···
1363
1332
}
1364
1333
1365
1334
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY) {
1366
-
desc_mr = mvdev->mr[mvdev->group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
1335
+
desc_mr = mvdev->mres.mr[mvdev->mres.group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP]];
1367
1336
1368
1337
if (desc_mr && MLX5_CAP_DEV_VDPA_EMULATION(mvdev->mdev, desc_group_mkey_supported))
1369
1338
MLX5_SET(virtio_q, vq_ctx, desc_group_mkey, desc_mr->mkey);
···
1372
1341
}
1373
1342
1374
1343
MLX5_SET64(virtio_net_q_object, obj_context, modify_field_select, mvq->modified_fields);
1375
-
err = mlx5_cmd_exec(ndev->mvdev.mdev, in, inlen, out, sizeof(out));
1376
-
if (err)
1377
-
goto done;
1344
+
}
1378
1345
1379
-
if (state_change)
1380
-
mvq->fw_state = state;
1346
+
static void modify_virtqueue_end(struct mlx5_vdpa_net *ndev,
1347
+
struct mlx5_vdpa_virtqueue *mvq,
1348
+
int state)
1349
+
{
1350
+
struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
1381
1351
1382
1352
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_VIRTIO_Q_MKEY) {
1353
+
unsigned int asid = mvdev->mres.group2asid[MLX5_VDPA_DATAVQ_GROUP];
1354
+
struct mlx5_vdpa_mr *vq_mr = mvdev->mres.mr[asid];
1355
+
1383
1356
mlx5_vdpa_put_mr(mvdev, mvq->vq_mr);
1384
1357
mlx5_vdpa_get_mr(mvdev, vq_mr);
1385
1358
mvq->vq_mr = vq_mr;
1386
1359
}
1387
1360
1388
1361
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_DESC_GROUP_MKEY) {
1362
+
unsigned int asid = mvdev->mres.group2asid[MLX5_VDPA_DATAVQ_DESC_GROUP];
1363
+
struct mlx5_vdpa_mr *desc_mr = mvdev->mres.mr[asid];
1364
+
1389
1365
mlx5_vdpa_put_mr(mvdev, mvq->desc_mr);
1390
1366
mlx5_vdpa_get_mr(mvdev, desc_mr);
1391
1367
mvq->desc_mr = desc_mr;
1392
1368
}
1393
1369
1370
+
if (mvq->modified_fields & MLX5_VIRTQ_MODIFY_MASK_STATE)
1371
+
mvq->fw_state = state;
1372
+
1394
1373
mvq->modified_fields = 0;
1395
-
1396
-
done:
1397
-
kfree(in);
1398
-
return err;
1399
-
}
1400
-
1401
-
static int modify_virtqueue_state(struct mlx5_vdpa_net *ndev,
1402
-
struct mlx5_vdpa_virtqueue *mvq,
1403
-
unsigned int state)
1404
-
{
1405
-
mvq->modified_fields |= MLX5_VIRTQ_MODIFY_MASK_STATE;
1406
-
return modify_virtqueue(ndev, mvq, state);
1407
1374
}
1408
1375
1409
1376
static int counter_set_alloc(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
···
1554
1525
return err;
1555
1526
}
1556
1527
1557
-
static int suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
1528
+
static int modify_virtqueues(struct mlx5_vdpa_net *ndev, int start_vq, int num_vqs, int state)
1558
1529
{
1559
-
struct mlx5_virtq_attr attr;
1530
+
struct mlx5_vdpa_dev *mvdev = &ndev->mvdev;
1531
+
struct mlx5_virtqueue_modify_mem *cmd_mem;
1532
+
struct mlx5_vdpa_async_cmd *cmds;
1533
+
int err = 0;
1534
+
1535
+
WARN(start_vq + num_vqs > mvdev->max_vqs, "modify vq range invalid [%d, %d), max_vqs: %u\n",
1536
+
start_vq, start_vq + num_vqs, mvdev->max_vqs);
1537
+
1538
+
cmds = kvcalloc(num_vqs, sizeof(*cmds), GFP_KERNEL);
1539
+
cmd_mem = kvcalloc(num_vqs, sizeof(*cmd_mem), GFP_KERNEL);
1540
+
if (!cmds || !cmd_mem) {
1541
+
err = -ENOMEM;
1542
+
goto done;
1543
+
}
1544
+
1545
+
for (int i = 0; i < num_vqs; i++) {
1546
+
struct mlx5_vdpa_async_cmd *cmd = &cmds[i];
1547
+
struct mlx5_vdpa_virtqueue *mvq;
1548
+
int vq_idx = start_vq + i;
1549
+
1550
+
mvq = &ndev->vqs[vq_idx];
1551
+
1552
+
if (!modifiable_virtqueue_fields(mvq)) {
1553
+
err = -EINVAL;
1554
+
goto done;
1555
+
}
1556
+
1557
+
if (mvq->fw_state != state) {
1558
+
if (!is_valid_state_change(mvq->fw_state, state, is_resumable(ndev))) {
1559
+
err = -EINVAL;
1560
+
goto done;
1561
+
}
1562
+
1563
+
mvq->modified_fields |= MLX5_VIRTQ_MODIFY_MASK_STATE;
1564
+
}
1565
+
1566
+
cmd->in = &cmd_mem[i].in;
1567
+
cmd->inlen = sizeof(cmd_mem[i].in);
1568
+
cmd->out = &cmd_mem[i].out;
1569
+
cmd->outlen = sizeof(cmd_mem[i].out);
1570
+
fill_modify_virtqueue_cmd(ndev, mvq, state, &cmd_mem[i]);
1571
+
}
1572
+
1573
+
err = mlx5_vdpa_exec_async_cmds(&ndev->mvdev, cmds, num_vqs);
1574
+
if (err) {
1575
+
mlx5_vdpa_err(mvdev, "error issuing modify cmd for vq range [%d, %d)\n",
1576
+
start_vq, start_vq + num_vqs);
1577
+
goto done;
1578
+
}
1579
+
1580
+
for (int i = 0; i < num_vqs; i++) {
1581
+
struct mlx5_vdpa_async_cmd *cmd = &cmds[i];
1582
+
struct mlx5_vdpa_virtqueue *mvq;
1583
+
int vq_idx = start_vq + i;
1584
+
1585
+
mvq = &ndev->vqs[vq_idx];
1586
+
1587
+
if (cmd->err) {
1588
+
mlx5_vdpa_err(mvdev, "modify vq %d failed, state: %d -> %d, err: %d\n",
1589
+
vq_idx, mvq->fw_state, state, err);
1590
+
if (!err)
1591
+
err = cmd->err;
1592
+
continue;
1593
+
}
1594
+
1595
+
modify_virtqueue_end(ndev, mvq, state);
1596
+
}
1597
+
1598
+
done:
1599
+
kvfree(cmd_mem);
1600
+
kvfree(cmds);
1601
+
return err;
1602
+
}
1603
+
1604
+
static int suspend_vqs(struct mlx5_vdpa_net *ndev, int start_vq, int num_vqs)
1605
+
{
1606
+
struct mlx5_vdpa_virtqueue *mvq;
1607
+
struct mlx5_virtq_attr *attrs;
1608
+
int vq_idx, i;
1560
1609
int err;
1561
1610
1611
+
if (start_vq >= ndev->cur_num_vqs)
1612
+
return -EINVAL;
1613
+
1614
+
mvq = &ndev->vqs[start_vq];
1562
1615
if (!mvq->initialized)
1563
1616
return 0;
1564
1617
1565
1618
if (mvq->fw_state != MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY)
1566
1619
return 0;
1567
1620
1568
-
err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND);
1569
-
if (err) {
1570
-
mlx5_vdpa_warn(&ndev->mvdev, "modify to suspend failed, err: %d\n", err);
1621
+
err = modify_virtqueues(ndev, start_vq, num_vqs, MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND);
1622
+
if (err)
1571
1623
return err;
1624
+
1625
+
attrs = kcalloc(num_vqs, sizeof(struct mlx5_virtq_attr), GFP_KERNEL);
1626
+
if (!attrs)
1627
+
return -ENOMEM;
1628
+
1629
+
err = query_virtqueues(ndev, start_vq, num_vqs, attrs);
1630
+
if (err)
1631
+
goto done;
1632
+
1633
+
for (i = 0, vq_idx = start_vq; i < num_vqs; i++, vq_idx++) {
1634
+
mvq = &ndev->vqs[vq_idx];
1635
+
mvq->avail_idx = attrs[i].available_index;
1636
+
mvq->used_idx = attrs[i].used_index;
1572
1637
}
1573
1638
1574
-
err = query_virtqueue(ndev, mvq, &attr);
1575
-
if (err) {
1576
-
mlx5_vdpa_warn(&ndev->mvdev, "failed to query virtqueue, err: %d\n", err);
1577
-
return err;
1578
-
}
1579
-
1580
-
mvq->avail_idx = attr.available_index;
1581
-
mvq->used_idx = attr.used_index;
1582
-
1583
-
return 0;
1584
-
}
1585
-
1586
-
static int suspend_vqs(struct mlx5_vdpa_net *ndev)
1587
-
{
1588
-
int err = 0;
1589
-
int i;
1590
-
1591
-
for (i = 0; i < ndev->cur_num_vqs; i++) {
1592
-
int local_err = suspend_vq(ndev, &ndev->vqs[i]);
1593
-
1594
-
err = local_err ? local_err : err;
1595
-
}
1596
-
1639
+
done:
1640
+
kfree(attrs);
1597
1641
return err;
1598
1642
}
1599
1643
1600
-
static int resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
1644
+
static int suspend_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
1601
1645
{
1646
+
return suspend_vqs(ndev, mvq->index, 1);
1647
+
}
1648
+
1649
+
static int resume_vqs(struct mlx5_vdpa_net *ndev, int start_vq, int num_vqs)
1650
+
{
1651
+
struct mlx5_vdpa_virtqueue *mvq;
1602
1652
int err;
1603
1653
1654
+
if (start_vq >= ndev->mvdev.max_vqs)
1655
+
return -EINVAL;
1656
+
1657
+
mvq = &ndev->vqs[start_vq];
1604
1658
if (!mvq->initialized)
1605
1659
return 0;
1606
1660
···
1695
1583
/* Due to a FW quirk we need to modify the VQ fields first then change state.
1696
1584
* This should be fixed soon. After that, a single command can be used.
1697
1585
*/
1698
-
err = modify_virtqueue(ndev, mvq, 0);
1699
-
if (err) {
1700
-
mlx5_vdpa_warn(&ndev->mvdev,
1701
-
"modify vq properties failed for vq %u, err: %d\n",
1702
-
mvq->index, err);
1586
+
err = modify_virtqueues(ndev, start_vq, num_vqs, mvq->fw_state);
1587
+
if (err)
1703
1588
return err;
1704
-
}
1705
1589
break;
1706
1590
case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND:
1707
1591
if (!is_resumable(ndev)) {
···
1708
1600
case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY:
1709
1601
return 0;
1710
1602
default:
1711
-
mlx5_vdpa_warn(&ndev->mvdev, "resume vq %u called from bad state %d\n",
1603
+
mlx5_vdpa_err(&ndev->mvdev, "resume vq %u called from bad state %d\n",
1712
1604
mvq->index, mvq->fw_state);
1713
1605
return -EINVAL;
1714
1606
}
1715
1607
1716
-
err = modify_virtqueue_state(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY);
1717
-
if (err)
1718
-
mlx5_vdpa_warn(&ndev->mvdev, "modify to resume failed for vq %u, err: %d\n",
1719
-
mvq->index, err);
1720
-
1721
-
return err;
1608
+
return modify_virtqueues(ndev, start_vq, num_vqs, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY);
1722
1609
}
1723
1610
1724
-
static int resume_vqs(struct mlx5_vdpa_net *ndev)
1611
+
static int resume_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
1725
1612
{
1726
-
int err = 0;
1727
-
1728
-
for (int i = 0; i < ndev->cur_num_vqs; i++) {
1729
-
int local_err = resume_vq(ndev, &ndev->vqs[i]);
1730
-
1731
-
err = local_err ? local_err : err;
1732
-
}
1733
-
1734
-
return err;
1613
+
return resume_vqs(ndev, mvq->index, 1);
1735
1614
}
1736
1615
1737
1616
static void teardown_vq(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq)
···
2097
2002
2098
2003
ns = mlx5_get_flow_namespace(ndev->mvdev.mdev, MLX5_FLOW_NAMESPACE_BYPASS);
2099
2004
if (!ns) {
2100
-
mlx5_vdpa_warn(&ndev->mvdev, "failed to get flow namespace\n");
2005
+
mlx5_vdpa_err(&ndev->mvdev, "failed to get flow namespace\n");
2101
2006
return -EOPNOTSUPP;
2102
2007
}
2103
2008
2104
2009
ndev->rxft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr);
2105
2010
if (IS_ERR(ndev->rxft)) {
2106
-
mlx5_vdpa_warn(&ndev->mvdev, "failed to create flow table\n");
2011
+
mlx5_vdpa_err(&ndev->mvdev, "failed to create flow table\n");
2107
2012
return PTR_ERR(ndev->rxft);
2108
2013
}
2109
2014
mlx5_vdpa_add_rx_flow_table(ndev);
···
2219
2124
static int change_num_qps(struct mlx5_vdpa_dev *mvdev, int newqps)
2220
2125
{
2221
2126
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
2222
-
int cur_qps = ndev->cur_num_vqs / 2;
2127
+
int cur_vqs = ndev->cur_num_vqs;
2128
+
int new_vqs = newqps * 2;
2223
2129
int err;
2224
2130
int i;
2225
2131
2226
-
if (cur_qps > newqps) {
2227
-
err = modify_rqt(ndev, 2 * newqps);
2132
+
if (cur_vqs > new_vqs) {
2133
+
err = modify_rqt(ndev, new_vqs);
2228
2134
if (err)
2229
2135
return err;
2230
2136
2231
-
for (i = ndev->cur_num_vqs - 1; i >= 2 * newqps; i--) {
2232
-
struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[i];
2233
-
2234
-
if (is_resumable(ndev))
2235
-
suspend_vq(ndev, mvq);
2236
-
else
2237
-
teardown_vq(ndev, mvq);
2137
+
if (is_resumable(ndev)) {
2138
+
suspend_vqs(ndev, new_vqs, cur_vqs - new_vqs);
2139
+
} else {
2140
+
for (i = new_vqs; i < cur_vqs; i++)
2141
+
teardown_vq(ndev, &ndev->vqs[i]);
2238
2142
}
2239
2143
2240
-
ndev->cur_num_vqs = 2 * newqps;
2144
+
ndev->cur_num_vqs = new_vqs;
2241
2145
} else {
2242
-
ndev->cur_num_vqs = 2 * newqps;
2243
-
for (i = cur_qps * 2; i < 2 * newqps; i++) {
2244
-
struct mlx5_vdpa_virtqueue *mvq = &ndev->vqs[i];
2146
+
ndev->cur_num_vqs = new_vqs;
2245
2147
2246
-
err = mvq->initialized ? resume_vq(ndev, mvq) : setup_vq(ndev, mvq, true);
2148
+
for (i = cur_vqs; i < new_vqs; i++) {
2149
+
err = setup_vq(ndev, &ndev->vqs[i], false);
2247
2150
if (err)
2248
2151
goto clean_added;
2249
2152
}
2250
-
err = modify_rqt(ndev, 2 * newqps);
2153
+
2154
+
err = resume_vqs(ndev, cur_vqs, new_vqs - cur_vqs);
2155
+
if (err)
2156
+
goto clean_added;
2157
+
2158
+
err = modify_rqt(ndev, new_vqs);
2251
2159
if (err)
2252
2160
goto clean_added;
2253
2161
}
2254
2162
return 0;
2255
2163
2256
2164
clean_added:
2257
-
for (--i; i >= 2 * cur_qps; --i)
2165
+
for (--i; i >= cur_vqs; --i)
2258
2166
teardown_vq(ndev, &ndev->vqs[i]);
2259
2167
2260
-
ndev->cur_num_vqs = 2 * cur_qps;
2168
+
ndev->cur_num_vqs = cur_vqs;
2261
2169
2262
2170
return err;
2263
2171
}
···
2626
2528
return 0;
2627
2529
}
2628
2530
2629
-
err = query_virtqueue(ndev, mvq, &attr);
2531
+
err = query_virtqueues(ndev, mvq->index, 1, &attr);
2630
2532
if (err) {
2631
-
mlx5_vdpa_warn(mvdev, "failed to query virtqueue\n");
2533
+
mlx5_vdpa_err(mvdev, "failed to query virtqueue\n");
2632
2534
return err;
2633
2535
}
2634
2536
state->split.avail_index = attr.used_index;
···
2853
2755
struct mlx5_eqe *eqe = param;
2854
2756
int ret = NOTIFY_DONE;
2855
2757
2758
+
if (ndev->mvdev.suspended)
2759
+
return NOTIFY_DONE;
2760
+
2856
2761
if (event == MLX5_EVENT_TYPE_PORT_CHANGE) {
2857
2762
switch (eqe->sub_type) {
2858
2763
case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
···
2980
2879
int err;
2981
2880
2982
2881
if (mvq->initialized) {
2983
-
err = query_virtqueue(ndev, mvq, &attr);
2882
+
err = query_virtqueues(ndev, mvq->index, 1, &attr);
2984
2883
if (err)
2985
2884
return err;
2986
2885
}
···
3049
2948
bool teardown = !is_resumable(ndev);
3050
2949
int err;
3051
2950
3052
-
suspend_vqs(ndev);
2951
+
suspend_vqs(ndev, 0, ndev->cur_num_vqs);
3053
2952
if (teardown) {
3054
2953
err = save_channels_info(ndev);
3055
2954
if (err)
···
3074
2973
return err;
3075
2974
}
3076
2975
3077
-
resume_vqs(ndev);
2976
+
resume_vqs(ndev, 0, ndev->cur_num_vqs);
3078
2977
3079
2978
return 0;
3080
2979
}
···
3198
3097
teardown_vq_resources(ndev);
3199
3098
3200
3099
if (ndev->setup) {
3201
-
err = resume_vqs(ndev);
3100
+
err = resume_vqs(ndev, 0, ndev->cur_num_vqs);
3202
3101
if (err) {
3203
3102
mlx5_vdpa_warn(mvdev, "failed to resume VQs\n");
3204
3103
goto err_driver;
···
3223
3122
err_driver:
3224
3123
unregister_link_notifier(ndev);
3225
3124
err_setup:
3226
-
mlx5_vdpa_destroy_mr_resources(&ndev->mvdev);
3125
+
mlx5_vdpa_clean_mrs(&ndev->mvdev);
3227
3126
ndev->mvdev.status |= VIRTIO_CONFIG_S_FAILED;
3228
3127
err_clear:
3229
3128
up_write(&ndev->reslock);
···
3235
3134
3236
3135
/* default mapping all groups are mapped to asid 0 */
3237
3136
for (i = 0; i < MLX5_VDPA_NUMVQ_GROUPS; i++)
3238
-
mvdev->group2asid[i] = 0;
3137
+
mvdev->mres.group2asid[i] = 0;
3239
3138
}
3240
3139
3241
3140
static bool needs_vqs_reset(const struct mlx5_vdpa_dev *mvdev)
···
3275
3174
}
3276
3175
3277
3176
if (flags & VDPA_RESET_F_CLEAN_MAP)
3278
-
mlx5_vdpa_destroy_mr_resources(&ndev->mvdev);
3177
+
mlx5_vdpa_clean_mrs(&ndev->mvdev);
3279
3178
ndev->mvdev.status = 0;
3280
3179
ndev->mvdev.suspended = false;
3281
3180
ndev->cur_num_vqs = MLX5V_DEFAULT_VQ_COUNT;
···
3290
3189
if ((flags & VDPA_RESET_F_CLEAN_MAP) &&
3291
3190
MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) {
3292
3191
if (mlx5_vdpa_create_dma_mr(mvdev))
3293
-
mlx5_vdpa_warn(mvdev, "create MR failed\n");
3192
+
mlx5_vdpa_err(mvdev, "create MR failed\n");
3294
3193
}
3295
3194
if (vq_reset)
3296
3195
setup_vq_resources(ndev, false);
···
3345
3244
new_mr = mlx5_vdpa_create_mr(mvdev, iotlb);
3346
3245
if (IS_ERR(new_mr)) {
3347
3246
err = PTR_ERR(new_mr);
3348
-
mlx5_vdpa_warn(mvdev, "create map failed(%d)\n", err);
3247
+
mlx5_vdpa_err(mvdev, "create map failed(%d)\n", err);
3349
3248
return err;
3350
3249
}
3351
3250
} else {
···
3353
3252
new_mr = NULL;
3354
3253
}
3355
3254
3356
-
if (!mvdev->mr[asid]) {
3255
+
if (!mvdev->mres.mr[asid]) {
3357
3256
mlx5_vdpa_update_mr(mvdev, new_mr, asid);
3358
3257
} else {
3359
3258
err = mlx5_vdpa_change_map(mvdev, new_mr, asid);
3360
3259
if (err) {
3361
-
mlx5_vdpa_warn(mvdev, "change map failed(%d)\n", err);
3260
+
mlx5_vdpa_err(mvdev, "change map failed(%d)\n", err);
3362
3261
goto out_err;
3363
3262
}
3364
3263
}
···
3433
3332
ndev = to_mlx5_vdpa_ndev(mvdev);
3434
3333
3435
3334
free_fixed_resources(ndev);
3436
-
mlx5_vdpa_destroy_mr_resources(mvdev);
3335
+
mlx5_vdpa_clean_mrs(mvdev);
3336
+
mlx5_vdpa_destroy_mr_resources(&ndev->mvdev);
3337
+
mlx5_cmd_cleanup_async_ctx(&mvdev->async_ctx);
3338
+
3437
3339
if (!is_zero_ether_addr(ndev->config.mac)) {
3438
3340
pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev));
3439
3341
mlx5_mpfs_del_mac(pfmdev, ndev->config.mac);
···
3604
3500
mlx5_vdpa_info(mvdev, "suspending device\n");
3605
3501
3606
3502
down_write(&ndev->reslock);
3607
-
unregister_link_notifier(ndev);
3608
-
err = suspend_vqs(ndev);
3503
+
err = suspend_vqs(ndev, 0, ndev->cur_num_vqs);
3609
3504
mlx5_vdpa_cvq_suspend(mvdev);
3610
3505
mvdev->suspended = true;
3611
3506
up_write(&ndev->reslock);
···
3624
3521
3625
3522
down_write(&ndev->reslock);
3626
3523
mvdev->suspended = false;
3627
-
err = resume_vqs(ndev);
3628
-
register_link_notifier(ndev);
3524
+
err = resume_vqs(ndev, 0, ndev->cur_num_vqs);
3525
+
queue_link_work(ndev);
3629
3526
up_write(&ndev->reslock);
3630
3527
3631
3528
return err;
···
3640
3537
if (group >= MLX5_VDPA_NUMVQ_GROUPS)
3641
3538
return -EINVAL;
3642
3539
3643
-
mvdev->group2asid[group] = asid;
3540
+
mvdev->mres.group2asid[group] = asid;
3644
3541
3645
-
mutex_lock(&mvdev->mr_mtx);
3646
-
if (group == MLX5_VDPA_CVQ_GROUP && mvdev->mr[asid])
3647
-
err = mlx5_vdpa_update_cvq_iotlb(mvdev, mvdev->mr[asid]->iotlb, asid);
3648
-
mutex_unlock(&mvdev->mr_mtx);
3542
+
mutex_lock(&mvdev->mres.lock);
3543
+
if (group == MLX5_VDPA_CVQ_GROUP && mvdev->mres.mr[asid])
3544
+
err = mlx5_vdpa_update_cvq_iotlb(mvdev, mvdev->mres.mr[asid]->iotlb, asid);
3545
+
mutex_unlock(&mvdev->mres.lock);
3649
3546
3650
3547
return err;
3651
3548
}
···
3957
3854
ndev->rqt_size = 1;
3958
3855
}
3959
3856
3857
+
mlx5_cmd_init_async_ctx(mdev, &mvdev->async_ctx);
3858
+
3960
3859
ndev->mvdev.mlx_features = device_features;
3961
3860
mvdev->vdev.dma_dev = &mdev->pdev->dev;
3962
3861
err = mlx5_vdpa_alloc_resources(&ndev->mvdev);
3963
3862
if (err)
3964
3863
goto err_mpfs;
3965
3864
3966
-
INIT_LIST_HEAD(&mvdev->mr_list_head);
3865
+
err = mlx5_vdpa_init_mr_resources(mvdev);
3866
+
if (err)
3867
+
goto err_res;
3967
3868
3968
3869
if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) {
3969
3870
err = mlx5_vdpa_create_dma_mr(mvdev);
3970
3871
if (err)
3971
-
goto err_res;
3872
+
goto err_mr_res;
3972
3873
}
3973
3874
3974
3875
err = alloc_fixed_resources(ndev);
···
4013
3906
err_res2:
4014
3907
free_fixed_resources(ndev);
4015
3908
err_mr:
3909
+
mlx5_vdpa_clean_mrs(mvdev);
3910
+
err_mr_res:
4016
3911
mlx5_vdpa_destroy_mr_resources(mvdev);
4017
3912
err_res:
4018
3913
mlx5_vdpa_free_resources(&ndev->mvdev);
···
4046
3937
mgtdev->ndev = NULL;
4047
3938
}
4048
3939
3940
+
static int mlx5_vdpa_set_attr(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device *dev,
3941
+
const struct vdpa_dev_set_config *add_config)
3942
+
{
3943
+
struct virtio_net_config *config;
3944
+
struct mlx5_core_dev *pfmdev;
3945
+
struct mlx5_vdpa_dev *mvdev;
3946
+
struct mlx5_vdpa_net *ndev;
3947
+
struct mlx5_core_dev *mdev;
3948
+
int err = -EOPNOTSUPP;
3949
+
3950
+
mvdev = to_mvdev(dev);
3951
+
ndev = to_mlx5_vdpa_ndev(mvdev);
3952
+
mdev = mvdev->mdev;
3953
+
config = &ndev->config;
3954
+
3955
+
down_write(&ndev->reslock);
3956
+
if (add_config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
3957
+
pfmdev = pci_get_drvdata(pci_physfn(mdev->pdev));
3958
+
err = mlx5_mpfs_add_mac(pfmdev, config->mac);
3959
+
if (!err)
3960
+
ether_addr_copy(config->mac, add_config->net.mac);
3961
+
}
3962
+
3963
+
up_write(&ndev->reslock);
3964
+
return err;
3965
+
}
3966
+
4049
3967
static const struct vdpa_mgmtdev_ops mdev_ops = {
4050
3968
.dev_add = mlx5_vdpa_dev_add,
4051
3969
.dev_del = mlx5_vdpa_dev_del,
3970
+
.dev_set_attr = mlx5_vdpa_set_attr,
4052
3971
};
4053
3972
4054
3973
static struct virtio_device_id id_table[] = {
-1
drivers/vdpa/pds/cmds.h
-1
drivers/vdpa/pds/cmds.h
+79
drivers/vdpa/vdpa.c
+79
drivers/vdpa/vdpa.c
···
1361
1361
return err;
1362
1362
}
1363
1363
1364
+
static int vdpa_dev_net_device_attr_set(struct vdpa_device *vdev,
1365
+
struct genl_info *info)
1366
+
{
1367
+
struct vdpa_dev_set_config set_config = {};
1368
+
struct vdpa_mgmt_dev *mdev = vdev->mdev;
1369
+
struct nlattr **nl_attrs = info->attrs;
1370
+
const u8 *macaddr;
1371
+
int err = -EOPNOTSUPP;
1372
+
1373
+
down_write(&vdev->cf_lock);
1374
+
if (nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR]) {
1375
+
set_config.mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR);
1376
+
macaddr = nla_data(nl_attrs[VDPA_ATTR_DEV_NET_CFG_MACADDR]);
1377
+
1378
+
if (is_valid_ether_addr(macaddr)) {
1379
+
ether_addr_copy(set_config.net.mac, macaddr);
1380
+
if (mdev->ops->dev_set_attr) {
1381
+
err = mdev->ops->dev_set_attr(mdev, vdev,
1382
+
&set_config);
1383
+
} else {
1384
+
NL_SET_ERR_MSG_FMT_MOD(info->extack,
1385
+
"Operation not supported by the device.");
1386
+
}
1387
+
} else {
1388
+
NL_SET_ERR_MSG_FMT_MOD(info->extack,
1389
+
"Invalid MAC address");
1390
+
}
1391
+
}
1392
+
up_write(&vdev->cf_lock);
1393
+
return err;
1394
+
}
1395
+
1396
+
static int vdpa_nl_cmd_dev_attr_set_doit(struct sk_buff *skb,
1397
+
struct genl_info *info)
1398
+
{
1399
+
struct vdpa_device *vdev;
1400
+
struct device *dev;
1401
+
const char *name;
1402
+
u64 classes;
1403
+
int err = 0;
1404
+
1405
+
if (!info->attrs[VDPA_ATTR_DEV_NAME])
1406
+
return -EINVAL;
1407
+
1408
+
name = nla_data(info->attrs[VDPA_ATTR_DEV_NAME]);
1409
+
1410
+
down_write(&vdpa_dev_lock);
1411
+
dev = bus_find_device(&vdpa_bus, NULL, name, vdpa_name_match);
1412
+
if (!dev) {
1413
+
NL_SET_ERR_MSG_MOD(info->extack, "device not found");
1414
+
err = -ENODEV;
1415
+
goto dev_err;
1416
+
}
1417
+
vdev = container_of(dev, struct vdpa_device, dev);
1418
+
if (!vdev->mdev) {
1419
+
NL_SET_ERR_MSG_MOD(info->extack, "unmanaged vdpa device");
1420
+
err = -EINVAL;
1421
+
goto mdev_err;
1422
+
}
1423
+
classes = vdpa_mgmtdev_get_classes(vdev->mdev, NULL);
1424
+
if (classes & BIT_ULL(VIRTIO_ID_NET)) {
1425
+
err = vdpa_dev_net_device_attr_set(vdev, info);
1426
+
} else {
1427
+
NL_SET_ERR_MSG_FMT_MOD(info->extack, "%s device not supported",
1428
+
name);
1429
+
}
1430
+
1431
+
mdev_err:
1432
+
put_device(dev);
1433
+
dev_err:
1434
+
up_write(&vdpa_dev_lock);
1435
+
return err;
1436
+
}
1437
+
1364
1438
static int vdpa_dev_config_dump(struct device *dev, void *data)
1365
1439
{
1366
1440
struct vdpa_device *vdev = container_of(dev, struct vdpa_device, dev);
···
1569
1495
{
1570
1496
.cmd = VDPA_CMD_DEV_VSTATS_GET,
1571
1497
.doit = vdpa_nl_cmd_dev_stats_get_doit,
1498
+
.flags = GENL_ADMIN_PERM,
1499
+
},
1500
+
{
1501
+
.cmd = VDPA_CMD_DEV_ATTR_SET,
1502
+
.doit = vdpa_nl_cmd_dev_attr_set_doit,
1572
1503
.flags = GENL_ADMIN_PERM,
1573
1504
},
1574
1505
};
+20
-1
drivers/vdpa/vdpa_sim/vdpa_sim_net.c
+20
-1
drivers/vdpa/vdpa_sim/vdpa_sim_net.c
···
414
414
net_config->status = cpu_to_vdpasim16(vdpasim, VIRTIO_NET_S_LINK_UP);
415
415
}
416
416
417
+
static int vdpasim_net_set_attr(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev,
418
+
const struct vdpa_dev_set_config *config)
419
+
{
420
+
struct vdpasim *vdpasim = container_of(dev, struct vdpasim, vdpa);
421
+
struct virtio_net_config *vio_config = vdpasim->config;
422
+
423
+
mutex_lock(&vdpasim->mutex);
424
+
425
+
if (config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
426
+
ether_addr_copy(vio_config->mac, config->net.mac);
427
+
mutex_unlock(&vdpasim->mutex);
428
+
return 0;
429
+
}
430
+
431
+
mutex_unlock(&vdpasim->mutex);
432
+
return -EOPNOTSUPP;
433
+
}
434
+
417
435
static void vdpasim_net_setup_config(struct vdpasim *vdpasim,
418
436
const struct vdpa_dev_set_config *config)
419
437
{
···
528
510
529
511
static const struct vdpa_mgmtdev_ops vdpasim_net_mgmtdev_ops = {
530
512
.dev_add = vdpasim_net_dev_add,
531
-
.dev_del = vdpasim_net_dev_del
513
+
.dev_del = vdpasim_net_dev_del,
514
+
.dev_set_attr = vdpasim_net_set_attr
532
515
};
533
516
534
517
static struct virtio_device_id id_table[] = {
+13
-3
drivers/vhost/vdpa.c
+13
-3
drivers/vhost/vdpa.c
···
209
209
if (irq < 0)
210
210
return;
211
211
212
-
irq_bypass_unregister_producer(&vq->call_ctx.producer);
213
212
if (!vq->call_ctx.ctx)
214
213
return;
215
214
216
-
vq->call_ctx.producer.token = vq->call_ctx.ctx;
217
215
vq->call_ctx.producer.irq = irq;
218
216
ret = irq_bypass_register_producer(&vq->call_ctx.producer);
219
217
if (unlikely(ret))
···
707
709
vq->last_avail_idx = vq_state.split.avail_index;
708
710
}
709
711
break;
712
+
case VHOST_SET_VRING_CALL:
713
+
if (vq->call_ctx.ctx) {
714
+
if (ops->get_status(vdpa) &
715
+
VIRTIO_CONFIG_S_DRIVER_OK)
716
+
vhost_vdpa_unsetup_vq_irq(v, idx);
717
+
vq->call_ctx.producer.token = NULL;
718
+
}
719
+
break;
710
720
}
711
721
712
722
r = vhost_vring_ioctl(&v->vdev, cmd, argp);
···
753
747
cb.callback = vhost_vdpa_virtqueue_cb;
754
748
cb.private = vq;
755
749
cb.trigger = vq->call_ctx.ctx;
750
+
vq->call_ctx.producer.token = vq->call_ctx.ctx;
751
+
if (ops->get_status(vdpa) &
752
+
VIRTIO_CONFIG_S_DRIVER_OK)
753
+
vhost_vdpa_setup_vq_irq(v, idx);
756
754
} else {
757
755
cb.callback = NULL;
758
756
cb.private = NULL;
759
757
cb.trigger = NULL;
760
758
}
761
759
ops->set_vq_cb(vdpa, idx, &cb);
762
-
vhost_vdpa_setup_vq_irq(v, idx);
763
760
break;
764
761
765
762
case VHOST_SET_VRING_NUM:
···
1428
1419
for (i = 0; i < nvqs; i++) {
1429
1420
vqs[i] = &v->vqs[i];
1430
1421
vqs[i]->handle_kick = handle_vq_kick;
1422
+
vqs[i]->call_ctx.ctx = NULL;
1431
1423
}
1432
1424
vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,
1433
1425
vhost_vdpa_process_iotlb_msg);
+18
drivers/virtio/virtio_balloon.c
+18
drivers/virtio/virtio_balloon.c
···
355
355
{
356
356
unsigned long events[NR_VM_EVENT_ITEMS];
357
357
unsigned int idx = 0;
358
+
unsigned int zid;
359
+
unsigned long stall = 0;
358
360
359
361
all_vm_events(events);
360
362
update_stat(vb, idx++, VIRTIO_BALLOON_S_SWAP_IN,
···
365
363
pages_to_bytes(events[PSWPOUT]));
366
364
update_stat(vb, idx++, VIRTIO_BALLOON_S_MAJFLT, events[PGMAJFAULT]);
367
365
update_stat(vb, idx++, VIRTIO_BALLOON_S_MINFLT, events[PGFAULT]);
366
+
update_stat(vb, idx++, VIRTIO_BALLOON_S_OOM_KILL, events[OOM_KILL]);
367
+
368
+
/* sum all the stall events */
369
+
for (zid = 0; zid < MAX_NR_ZONES; zid++)
370
+
stall += events[ALLOCSTALL_NORMAL - ZONE_NORMAL + zid];
371
+
372
+
update_stat(vb, idx++, VIRTIO_BALLOON_S_ALLOC_STALL, stall);
373
+
374
+
update_stat(vb, idx++, VIRTIO_BALLOON_S_ASYNC_SCAN,
375
+
pages_to_bytes(events[PGSCAN_KSWAPD]));
376
+
update_stat(vb, idx++, VIRTIO_BALLOON_S_DIRECT_SCAN,
377
+
pages_to_bytes(events[PGSCAN_DIRECT]));
378
+
update_stat(vb, idx++, VIRTIO_BALLOON_S_ASYNC_RECLAIM,
379
+
pages_to_bytes(events[PGSTEAL_KSWAPD]));
380
+
update_stat(vb, idx++, VIRTIO_BALLOON_S_DIRECT_RECLAIM,
381
+
pages_to_bytes(events[PGSTEAL_DIRECT]));
368
382
369
383
#ifdef CONFIG_HUGETLB_PAGE
370
384
update_stat(vb, idx++, VIRTIO_BALLOON_S_HTLB_PGALLOC,
+150
-14
fs/fuse/virtio_fs.c
+150
-14
fs/fuse/virtio_fs.c
···
56
56
bool connected;
57
57
long in_flight;
58
58
struct completion in_flight_zero; /* No inflight requests */
59
+
struct kobject *kobj;
59
60
char name[VQ_NAME_LEN];
60
61
} ____cacheline_aligned_in_smp;
61
62
62
63
/* A virtio-fs device instance */
63
64
struct virtio_fs {
64
65
struct kobject kobj;
66
+
struct kobject *mqs_kobj;
65
67
struct list_head list; /* on virtio_fs_instances */
66
68
char *tag;
67
69
struct virtio_fs_vq *vqs;
···
202
200
.default_groups = virtio_fs_groups,
203
201
};
204
202
203
+
static struct virtio_fs_vq *virtio_fs_kobj_to_vq(struct virtio_fs *fs,
204
+
struct kobject *kobj)
205
+
{
206
+
int i;
207
+
208
+
for (i = 0; i < fs->nvqs; i++) {
209
+
if (kobj == fs->vqs[i].kobj)
210
+
return &fs->vqs[i];
211
+
}
212
+
return NULL;
213
+
}
214
+
215
+
static ssize_t name_show(struct kobject *kobj,
216
+
struct kobj_attribute *attr, char *buf)
217
+
{
218
+
struct virtio_fs *fs = container_of(kobj->parent->parent, struct virtio_fs, kobj);
219
+
struct virtio_fs_vq *fsvq = virtio_fs_kobj_to_vq(fs, kobj);
220
+
221
+
if (!fsvq)
222
+
return -EINVAL;
223
+
return sysfs_emit(buf, "%s\n", fsvq->name);
224
+
}
225
+
226
+
static struct kobj_attribute virtio_fs_vq_name_attr = __ATTR_RO(name);
227
+
228
+
static ssize_t cpu_list_show(struct kobject *kobj,
229
+
struct kobj_attribute *attr, char *buf)
230
+
{
231
+
struct virtio_fs *fs = container_of(kobj->parent->parent, struct virtio_fs, kobj);
232
+
struct virtio_fs_vq *fsvq = virtio_fs_kobj_to_vq(fs, kobj);
233
+
unsigned int cpu, qid;
234
+
const size_t size = PAGE_SIZE - 1;
235
+
bool first = true;
236
+
int ret = 0, pos = 0;
237
+
238
+
if (!fsvq)
239
+
return -EINVAL;
240
+
241
+
qid = fsvq->vq->index;
242
+
for (cpu = 0; cpu < nr_cpu_ids; cpu++) {
243
+
if (qid < VQ_REQUEST || (fs->mq_map[cpu] == qid - VQ_REQUEST)) {
244
+
if (first)
245
+
ret = snprintf(buf + pos, size - pos, "%u", cpu);
246
+
else
247
+
ret = snprintf(buf + pos, size - pos, ", %u", cpu);
248
+
249
+
if (ret >= size - pos)
250
+
break;
251
+
first = false;
252
+
pos += ret;
253
+
}
254
+
}
255
+
ret = snprintf(buf + pos, size + 1 - pos, "\n");
256
+
return pos + ret;
257
+
}
258
+
259
+
static struct kobj_attribute virtio_fs_vq_cpu_list_attr = __ATTR_RO(cpu_list);
260
+
261
+
static struct attribute *virtio_fs_vq_attrs[] = {
262
+
&virtio_fs_vq_name_attr.attr,
263
+
&virtio_fs_vq_cpu_list_attr.attr,
264
+
NULL
265
+
};
266
+
267
+
static struct attribute_group virtio_fs_vq_attr_group = {
268
+
.attrs = virtio_fs_vq_attrs,
269
+
};
270
+
205
271
/* Make sure virtiofs_mutex is held */
272
+
static void virtio_fs_put_locked(struct virtio_fs *fs)
273
+
{
274
+
lockdep_assert_held(&virtio_fs_mutex);
275
+
276
+
kobject_put(&fs->kobj);
277
+
}
278
+
206
279
static void virtio_fs_put(struct virtio_fs *fs)
207
280
{
208
-
kobject_put(&fs->kobj);
281
+
mutex_lock(&virtio_fs_mutex);
282
+
virtio_fs_put_locked(fs);
283
+
mutex_unlock(&virtio_fs_mutex);
209
284
}
210
285
211
286
static void virtio_fs_fiq_release(struct fuse_iqueue *fiq)
212
287
{
213
288
struct virtio_fs *vfs = fiq->priv;
214
289
215
-
mutex_lock(&virtio_fs_mutex);
216
290
virtio_fs_put(vfs);
217
-
mutex_unlock(&virtio_fs_mutex);
218
291
}
219
292
220
293
static void virtio_fs_drain_queue(struct virtio_fs_vq *fsvq)
···
350
273
}
351
274
}
352
275
276
+
static void virtio_fs_delete_queues_sysfs(struct virtio_fs *fs)
277
+
{
278
+
struct virtio_fs_vq *fsvq;
279
+
int i;
280
+
281
+
for (i = 0; i < fs->nvqs; i++) {
282
+
fsvq = &fs->vqs[i];
283
+
kobject_put(fsvq->kobj);
284
+
}
285
+
}
286
+
287
+
static int virtio_fs_add_queues_sysfs(struct virtio_fs *fs)
288
+
{
289
+
struct virtio_fs_vq *fsvq;
290
+
char buff[12];
291
+
int i, j, ret;
292
+
293
+
for (i = 0; i < fs->nvqs; i++) {
294
+
fsvq = &fs->vqs[i];
295
+
296
+
sprintf(buff, "%d", i);
297
+
fsvq->kobj = kobject_create_and_add(buff, fs->mqs_kobj);
298
+
if (!fs->mqs_kobj) {
299
+
ret = -ENOMEM;
300
+
goto out_del;
301
+
}
302
+
303
+
ret = sysfs_create_group(fsvq->kobj, &virtio_fs_vq_attr_group);
304
+
if (ret) {
305
+
kobject_put(fsvq->kobj);
306
+
goto out_del;
307
+
}
308
+
}
309
+
310
+
return 0;
311
+
312
+
out_del:
313
+
for (j = 0; j < i; j++) {
314
+
fsvq = &fs->vqs[j];
315
+
kobject_put(fsvq->kobj);
316
+
}
317
+
return ret;
318
+
}
319
+
353
320
/* Add a new instance to the list or return -EEXIST if tag name exists*/
354
321
static int virtio_fs_add_instance(struct virtio_device *vdev,
355
322
struct virtio_fs *fs)
···
417
296
*/
418
297
fs->kobj.kset = virtio_fs_kset;
419
298
ret = kobject_add(&fs->kobj, NULL, "%d", vdev->index);
420
-
if (ret < 0) {
421
-
mutex_unlock(&virtio_fs_mutex);
422
-
return ret;
299
+
if (ret < 0)
300
+
goto out_unlock;
301
+
302
+
fs->mqs_kobj = kobject_create_and_add("mqs", &fs->kobj);
303
+
if (!fs->mqs_kobj) {
304
+
ret = -ENOMEM;
305
+
goto out_del;
423
306
}
424
307
425
308
ret = sysfs_create_link(&fs->kobj, &vdev->dev.kobj, "device");
426
-
if (ret < 0) {
427
-
kobject_del(&fs->kobj);
428
-
mutex_unlock(&virtio_fs_mutex);
429
-
return ret;
430
-
}
309
+
if (ret < 0)
310
+
goto out_put;
311
+
312
+
ret = virtio_fs_add_queues_sysfs(fs);
313
+
if (ret)
314
+
goto out_remove;
431
315
432
316
list_add_tail(&fs->list, &virtio_fs_instances);
433
317
···
441
315
kobject_uevent(&fs->kobj, KOBJ_ADD);
442
316
443
317
return 0;
318
+
319
+
out_remove:
320
+
sysfs_remove_link(&fs->kobj, "device");
321
+
out_put:
322
+
kobject_put(fs->mqs_kobj);
323
+
out_del:
324
+
kobject_del(&fs->kobj);
325
+
out_unlock:
326
+
mutex_unlock(&virtio_fs_mutex);
327
+
return ret;
444
328
}
445
329
446
330
/* Return the virtio_fs with a given tag, or NULL */
···
1179
1043
mutex_lock(&virtio_fs_mutex);
1180
1044
/* This device is going away. No one should get new reference */
1181
1045
list_del_init(&fs->list);
1046
+
virtio_fs_delete_queues_sysfs(fs);
1182
1047
sysfs_remove_link(&fs->kobj, "device");
1048
+
kobject_put(fs->mqs_kobj);
1183
1049
kobject_del(&fs->kobj);
1184
1050
virtio_fs_stop_all_queues(fs);
1185
1051
virtio_fs_drain_all_queues_locked(fs);
···
1190
1052
1191
1053
vdev->priv = NULL;
1192
1054
/* Put device reference on virtio_fs object */
1193
-
virtio_fs_put(fs);
1055
+
virtio_fs_put_locked(fs);
1194
1056
mutex_unlock(&virtio_fs_mutex);
1195
1057
}
1196
1058
···
1719
1581
1720
1582
out_err:
1721
1583
kfree(fc);
1722
-
mutex_lock(&virtio_fs_mutex);
1723
1584
virtio_fs_put(fs);
1724
-
mutex_unlock(&virtio_fs_mutex);
1725
1585
return err;
1726
1586
}
1727
1587
+9
include/linux/vdpa.h
+9
include/linux/vdpa.h
···
582
582
* @dev: vdpa device to remove
583
583
* Driver need to remove the specified device by calling
584
584
* _vdpa_unregister_device().
585
+
* @dev_set_attr: change a vdpa device's attr after it was create
586
+
* @mdev: parent device to use for device
587
+
* @dev: vdpa device structure
588
+
* @config:Attributes to be set for the device.
589
+
* The driver needs to check the mask of the structure and then set
590
+
* the related information to the vdpa device. The driver must return 0
591
+
* if set successfully.
585
592
*/
586
593
struct vdpa_mgmtdev_ops {
587
594
int (*dev_add)(struct vdpa_mgmt_dev *mdev, const char *name,
588
595
const struct vdpa_dev_set_config *config);
589
596
void (*dev_del)(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev);
597
+
int (*dev_set_attr)(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev,
598
+
const struct vdpa_dev_set_config *config);
590
599
};
591
600
592
601
/**
+1
include/uapi/linux/vdpa.h
+1
include/uapi/linux/vdpa.h
+14
-2
include/uapi/linux/virtio_balloon.h
+14
-2
include/uapi/linux/virtio_balloon.h
···
71
71
#define VIRTIO_BALLOON_S_CACHES 7 /* Disk caches */
72
72
#define VIRTIO_BALLOON_S_HTLB_PGALLOC 8 /* Hugetlb page allocations */
73
73
#define VIRTIO_BALLOON_S_HTLB_PGFAIL 9 /* Hugetlb page allocation failures */
74
-
#define VIRTIO_BALLOON_S_NR 10
74
+
#define VIRTIO_BALLOON_S_OOM_KILL 10 /* OOM killer invocations */
75
+
#define VIRTIO_BALLOON_S_ALLOC_STALL 11 /* Stall count of memory allocatoin */
76
+
#define VIRTIO_BALLOON_S_ASYNC_SCAN 12 /* Amount of memory scanned asynchronously */
77
+
#define VIRTIO_BALLOON_S_DIRECT_SCAN 13 /* Amount of memory scanned directly */
78
+
#define VIRTIO_BALLOON_S_ASYNC_RECLAIM 14 /* Amount of memory reclaimed asynchronously */
79
+
#define VIRTIO_BALLOON_S_DIRECT_RECLAIM 15 /* Amount of memory reclaimed directly */
80
+
#define VIRTIO_BALLOON_S_NR 16
75
81
76
82
#define VIRTIO_BALLOON_S_NAMES_WITH_PREFIX(VIRTIO_BALLOON_S_NAMES_prefix) { \
77
83
VIRTIO_BALLOON_S_NAMES_prefix "swap-in", \
···
89
83
VIRTIO_BALLOON_S_NAMES_prefix "available-memory", \
90
84
VIRTIO_BALLOON_S_NAMES_prefix "disk-caches", \
91
85
VIRTIO_BALLOON_S_NAMES_prefix "hugetlb-allocations", \
92
-
VIRTIO_BALLOON_S_NAMES_prefix "hugetlb-failures" \
86
+
VIRTIO_BALLOON_S_NAMES_prefix "hugetlb-failures", \
87
+
VIRTIO_BALLOON_S_NAMES_prefix "oom-kills", \
88
+
VIRTIO_BALLOON_S_NAMES_prefix "alloc-stalls", \
89
+
VIRTIO_BALLOON_S_NAMES_prefix "async-scans", \
90
+
VIRTIO_BALLOON_S_NAMES_prefix "direct-scans", \
91
+
VIRTIO_BALLOON_S_NAMES_prefix "async-reclaims", \
92
+
VIRTIO_BALLOON_S_NAMES_prefix "direct-reclaims" \
93
93
}
94
94
95
95
#define VIRTIO_BALLOON_S_NAMES VIRTIO_BALLOON_S_NAMES_WITH_PREFIX("")
+94
-50
net/vmw_vsock/virtio_transport.c
+94
-50
net/vmw_vsock/virtio_transport.c
···
94
94
return ret;
95
95
}
96
96
97
+
/* Caller need to hold vsock->tx_lock on vq */
98
+
static int virtio_transport_send_skb(struct sk_buff *skb, struct virtqueue *vq,
99
+
struct virtio_vsock *vsock)
100
+
{
101
+
int ret, in_sg = 0, out_sg = 0;
102
+
struct scatterlist **sgs;
103
+
104
+
sgs = vsock->out_sgs;
105
+
sg_init_one(sgs[out_sg], virtio_vsock_hdr(skb),
106
+
sizeof(*virtio_vsock_hdr(skb)));
107
+
out_sg++;
108
+
109
+
if (!skb_is_nonlinear(skb)) {
110
+
if (skb->len > 0) {
111
+
sg_init_one(sgs[out_sg], skb->data, skb->len);
112
+
out_sg++;
113
+
}
114
+
} else {
115
+
struct skb_shared_info *si;
116
+
int i;
117
+
118
+
/* If skb is nonlinear, then its buffer must contain
119
+
* only header and nothing more. Data is stored in
120
+
* the fragged part.
121
+
*/
122
+
WARN_ON_ONCE(skb_headroom(skb) != sizeof(*virtio_vsock_hdr(skb)));
123
+
124
+
si = skb_shinfo(skb);
125
+
126
+
for (i = 0; i < si->nr_frags; i++) {
127
+
skb_frag_t *skb_frag = &si->frags[i];
128
+
void *va;
129
+
130
+
/* We will use 'page_to_virt()' for the userspace page
131
+
* here, because virtio or dma-mapping layers will call
132
+
* 'virt_to_phys()' later to fill the buffer descriptor.
133
+
* We don't touch memory at "virtual" address of this page.
134
+
*/
135
+
va = page_to_virt(skb_frag_page(skb_frag));
136
+
sg_init_one(sgs[out_sg],
137
+
va + skb_frag_off(skb_frag),
138
+
skb_frag_size(skb_frag));
139
+
out_sg++;
140
+
}
141
+
}
142
+
143
+
ret = virtqueue_add_sgs(vq, sgs, out_sg, in_sg, skb, GFP_KERNEL);
144
+
/* Usually this means that there is no more space available in
145
+
* the vq
146
+
*/
147
+
if (ret < 0)
148
+
return ret;
149
+
150
+
virtio_transport_deliver_tap_pkt(skb);
151
+
return 0;
152
+
}
153
+
97
154
static void
98
155
virtio_transport_send_pkt_work(struct work_struct *work)
99
156
{
···
168
111
vq = vsock->vqs[VSOCK_VQ_TX];
169
112
170
113
for (;;) {
171
-
int ret, in_sg = 0, out_sg = 0;
172
-
struct scatterlist **sgs;
173
114
struct sk_buff *skb;
174
115
bool reply;
116
+
int ret;
175
117
176
118
skb = virtio_vsock_skb_dequeue(&vsock->send_pkt_queue);
177
119
if (!skb)
178
120
break;
179
121
180
122
reply = virtio_vsock_skb_reply(skb);
181
-
sgs = vsock->out_sgs;
182
-
sg_init_one(sgs[out_sg], virtio_vsock_hdr(skb),
183
-
sizeof(*virtio_vsock_hdr(skb)));
184
-
out_sg++;
185
123
186
-
if (!skb_is_nonlinear(skb)) {
187
-
if (skb->len > 0) {
188
-
sg_init_one(sgs[out_sg], skb->data, skb->len);
189
-
out_sg++;
190
-
}
191
-
} else {
192
-
struct skb_shared_info *si;
193
-
int i;
194
-
195
-
/* If skb is nonlinear, then its buffer must contain
196
-
* only header and nothing more. Data is stored in
197
-
* the fragged part.
198
-
*/
199
-
WARN_ON_ONCE(skb_headroom(skb) != sizeof(*virtio_vsock_hdr(skb)));
200
-
201
-
si = skb_shinfo(skb);
202
-
203
-
for (i = 0; i < si->nr_frags; i++) {
204
-
skb_frag_t *skb_frag = &si->frags[i];
205
-
void *va;
206
-
207
-
/* We will use 'page_to_virt()' for the userspace page
208
-
* here, because virtio or dma-mapping layers will call
209
-
* 'virt_to_phys()' later to fill the buffer descriptor.
210
-
* We don't touch memory at "virtual" address of this page.
211
-
*/
212
-
va = page_to_virt(skb_frag_page(skb_frag));
213
-
sg_init_one(sgs[out_sg],
214
-
va + skb_frag_off(skb_frag),
215
-
skb_frag_size(skb_frag));
216
-
out_sg++;
217
-
}
218
-
}
219
-
220
-
ret = virtqueue_add_sgs(vq, sgs, out_sg, in_sg, skb, GFP_KERNEL);
221
-
/* Usually this means that there is no more space available in
222
-
* the vq
223
-
*/
124
+
ret = virtio_transport_send_skb(skb, vq, vsock);
224
125
if (ret < 0) {
225
126
virtio_vsock_skb_queue_head(&vsock->send_pkt_queue, skb);
226
127
break;
227
128
}
228
-
229
-
virtio_transport_deliver_tap_pkt(skb);
230
129
231
130
if (reply) {
232
131
struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];
···
208
195
queue_work(virtio_vsock_workqueue, &vsock->rx_work);
209
196
}
210
197
198
+
/* Caller need to hold RCU for vsock.
199
+
* Returns 0 if the packet is successfully put on the vq.
200
+
*/
201
+
static int virtio_transport_send_skb_fast_path(struct virtio_vsock *vsock, struct sk_buff *skb)
202
+
{
203
+
struct virtqueue *vq = vsock->vqs[VSOCK_VQ_TX];
204
+
int ret;
205
+
206
+
/* Inside RCU, can't sleep! */
207
+
ret = mutex_trylock(&vsock->tx_lock);
208
+
if (unlikely(ret == 0))
209
+
return -EBUSY;
210
+
211
+
ret = virtio_transport_send_skb(skb, vq, vsock);
212
+
if (ret == 0)
213
+
virtqueue_kick(vq);
214
+
215
+
mutex_unlock(&vsock->tx_lock);
216
+
217
+
return ret;
218
+
}
219
+
211
220
static int
212
221
virtio_transport_send_pkt(struct sk_buff *skb)
213
222
{
···
253
218
goto out_rcu;
254
219
}
255
220
256
-
if (virtio_vsock_skb_reply(skb))
257
-
atomic_inc(&vsock->queued_replies);
221
+
/* If send_pkt_queue is empty, we can safely bypass this queue
222
+
* because packet order is maintained and (try) to put the packet
223
+
* on the virtqueue using virtio_transport_send_skb_fast_path.
224
+
* If this fails we simply put the packet on the intermediate
225
+
* queue and schedule the worker.
226
+
*/
227
+
if (!skb_queue_empty_lockless(&vsock->send_pkt_queue) ||
228
+
virtio_transport_send_skb_fast_path(vsock, skb)) {
229
+
if (virtio_vsock_skb_reply(skb))
230
+
atomic_inc(&vsock->queued_replies);
258
231
259
-
virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb);
260
-
queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);
232
+
virtio_vsock_skb_queue_tail(&vsock->send_pkt_queue, skb);
233
+
queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);
234
+
}
261
235
262
236
out_rcu:
263
237
rcu_read_unlock();
+1
-1
tools/virtio/ringtest/main.c
+1
-1
tools/virtio/ringtest/main.c