Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost
Pull virtio updates from Michael Tsirkin:
- vdpa generic device type support
- more virtio hardening for broken devices (but on the same theme,
revert some virtio hotplug hardening patches - they were misusing
some interrupt flags and had to be reverted)
- RSS support in virtio-net
- max device MTU support in mlx5 vdpa
- akcipher support in virtio-crypto
- shared IRQ support in ifcvf vdpa
- a minor performance improvement in vhost
- enable virtio mem for ARM64
- beginnings of advance dma support
- cleanups, fixes all over the place
* tag 'for_linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost: (33 commits)
vdpa/mlx5: Avoid processing works if workqueue was destroyed
vhost: handle error while adding split ranges to iotlb
vdpa: support exposing the count of vqs to userspace
vdpa: change the type of nvqs to u32
vdpa: support exposing the config size to userspace
vdpa/mlx5: re-create forwarding rules after mac modified
virtio: pci: check bar values read from virtio config space
Revert "virtio_pci: harden MSI-X interrupts"
Revert "virtio-pci: harden INTX interrupts"
drivers/net/virtio_net: Added RSS hash report control.
drivers/net/virtio_net: Added RSS hash report.
drivers/net/virtio_net: Added basic RSS support.
drivers/net/virtio_net: Fixed padded vheader to use v1 with hash.
virtio: use virtio_device_ready() in virtio_device_restore()
tools/virtio: compile with -pthread
tools/virtio: fix after premapped buf support
virtio_ring: remove flags check for unmap packed indirect desc
virtio_ring: remove flags check for unmap split indirect desc
virtio_ring: rename vring_unmap_state_packed() to vring_unmap_extra_packed()
net/mlx5: Add support for configuring max device MTU
...
+1639
-297
+3
drivers/crypto/virtio/Kconfig
+3
drivers/crypto/virtio/Kconfig
···
3
3
tristate "VirtIO crypto driver"
4
4
depends on VIRTIO
5
5
select CRYPTO_AEAD
6
+
select CRYPTO_AKCIPHER2
6
7
select CRYPTO_SKCIPHER
7
8
select CRYPTO_ENGINE
9
+
select CRYPTO_RSA
10
+
select MPILIB
8
11
help
9
12
This driver provides support for virtio crypto device. If you
10
13
choose 'M' here, this module will be called virtio_crypto.
+2
-1
drivers/crypto/virtio/Makefile
+2
-1
drivers/crypto/virtio/Makefile
+585
drivers/crypto/virtio/virtio_crypto_akcipher_algs.c
+585
drivers/crypto/virtio/virtio_crypto_akcipher_algs.c
···
1
+
// SPDX-License-Identifier: GPL-2.0-or-later
2
+
/* Asymmetric algorithms supported by virtio crypto device
3
+
*
4
+
* Authors: zhenwei pi <pizhenwei@bytedance.com>
5
+
* lei he <helei.sig11@bytedance.com>
6
+
*
7
+
* Copyright 2022 Bytedance CO., LTD.
8
+
*/
9
+
10
+
#include <linux/mpi.h>
11
+
#include <linux/scatterlist.h>
12
+
#include <crypto/algapi.h>
13
+
#include <crypto/internal/akcipher.h>
14
+
#include <crypto/internal/rsa.h>
15
+
#include <linux/err.h>
16
+
#include <crypto/scatterwalk.h>
17
+
#include <linux/atomic.h>
18
+
19
+
#include <uapi/linux/virtio_crypto.h>
20
+
#include "virtio_crypto_common.h"
21
+
22
+
struct virtio_crypto_rsa_ctx {
23
+
MPI n;
24
+
};
25
+
26
+
struct virtio_crypto_akcipher_ctx {
27
+
struct crypto_engine_ctx enginectx;
28
+
struct virtio_crypto *vcrypto;
29
+
struct crypto_akcipher *tfm;
30
+
bool session_valid;
31
+
__u64 session_id;
32
+
union {
33
+
struct virtio_crypto_rsa_ctx rsa_ctx;
34
+
};
35
+
};
36
+
37
+
struct virtio_crypto_akcipher_request {
38
+
struct virtio_crypto_request base;
39
+
struct virtio_crypto_akcipher_ctx *akcipher_ctx;
40
+
struct akcipher_request *akcipher_req;
41
+
void *src_buf;
42
+
void *dst_buf;
43
+
uint32_t opcode;
44
+
};
45
+
46
+
struct virtio_crypto_akcipher_algo {
47
+
uint32_t algonum;
48
+
uint32_t service;
49
+
unsigned int active_devs;
50
+
struct akcipher_alg algo;
51
+
};
52
+
53
+
static DEFINE_MUTEX(algs_lock);
54
+
55
+
static void virtio_crypto_akcipher_finalize_req(
56
+
struct virtio_crypto_akcipher_request *vc_akcipher_req,
57
+
struct akcipher_request *req, int err)
58
+
{
59
+
virtcrypto_clear_request(&vc_akcipher_req->base);
60
+
61
+
crypto_finalize_akcipher_request(vc_akcipher_req->base.dataq->engine, req, err);
62
+
}
63
+
64
+
static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request *vc_req, int len)
65
+
{
66
+
struct virtio_crypto_akcipher_request *vc_akcipher_req =
67
+
container_of(vc_req, struct virtio_crypto_akcipher_request, base);
68
+
struct akcipher_request *akcipher_req;
69
+
int error;
70
+
71
+
switch (vc_req->status) {
72
+
case VIRTIO_CRYPTO_OK:
73
+
error = 0;
74
+
break;
75
+
case VIRTIO_CRYPTO_INVSESS:
76
+
case VIRTIO_CRYPTO_ERR:
77
+
error = -EINVAL;
78
+
break;
79
+
case VIRTIO_CRYPTO_BADMSG:
80
+
error = -EBADMSG;
81
+
break;
82
+
83
+
case VIRTIO_CRYPTO_KEY_REJECTED:
84
+
error = -EKEYREJECTED;
85
+
break;
86
+
87
+
default:
88
+
error = -EIO;
89
+
break;
90
+
}
91
+
92
+
akcipher_req = vc_akcipher_req->akcipher_req;
93
+
if (vc_akcipher_req->opcode != VIRTIO_CRYPTO_AKCIPHER_VERIFY)
94
+
sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst),
95
+
vc_akcipher_req->dst_buf, akcipher_req->dst_len);
96
+
virtio_crypto_akcipher_finalize_req(vc_akcipher_req, akcipher_req, error);
97
+
}
98
+
99
+
static int virtio_crypto_alg_akcipher_init_session(struct virtio_crypto_akcipher_ctx *ctx,
100
+
struct virtio_crypto_ctrl_header *header, void *para,
101
+
const uint8_t *key, unsigned int keylen)
102
+
{
103
+
struct scatterlist outhdr_sg, key_sg, inhdr_sg, *sgs[3];
104
+
struct virtio_crypto *vcrypto = ctx->vcrypto;
105
+
uint8_t *pkey;
106
+
unsigned int inlen;
107
+
int err;
108
+
unsigned int num_out = 0, num_in = 0;
109
+
110
+
pkey = kmemdup(key, keylen, GFP_ATOMIC);
111
+
if (!pkey)
112
+
return -ENOMEM;
113
+
114
+
spin_lock(&vcrypto->ctrl_lock);
115
+
memcpy(&vcrypto->ctrl.header, header, sizeof(vcrypto->ctrl.header));
116
+
memcpy(&vcrypto->ctrl.u, para, sizeof(vcrypto->ctrl.u));
117
+
vcrypto->input.status = cpu_to_le32(VIRTIO_CRYPTO_ERR);
118
+
119
+
sg_init_one(&outhdr_sg, &vcrypto->ctrl, sizeof(vcrypto->ctrl));
120
+
sgs[num_out++] = &outhdr_sg;
121
+
122
+
sg_init_one(&key_sg, pkey, keylen);
123
+
sgs[num_out++] = &key_sg;
124
+
125
+
sg_init_one(&inhdr_sg, &vcrypto->input, sizeof(vcrypto->input));
126
+
sgs[num_out + num_in++] = &inhdr_sg;
127
+
128
+
err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC);
129
+
if (err < 0)
130
+
goto out;
131
+
132
+
virtqueue_kick(vcrypto->ctrl_vq);
133
+
while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) &&
134
+
!virtqueue_is_broken(vcrypto->ctrl_vq))
135
+
cpu_relax();
136
+
137
+
if (le32_to_cpu(vcrypto->input.status) != VIRTIO_CRYPTO_OK) {
138
+
err = -EINVAL;
139
+
goto out;
140
+
}
141
+
142
+
ctx->session_id = le64_to_cpu(vcrypto->input.session_id);
143
+
ctx->session_valid = true;
144
+
err = 0;
145
+
146
+
out:
147
+
spin_unlock(&vcrypto->ctrl_lock);
148
+
kfree_sensitive(pkey);
149
+
150
+
if (err < 0)
151
+
pr_err("virtio_crypto: Create session failed status: %u\n",
152
+
le32_to_cpu(vcrypto->input.status));
153
+
154
+
return err;
155
+
}
156
+
157
+
static int virtio_crypto_alg_akcipher_close_session(struct virtio_crypto_akcipher_ctx *ctx)
158
+
{
159
+
struct scatterlist outhdr_sg, inhdr_sg, *sgs[2];
160
+
struct virtio_crypto_destroy_session_req *destroy_session;
161
+
struct virtio_crypto *vcrypto = ctx->vcrypto;
162
+
unsigned int num_out = 0, num_in = 0, inlen;
163
+
int err;
164
+
165
+
spin_lock(&vcrypto->ctrl_lock);
166
+
if (!ctx->session_valid) {
167
+
err = 0;
168
+
goto out;
169
+
}
170
+
vcrypto->ctrl_status.status = VIRTIO_CRYPTO_ERR;
171
+
vcrypto->ctrl.header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION);
172
+
vcrypto->ctrl.header.queue_id = 0;
173
+
174
+
destroy_session = &vcrypto->ctrl.u.destroy_session;
175
+
destroy_session->session_id = cpu_to_le64(ctx->session_id);
176
+
177
+
sg_init_one(&outhdr_sg, &vcrypto->ctrl, sizeof(vcrypto->ctrl));
178
+
sgs[num_out++] = &outhdr_sg;
179
+
180
+
sg_init_one(&inhdr_sg, &vcrypto->ctrl_status.status, sizeof(vcrypto->ctrl_status.status));
181
+
sgs[num_out + num_in++] = &inhdr_sg;
182
+
183
+
err = virtqueue_add_sgs(vcrypto->ctrl_vq, sgs, num_out, num_in, vcrypto, GFP_ATOMIC);
184
+
if (err < 0)
185
+
goto out;
186
+
187
+
virtqueue_kick(vcrypto->ctrl_vq);
188
+
while (!virtqueue_get_buf(vcrypto->ctrl_vq, &inlen) &&
189
+
!virtqueue_is_broken(vcrypto->ctrl_vq))
190
+
cpu_relax();
191
+
192
+
if (vcrypto->ctrl_status.status != VIRTIO_CRYPTO_OK) {
193
+
err = -EINVAL;
194
+
goto out;
195
+
}
196
+
197
+
err = 0;
198
+
ctx->session_valid = false;
199
+
200
+
out:
201
+
spin_unlock(&vcrypto->ctrl_lock);
202
+
if (err < 0) {
203
+
pr_err("virtio_crypto: Close session failed status: %u, session_id: 0x%llx\n",
204
+
vcrypto->ctrl_status.status, destroy_session->session_id);
205
+
}
206
+
207
+
return err;
208
+
}
209
+
210
+
static int __virtio_crypto_akcipher_do_req(struct virtio_crypto_akcipher_request *vc_akcipher_req,
211
+
struct akcipher_request *req, struct data_queue *data_vq)
212
+
{
213
+
struct virtio_crypto_akcipher_ctx *ctx = vc_akcipher_req->akcipher_ctx;
214
+
struct virtio_crypto_request *vc_req = &vc_akcipher_req->base;
215
+
struct virtio_crypto *vcrypto = ctx->vcrypto;
216
+
struct virtio_crypto_op_data_req *req_data = vc_req->req_data;
217
+
struct scatterlist *sgs[4], outhdr_sg, inhdr_sg, srcdata_sg, dstdata_sg;
218
+
void *src_buf = NULL, *dst_buf = NULL;
219
+
unsigned int num_out = 0, num_in = 0;
220
+
int node = dev_to_node(&vcrypto->vdev->dev);
221
+
unsigned long flags;
222
+
int ret = -ENOMEM;
223
+
bool verify = vc_akcipher_req->opcode == VIRTIO_CRYPTO_AKCIPHER_VERIFY;
224
+
unsigned int src_len = verify ? req->src_len + req->dst_len : req->src_len;
225
+
226
+
/* out header */
227
+
sg_init_one(&outhdr_sg, req_data, sizeof(*req_data));
228
+
sgs[num_out++] = &outhdr_sg;
229
+
230
+
/* src data */
231
+
src_buf = kcalloc_node(src_len, 1, GFP_KERNEL, node);
232
+
if (!src_buf)
233
+
goto err;
234
+
235
+
if (verify) {
236
+
/* for verify operation, both src and dst data work as OUT direction */
237
+
sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, src_len);
238
+
sg_init_one(&srcdata_sg, src_buf, src_len);
239
+
sgs[num_out++] = &srcdata_sg;
240
+
} else {
241
+
sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, src_len);
242
+
sg_init_one(&srcdata_sg, src_buf, src_len);
243
+
sgs[num_out++] = &srcdata_sg;
244
+
245
+
/* dst data */
246
+
dst_buf = kcalloc_node(req->dst_len, 1, GFP_KERNEL, node);
247
+
if (!dst_buf)
248
+
goto err;
249
+
250
+
sg_init_one(&dstdata_sg, dst_buf, req->dst_len);
251
+
sgs[num_out + num_in++] = &dstdata_sg;
252
+
}
253
+
254
+
vc_akcipher_req->src_buf = src_buf;
255
+
vc_akcipher_req->dst_buf = dst_buf;
256
+
257
+
/* in header */
258
+
sg_init_one(&inhdr_sg, &vc_req->status, sizeof(vc_req->status));
259
+
sgs[num_out + num_in++] = &inhdr_sg;
260
+
261
+
spin_lock_irqsave(&data_vq->lock, flags);
262
+
ret = virtqueue_add_sgs(data_vq->vq, sgs, num_out, num_in, vc_req, GFP_ATOMIC);
263
+
virtqueue_kick(data_vq->vq);
264
+
spin_unlock_irqrestore(&data_vq->lock, flags);
265
+
if (ret)
266
+
goto err;
267
+
268
+
return 0;
269
+
270
+
err:
271
+
kfree(src_buf);
272
+
kfree(dst_buf);
273
+
274
+
return -ENOMEM;
275
+
}
276
+
277
+
static int virtio_crypto_rsa_do_req(struct crypto_engine *engine, void *vreq)
278
+
{
279
+
struct akcipher_request *req = container_of(vreq, struct akcipher_request, base);
280
+
struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req);
281
+
struct virtio_crypto_request *vc_req = &vc_akcipher_req->base;
282
+
struct virtio_crypto_akcipher_ctx *ctx = vc_akcipher_req->akcipher_ctx;
283
+
struct virtio_crypto *vcrypto = ctx->vcrypto;
284
+
struct data_queue *data_vq = vc_req->dataq;
285
+
struct virtio_crypto_op_header *header;
286
+
struct virtio_crypto_akcipher_data_req *akcipher_req;
287
+
int ret;
288
+
289
+
vc_req->sgs = NULL;
290
+
vc_req->req_data = kzalloc_node(sizeof(*vc_req->req_data),
291
+
GFP_KERNEL, dev_to_node(&vcrypto->vdev->dev));
292
+
if (!vc_req->req_data)
293
+
return -ENOMEM;
294
+
295
+
/* build request header */
296
+
header = &vc_req->req_data->header;
297
+
header->opcode = cpu_to_le32(vc_akcipher_req->opcode);
298
+
header->algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA);
299
+
header->session_id = cpu_to_le64(ctx->session_id);
300
+
301
+
/* build request akcipher data */
302
+
akcipher_req = &vc_req->req_data->u.akcipher_req;
303
+
akcipher_req->para.src_data_len = cpu_to_le32(req->src_len);
304
+
akcipher_req->para.dst_data_len = cpu_to_le32(req->dst_len);
305
+
306
+
ret = __virtio_crypto_akcipher_do_req(vc_akcipher_req, req, data_vq);
307
+
if (ret < 0) {
308
+
kfree_sensitive(vc_req->req_data);
309
+
vc_req->req_data = NULL;
310
+
return ret;
311
+
}
312
+
313
+
return 0;
314
+
}
315
+
316
+
static int virtio_crypto_rsa_req(struct akcipher_request *req, uint32_t opcode)
317
+
{
318
+
struct crypto_akcipher *atfm = crypto_akcipher_reqtfm(req);
319
+
struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(atfm);
320
+
struct virtio_crypto_akcipher_request *vc_akcipher_req = akcipher_request_ctx(req);
321
+
struct virtio_crypto_request *vc_req = &vc_akcipher_req->base;
322
+
struct virtio_crypto *vcrypto = ctx->vcrypto;
323
+
/* Use the first data virtqueue as default */
324
+
struct data_queue *data_vq = &vcrypto->data_vq[0];
325
+
326
+
vc_req->dataq = data_vq;
327
+
vc_req->alg_cb = virtio_crypto_dataq_akcipher_callback;
328
+
vc_akcipher_req->akcipher_ctx = ctx;
329
+
vc_akcipher_req->akcipher_req = req;
330
+
vc_akcipher_req->opcode = opcode;
331
+
332
+
return crypto_transfer_akcipher_request_to_engine(data_vq->engine, req);
333
+
}
334
+
335
+
static int virtio_crypto_rsa_encrypt(struct akcipher_request *req)
336
+
{
337
+
return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_ENCRYPT);
338
+
}
339
+
340
+
static int virtio_crypto_rsa_decrypt(struct akcipher_request *req)
341
+
{
342
+
return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_DECRYPT);
343
+
}
344
+
345
+
static int virtio_crypto_rsa_sign(struct akcipher_request *req)
346
+
{
347
+
return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_SIGN);
348
+
}
349
+
350
+
static int virtio_crypto_rsa_verify(struct akcipher_request *req)
351
+
{
352
+
return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_VERIFY);
353
+
}
354
+
355
+
static int virtio_crypto_rsa_set_key(struct crypto_akcipher *tfm,
356
+
const void *key,
357
+
unsigned int keylen,
358
+
bool private,
359
+
int padding_algo,
360
+
int hash_algo)
361
+
{
362
+
struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
363
+
struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx;
364
+
struct virtio_crypto *vcrypto;
365
+
struct virtio_crypto_ctrl_header header;
366
+
struct virtio_crypto_akcipher_session_para para;
367
+
struct rsa_key rsa_key = {0};
368
+
int node = virtio_crypto_get_current_node();
369
+
uint32_t keytype;
370
+
int ret;
371
+
372
+
/* mpi_free will test n, just free it. */
373
+
mpi_free(rsa_ctx->n);
374
+
rsa_ctx->n = NULL;
375
+
376
+
if (private) {
377
+
keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE;
378
+
ret = rsa_parse_priv_key(&rsa_key, key, keylen);
379
+
} else {
380
+
keytype = VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC;
381
+
ret = rsa_parse_pub_key(&rsa_key, key, keylen);
382
+
}
383
+
384
+
if (ret)
385
+
return ret;
386
+
387
+
rsa_ctx->n = mpi_read_raw_data(rsa_key.n, rsa_key.n_sz);
388
+
if (!rsa_ctx->n)
389
+
return -ENOMEM;
390
+
391
+
if (!ctx->vcrypto) {
392
+
vcrypto = virtcrypto_get_dev_node(node, VIRTIO_CRYPTO_SERVICE_AKCIPHER,
393
+
VIRTIO_CRYPTO_AKCIPHER_RSA);
394
+
if (!vcrypto) {
395
+
pr_err("virtio_crypto: Could not find a virtio device in the system or unsupported algo\n");
396
+
return -ENODEV;
397
+
}
398
+
399
+
ctx->vcrypto = vcrypto;
400
+
} else {
401
+
virtio_crypto_alg_akcipher_close_session(ctx);
402
+
}
403
+
404
+
/* set ctrl header */
405
+
header.opcode = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION);
406
+
header.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA);
407
+
header.queue_id = 0;
408
+
409
+
/* set RSA para */
410
+
para.algo = cpu_to_le32(VIRTIO_CRYPTO_AKCIPHER_RSA);
411
+
para.keytype = cpu_to_le32(keytype);
412
+
para.keylen = cpu_to_le32(keylen);
413
+
para.u.rsa.padding_algo = cpu_to_le32(padding_algo);
414
+
para.u.rsa.hash_algo = cpu_to_le32(hash_algo);
415
+
416
+
return virtio_crypto_alg_akcipher_init_session(ctx, &header, ¶, key, keylen);
417
+
}
418
+
419
+
static int virtio_crypto_rsa_raw_set_priv_key(struct crypto_akcipher *tfm,
420
+
const void *key,
421
+
unsigned int keylen)
422
+
{
423
+
return virtio_crypto_rsa_set_key(tfm, key, keylen, 1,
424
+
VIRTIO_CRYPTO_RSA_RAW_PADDING,
425
+
VIRTIO_CRYPTO_RSA_NO_HASH);
426
+
}
427
+
428
+
429
+
static int virtio_crypto_p1pad_rsa_sha1_set_priv_key(struct crypto_akcipher *tfm,
430
+
const void *key,
431
+
unsigned int keylen)
432
+
{
433
+
return virtio_crypto_rsa_set_key(tfm, key, keylen, 1,
434
+
VIRTIO_CRYPTO_RSA_PKCS1_PADDING,
435
+
VIRTIO_CRYPTO_RSA_SHA1);
436
+
}
437
+
438
+
static int virtio_crypto_rsa_raw_set_pub_key(struct crypto_akcipher *tfm,
439
+
const void *key,
440
+
unsigned int keylen)
441
+
{
442
+
return virtio_crypto_rsa_set_key(tfm, key, keylen, 0,
443
+
VIRTIO_CRYPTO_RSA_RAW_PADDING,
444
+
VIRTIO_CRYPTO_RSA_NO_HASH);
445
+
}
446
+
447
+
static int virtio_crypto_p1pad_rsa_sha1_set_pub_key(struct crypto_akcipher *tfm,
448
+
const void *key,
449
+
unsigned int keylen)
450
+
{
451
+
return virtio_crypto_rsa_set_key(tfm, key, keylen, 0,
452
+
VIRTIO_CRYPTO_RSA_PKCS1_PADDING,
453
+
VIRTIO_CRYPTO_RSA_SHA1);
454
+
}
455
+
456
+
static unsigned int virtio_crypto_rsa_max_size(struct crypto_akcipher *tfm)
457
+
{
458
+
struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
459
+
struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx;
460
+
461
+
return mpi_get_size(rsa_ctx->n);
462
+
}
463
+
464
+
static int virtio_crypto_rsa_init_tfm(struct crypto_akcipher *tfm)
465
+
{
466
+
struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
467
+
468
+
ctx->tfm = tfm;
469
+
ctx->enginectx.op.do_one_request = virtio_crypto_rsa_do_req;
470
+
ctx->enginectx.op.prepare_request = NULL;
471
+
ctx->enginectx.op.unprepare_request = NULL;
472
+
473
+
return 0;
474
+
}
475
+
476
+
static void virtio_crypto_rsa_exit_tfm(struct crypto_akcipher *tfm)
477
+
{
478
+
struct virtio_crypto_akcipher_ctx *ctx = akcipher_tfm_ctx(tfm);
479
+
struct virtio_crypto_rsa_ctx *rsa_ctx = &ctx->rsa_ctx;
480
+
481
+
virtio_crypto_alg_akcipher_close_session(ctx);
482
+
virtcrypto_dev_put(ctx->vcrypto);
483
+
mpi_free(rsa_ctx->n);
484
+
rsa_ctx->n = NULL;
485
+
}
486
+
487
+
static struct virtio_crypto_akcipher_algo virtio_crypto_akcipher_algs[] = {
488
+
{
489
+
.algonum = VIRTIO_CRYPTO_AKCIPHER_RSA,
490
+
.service = VIRTIO_CRYPTO_SERVICE_AKCIPHER,
491
+
.algo = {
492
+
.encrypt = virtio_crypto_rsa_encrypt,
493
+
.decrypt = virtio_crypto_rsa_decrypt,
494
+
.set_pub_key = virtio_crypto_rsa_raw_set_pub_key,
495
+
.set_priv_key = virtio_crypto_rsa_raw_set_priv_key,
496
+
.max_size = virtio_crypto_rsa_max_size,
497
+
.init = virtio_crypto_rsa_init_tfm,
498
+
.exit = virtio_crypto_rsa_exit_tfm,
499
+
.reqsize = sizeof(struct virtio_crypto_akcipher_request),
500
+
.base = {
501
+
.cra_name = "rsa",
502
+
.cra_driver_name = "virtio-crypto-rsa",
503
+
.cra_priority = 150,
504
+
.cra_module = THIS_MODULE,
505
+
.cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx),
506
+
},
507
+
},
508
+
},
509
+
{
510
+
.algonum = VIRTIO_CRYPTO_AKCIPHER_RSA,
511
+
.service = VIRTIO_CRYPTO_SERVICE_AKCIPHER,
512
+
.algo = {
513
+
.encrypt = virtio_crypto_rsa_encrypt,
514
+
.decrypt = virtio_crypto_rsa_decrypt,
515
+
.sign = virtio_crypto_rsa_sign,
516
+
.verify = virtio_crypto_rsa_verify,
517
+
.set_pub_key = virtio_crypto_p1pad_rsa_sha1_set_pub_key,
518
+
.set_priv_key = virtio_crypto_p1pad_rsa_sha1_set_priv_key,
519
+
.max_size = virtio_crypto_rsa_max_size,
520
+
.init = virtio_crypto_rsa_init_tfm,
521
+
.exit = virtio_crypto_rsa_exit_tfm,
522
+
.reqsize = sizeof(struct virtio_crypto_akcipher_request),
523
+
.base = {
524
+
.cra_name = "pkcs1pad(rsa,sha1)",
525
+
.cra_driver_name = "virtio-pkcs1-rsa-with-sha1",
526
+
.cra_priority = 150,
527
+
.cra_module = THIS_MODULE,
528
+
.cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx),
529
+
},
530
+
},
531
+
},
532
+
};
533
+
534
+
int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto)
535
+
{
536
+
int ret = 0;
537
+
int i = 0;
538
+
539
+
mutex_lock(&algs_lock);
540
+
541
+
for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) {
542
+
uint32_t service = virtio_crypto_akcipher_algs[i].service;
543
+
uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum;
544
+
545
+
if (!virtcrypto_algo_is_supported(vcrypto, service, algonum))
546
+
continue;
547
+
548
+
if (virtio_crypto_akcipher_algs[i].active_devs == 0) {
549
+
ret = crypto_register_akcipher(&virtio_crypto_akcipher_algs[i].algo);
550
+
if (ret)
551
+
goto unlock;
552
+
}
553
+
554
+
virtio_crypto_akcipher_algs[i].active_devs++;
555
+
dev_info(&vcrypto->vdev->dev, "Registered akcipher algo %s\n",
556
+
virtio_crypto_akcipher_algs[i].algo.base.cra_name);
557
+
}
558
+
559
+
unlock:
560
+
mutex_unlock(&algs_lock);
561
+
return ret;
562
+
}
563
+
564
+
void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto)
565
+
{
566
+
int i = 0;
567
+
568
+
mutex_lock(&algs_lock);
569
+
570
+
for (i = 0; i < ARRAY_SIZE(virtio_crypto_akcipher_algs); i++) {
571
+
uint32_t service = virtio_crypto_akcipher_algs[i].service;
572
+
uint32_t algonum = virtio_crypto_akcipher_algs[i].algonum;
573
+
574
+
if (virtio_crypto_akcipher_algs[i].active_devs == 0 ||
575
+
!virtcrypto_algo_is_supported(vcrypto, service, algonum))
576
+
continue;
577
+
578
+
if (virtio_crypto_akcipher_algs[i].active_devs == 1)
579
+
crypto_unregister_akcipher(&virtio_crypto_akcipher_algs[i].algo);
580
+
581
+
virtio_crypto_akcipher_algs[i].active_devs--;
582
+
}
583
+
584
+
mutex_unlock(&algs_lock);
585
+
}
+2
-2
drivers/crypto/virtio/virtio_crypto_algs.c
drivers/crypto/virtio/virtio_crypto_skcipher_algs.c
+2
-2
drivers/crypto/virtio/virtio_crypto_algs.c
drivers/crypto/virtio/virtio_crypto_skcipher_algs.c
···
613
613
},
614
614
} };
615
615
616
-
int virtio_crypto_algs_register(struct virtio_crypto *vcrypto)
616
+
int virtio_crypto_skcipher_algs_register(struct virtio_crypto *vcrypto)
617
617
{
618
618
int ret = 0;
619
619
int i = 0;
···
644
644
return ret;
645
645
}
646
646
647
-
void virtio_crypto_algs_unregister(struct virtio_crypto *vcrypto)
647
+
void virtio_crypto_skcipher_algs_unregister(struct virtio_crypto *vcrypto)
648
648
{
649
649
int i = 0;
650
650
+5
-2
drivers/crypto/virtio/virtio_crypto_common.h
+5
-2
drivers/crypto/virtio/virtio_crypto_common.h
···
56
56
u32 mac_algo_l;
57
57
u32 mac_algo_h;
58
58
u32 aead_algo;
59
+
u32 akcipher_algo;
59
60
60
61
/* Maximum length of cipher key */
61
62
u32 max_cipher_key_len;
···
130
129
return node;
131
130
}
132
131
133
-
int virtio_crypto_algs_register(struct virtio_crypto *vcrypto);
134
-
void virtio_crypto_algs_unregister(struct virtio_crypto *vcrypto);
132
+
int virtio_crypto_skcipher_algs_register(struct virtio_crypto *vcrypto);
133
+
void virtio_crypto_skcipher_algs_unregister(struct virtio_crypto *vcrypto);
134
+
int virtio_crypto_akcipher_algs_register(struct virtio_crypto *vcrypto);
135
+
void virtio_crypto_akcipher_algs_unregister(struct virtio_crypto *vcrypto);
135
136
136
137
#endif /* _VIRTIO_CRYPTO_COMMON_H */
+5
-1
drivers/crypto/virtio/virtio_crypto_core.c
+5
-1
drivers/crypto/virtio/virtio_crypto_core.c
···
297
297
u32 mac_algo_l = 0;
298
298
u32 mac_algo_h = 0;
299
299
u32 aead_algo = 0;
300
+
u32 akcipher_algo = 0;
300
301
u32 crypto_services = 0;
301
302
302
303
if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
···
349
348
mac_algo_h, &mac_algo_h);
350
349
virtio_cread_le(vdev, struct virtio_crypto_config,
351
350
aead_algo, &aead_algo);
351
+
if (crypto_services & (1 << VIRTIO_CRYPTO_SERVICE_AKCIPHER))
352
+
virtio_cread_le(vdev, struct virtio_crypto_config,
353
+
akcipher_algo, &akcipher_algo);
352
354
353
355
/* Add virtio crypto device to global table */
354
356
err = virtcrypto_devmgr_add_dev(vcrypto);
···
378
374
vcrypto->mac_algo_h = mac_algo_h;
379
375
vcrypto->hash_algo = hash_algo;
380
376
vcrypto->aead_algo = aead_algo;
381
-
377
+
vcrypto->akcipher_algo = akcipher_algo;
382
378
383
379
dev_info(&vdev->dev,
384
380
"max_queues: %u, max_cipher_key_len: %u, max_auth_key_len: %u, max_size 0x%llx\n",
+14
-3
drivers/crypto/virtio/virtio_crypto_mgr.c
+14
-3
drivers/crypto/virtio/virtio_crypto_mgr.c
···
237
237
*/
238
238
int virtcrypto_dev_start(struct virtio_crypto *vcrypto)
239
239
{
240
-
if (virtio_crypto_algs_register(vcrypto)) {
241
-
pr_err("virtio_crypto: Failed to register crypto algs\n");
240
+
if (virtio_crypto_skcipher_algs_register(vcrypto)) {
241
+
pr_err("virtio_crypto: Failed to register crypto skcipher algs\n");
242
+
return -EFAULT;
243
+
}
244
+
245
+
if (virtio_crypto_akcipher_algs_register(vcrypto)) {
246
+
pr_err("virtio_crypto: Failed to register crypto akcipher algs\n");
247
+
virtio_crypto_skcipher_algs_unregister(vcrypto);
242
248
return -EFAULT;
243
249
}
244
250
···
263
257
*/
264
258
void virtcrypto_dev_stop(struct virtio_crypto *vcrypto)
265
259
{
266
-
virtio_crypto_algs_unregister(vcrypto);
260
+
virtio_crypto_skcipher_algs_unregister(vcrypto);
261
+
virtio_crypto_akcipher_algs_unregister(vcrypto);
267
262
}
268
263
269
264
/*
···
318
311
319
312
case VIRTIO_CRYPTO_SERVICE_AEAD:
320
313
algo_mask = vcrypto->aead_algo;
314
+
break;
315
+
316
+
case VIRTIO_CRYPTO_SERVICE_AKCIPHER:
317
+
algo_mask = vcrypto->akcipher_algo;
321
318
break;
322
319
}
323
320
+376
-13
drivers/net/virtio_net.c
+376
-13
drivers/net/virtio_net.c
···
169
169
struct xdp_rxq_info xdp_rxq;
170
170
};
171
171
172
+
/* This structure can contain rss message with maximum settings for indirection table and keysize
173
+
* Note, that default structure that describes RSS configuration virtio_net_rss_config
174
+
* contains same info but can't handle table values.
175
+
* In any case, structure would be passed to virtio hw through sg_buf split by parts
176
+
* because table sizes may be differ according to the device configuration.
177
+
*/
178
+
#define VIRTIO_NET_RSS_MAX_KEY_SIZE 40
179
+
#define VIRTIO_NET_RSS_MAX_TABLE_LEN 128
180
+
struct virtio_net_ctrl_rss {
181
+
u32 hash_types;
182
+
u16 indirection_table_mask;
183
+
u16 unclassified_queue;
184
+
u16 indirection_table[VIRTIO_NET_RSS_MAX_TABLE_LEN];
185
+
u16 max_tx_vq;
186
+
u8 hash_key_length;
187
+
u8 key[VIRTIO_NET_RSS_MAX_KEY_SIZE];
188
+
};
189
+
172
190
/* Control VQ buffers: protected by the rtnl lock */
173
191
struct control_buf {
174
192
struct virtio_net_ctrl_hdr hdr;
···
196
178
u8 allmulti;
197
179
__virtio16 vid;
198
180
__virtio64 offloads;
181
+
struct virtio_net_ctrl_rss rss;
199
182
};
200
183
201
184
struct virtnet_info {
···
224
205
225
206
/* Host will merge rx buffers for big packets (shake it! shake it!) */
226
207
bool mergeable_rx_bufs;
208
+
209
+
/* Host supports rss and/or hash report */
210
+
bool has_rss;
211
+
bool has_rss_hash_report;
212
+
u8 rss_key_size;
213
+
u16 rss_indir_table_size;
214
+
u32 rss_hash_types_supported;
215
+
u32 rss_hash_types_saved;
227
216
228
217
/* Has control virtqueue */
229
218
bool has_cvq;
···
269
242
};
270
243
271
244
struct padded_vnet_hdr {
272
-
struct virtio_net_hdr_mrg_rxbuf hdr;
245
+
struct virtio_net_hdr_v1_hash hdr;
273
246
/*
274
247
* hdr is in a separate sg buffer, and data sg buffer shares same page
275
248
* with this header sg. This padding makes next sg 16 byte aligned
276
249
* after the header.
277
250
*/
278
-
char padding[4];
251
+
char padding[12];
279
252
};
280
253
281
254
static bool is_xdp_frame(void *ptr)
···
423
396
424
397
hdr_len = vi->hdr_len;
425
398
if (vi->mergeable_rx_bufs)
426
-
hdr_padded_len = sizeof(*hdr);
399
+
hdr_padded_len = hdr_len;
427
400
else
428
401
hdr_padded_len = sizeof(struct padded_vnet_hdr);
429
402
···
1150
1123
return NULL;
1151
1124
}
1152
1125
1126
+
static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash,
1127
+
struct sk_buff *skb)
1128
+
{
1129
+
enum pkt_hash_types rss_hash_type;
1130
+
1131
+
if (!hdr_hash || !skb)
1132
+
return;
1133
+
1134
+
switch ((int)hdr_hash->hash_report) {
1135
+
case VIRTIO_NET_HASH_REPORT_TCPv4:
1136
+
case VIRTIO_NET_HASH_REPORT_UDPv4:
1137
+
case VIRTIO_NET_HASH_REPORT_TCPv6:
1138
+
case VIRTIO_NET_HASH_REPORT_UDPv6:
1139
+
case VIRTIO_NET_HASH_REPORT_TCPv6_EX:
1140
+
case VIRTIO_NET_HASH_REPORT_UDPv6_EX:
1141
+
rss_hash_type = PKT_HASH_TYPE_L4;
1142
+
break;
1143
+
case VIRTIO_NET_HASH_REPORT_IPv4:
1144
+
case VIRTIO_NET_HASH_REPORT_IPv6:
1145
+
case VIRTIO_NET_HASH_REPORT_IPv6_EX:
1146
+
rss_hash_type = PKT_HASH_TYPE_L3;
1147
+
break;
1148
+
case VIRTIO_NET_HASH_REPORT_NONE:
1149
+
default:
1150
+
rss_hash_type = PKT_HASH_TYPE_NONE;
1151
+
}
1152
+
skb_set_hash(skb, (unsigned int)hdr_hash->hash_value, rss_hash_type);
1153
+
}
1154
+
1153
1155
static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq,
1154
1156
void *buf, unsigned int len, void **ctx,
1155
1157
unsigned int *xdp_xmit,
···
1213
1157
return;
1214
1158
1215
1159
hdr = skb_vnet_hdr(skb);
1160
+
if (dev->features & NETIF_F_RXHASH && vi->has_rss_hash_report)
1161
+
virtio_skb_set_hash((const struct virtio_net_hdr_v1_hash *)hdr, skb);
1216
1162
1217
1163
if (hdr->hdr.flags & VIRTIO_NET_HDR_F_DATA_VALID)
1218
1164
skb->ip_summed = CHECKSUM_UNNECESSARY;
···
1324
1266
struct ewma_pkt_len *avg_pkt_len,
1325
1267
unsigned int room)
1326
1268
{
1327
-
const size_t hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1269
+
struct virtnet_info *vi = rq->vq->vdev->priv;
1270
+
const size_t hdr_len = vi->hdr_len;
1328
1271
unsigned int len;
1329
1272
1330
1273
if (room)
···
2242
2183
ring->tx_pending = ring->tx_max_pending;
2243
2184
}
2244
2185
2186
+
static bool virtnet_commit_rss_command(struct virtnet_info *vi)
2187
+
{
2188
+
struct net_device *dev = vi->dev;
2189
+
struct scatterlist sgs[4];
2190
+
unsigned int sg_buf_size;
2191
+
2192
+
/* prepare sgs */
2193
+
sg_init_table(sgs, 4);
2194
+
2195
+
sg_buf_size = offsetof(struct virtio_net_ctrl_rss, indirection_table);
2196
+
sg_set_buf(&sgs[0], &vi->ctrl->rss, sg_buf_size);
2197
+
2198
+
sg_buf_size = sizeof(uint16_t) * (vi->ctrl->rss.indirection_table_mask + 1);
2199
+
sg_set_buf(&sgs[1], vi->ctrl->rss.indirection_table, sg_buf_size);
2200
+
2201
+
sg_buf_size = offsetof(struct virtio_net_ctrl_rss, key)
2202
+
- offsetof(struct virtio_net_ctrl_rss, max_tx_vq);
2203
+
sg_set_buf(&sgs[2], &vi->ctrl->rss.max_tx_vq, sg_buf_size);
2204
+
2205
+
sg_buf_size = vi->rss_key_size;
2206
+
sg_set_buf(&sgs[3], vi->ctrl->rss.key, sg_buf_size);
2207
+
2208
+
if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ,
2209
+
vi->has_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG
2210
+
: VIRTIO_NET_CTRL_MQ_HASH_CONFIG, sgs)) {
2211
+
dev_warn(&dev->dev, "VIRTIONET issue with committing RSS sgs\n");
2212
+
return false;
2213
+
}
2214
+
return true;
2215
+
}
2216
+
2217
+
static void virtnet_init_default_rss(struct virtnet_info *vi)
2218
+
{
2219
+
u32 indir_val = 0;
2220
+
int i = 0;
2221
+
2222
+
vi->ctrl->rss.hash_types = vi->rss_hash_types_supported;
2223
+
vi->rss_hash_types_saved = vi->rss_hash_types_supported;
2224
+
vi->ctrl->rss.indirection_table_mask = vi->rss_indir_table_size
2225
+
? vi->rss_indir_table_size - 1 : 0;
2226
+
vi->ctrl->rss.unclassified_queue = 0;
2227
+
2228
+
for (; i < vi->rss_indir_table_size; ++i) {
2229
+
indir_val = ethtool_rxfh_indir_default(i, vi->curr_queue_pairs);
2230
+
vi->ctrl->rss.indirection_table[i] = indir_val;
2231
+
}
2232
+
2233
+
vi->ctrl->rss.max_tx_vq = vi->curr_queue_pairs;
2234
+
vi->ctrl->rss.hash_key_length = vi->rss_key_size;
2235
+
2236
+
netdev_rss_key_fill(vi->ctrl->rss.key, vi->rss_key_size);
2237
+
}
2238
+
2239
+
static void virtnet_get_hashflow(const struct virtnet_info *vi, struct ethtool_rxnfc *info)
2240
+
{
2241
+
info->data = 0;
2242
+
switch (info->flow_type) {
2243
+
case TCP_V4_FLOW:
2244
+
if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) {
2245
+
info->data = RXH_IP_SRC | RXH_IP_DST |
2246
+
RXH_L4_B_0_1 | RXH_L4_B_2_3;
2247
+
} else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
2248
+
info->data = RXH_IP_SRC | RXH_IP_DST;
2249
+
}
2250
+
break;
2251
+
case TCP_V6_FLOW:
2252
+
if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) {
2253
+
info->data = RXH_IP_SRC | RXH_IP_DST |
2254
+
RXH_L4_B_0_1 | RXH_L4_B_2_3;
2255
+
} else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
2256
+
info->data = RXH_IP_SRC | RXH_IP_DST;
2257
+
}
2258
+
break;
2259
+
case UDP_V4_FLOW:
2260
+
if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) {
2261
+
info->data = RXH_IP_SRC | RXH_IP_DST |
2262
+
RXH_L4_B_0_1 | RXH_L4_B_2_3;
2263
+
} else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) {
2264
+
info->data = RXH_IP_SRC | RXH_IP_DST;
2265
+
}
2266
+
break;
2267
+
case UDP_V6_FLOW:
2268
+
if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) {
2269
+
info->data = RXH_IP_SRC | RXH_IP_DST |
2270
+
RXH_L4_B_0_1 | RXH_L4_B_2_3;
2271
+
} else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) {
2272
+
info->data = RXH_IP_SRC | RXH_IP_DST;
2273
+
}
2274
+
break;
2275
+
case IPV4_FLOW:
2276
+
if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4)
2277
+
info->data = RXH_IP_SRC | RXH_IP_DST;
2278
+
2279
+
break;
2280
+
case IPV6_FLOW:
2281
+
if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6)
2282
+
info->data = RXH_IP_SRC | RXH_IP_DST;
2283
+
2284
+
break;
2285
+
default:
2286
+
info->data = 0;
2287
+
break;
2288
+
}
2289
+
}
2290
+
2291
+
static bool virtnet_set_hashflow(struct virtnet_info *vi, struct ethtool_rxnfc *info)
2292
+
{
2293
+
u32 new_hashtypes = vi->rss_hash_types_saved;
2294
+
bool is_disable = info->data & RXH_DISCARD;
2295
+
bool is_l4 = info->data == (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3);
2296
+
2297
+
/* supports only 'sd', 'sdfn' and 'r' */
2298
+
if (!((info->data == (RXH_IP_SRC | RXH_IP_DST)) | is_l4 | is_disable))
2299
+
return false;
2300
+
2301
+
switch (info->flow_type) {
2302
+
case TCP_V4_FLOW:
2303
+
new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv4);
2304
+
if (!is_disable)
2305
+
new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4
2306
+
| (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv4 : 0);
2307
+
break;
2308
+
case UDP_V4_FLOW:
2309
+
new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_UDPv4);
2310
+
if (!is_disable)
2311
+
new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4
2312
+
| (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv4 : 0);
2313
+
break;
2314
+
case IPV4_FLOW:
2315
+
new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv4;
2316
+
if (!is_disable)
2317
+
new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv4;
2318
+
break;
2319
+
case TCP_V6_FLOW:
2320
+
new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_TCPv6);
2321
+
if (!is_disable)
2322
+
new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6
2323
+
| (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv6 : 0);
2324
+
break;
2325
+
case UDP_V6_FLOW:
2326
+
new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_UDPv6);
2327
+
if (!is_disable)
2328
+
new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6
2329
+
| (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv6 : 0);
2330
+
break;
2331
+
case IPV6_FLOW:
2332
+
new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv6;
2333
+
if (!is_disable)
2334
+
new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv6;
2335
+
break;
2336
+
default:
2337
+
/* unsupported flow */
2338
+
return false;
2339
+
}
2340
+
2341
+
/* if unsupported hashtype was set */
2342
+
if (new_hashtypes != (new_hashtypes & vi->rss_hash_types_supported))
2343
+
return false;
2344
+
2345
+
if (new_hashtypes != vi->rss_hash_types_saved) {
2346
+
vi->rss_hash_types_saved = new_hashtypes;
2347
+
vi->ctrl->rss.hash_types = vi->rss_hash_types_saved;
2348
+
if (vi->dev->features & NETIF_F_RXHASH)
2349
+
return virtnet_commit_rss_command(vi);
2350
+
}
2351
+
2352
+
return true;
2353
+
}
2245
2354
2246
2355
static void virtnet_get_drvinfo(struct net_device *dev,
2247
2356
struct ethtool_drvinfo *info)
···
2638
2411
vi->duplex = duplex;
2639
2412
}
2640
2413
2414
+
static u32 virtnet_get_rxfh_key_size(struct net_device *dev)
2415
+
{
2416
+
return ((struct virtnet_info *)netdev_priv(dev))->rss_key_size;
2417
+
}
2418
+
2419
+
static u32 virtnet_get_rxfh_indir_size(struct net_device *dev)
2420
+
{
2421
+
return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size;
2422
+
}
2423
+
2424
+
static int virtnet_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, u8 *hfunc)
2425
+
{
2426
+
struct virtnet_info *vi = netdev_priv(dev);
2427
+
int i;
2428
+
2429
+
if (indir) {
2430
+
for (i = 0; i < vi->rss_indir_table_size; ++i)
2431
+
indir[i] = vi->ctrl->rss.indirection_table[i];
2432
+
}
2433
+
2434
+
if (key)
2435
+
memcpy(key, vi->ctrl->rss.key, vi->rss_key_size);
2436
+
2437
+
if (hfunc)
2438
+
*hfunc = ETH_RSS_HASH_TOP;
2439
+
2440
+
return 0;
2441
+
}
2442
+
2443
+
static int virtnet_set_rxfh(struct net_device *dev, const u32 *indir, const u8 *key, const u8 hfunc)
2444
+
{
2445
+
struct virtnet_info *vi = netdev_priv(dev);
2446
+
int i;
2447
+
2448
+
if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)
2449
+
return -EOPNOTSUPP;
2450
+
2451
+
if (indir) {
2452
+
for (i = 0; i < vi->rss_indir_table_size; ++i)
2453
+
vi->ctrl->rss.indirection_table[i] = indir[i];
2454
+
}
2455
+
if (key)
2456
+
memcpy(vi->ctrl->rss.key, key, vi->rss_key_size);
2457
+
2458
+
virtnet_commit_rss_command(vi);
2459
+
2460
+
return 0;
2461
+
}
2462
+
2463
+
static int virtnet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, u32 *rule_locs)
2464
+
{
2465
+
struct virtnet_info *vi = netdev_priv(dev);
2466
+
int rc = 0;
2467
+
2468
+
switch (info->cmd) {
2469
+
case ETHTOOL_GRXRINGS:
2470
+
info->data = vi->curr_queue_pairs;
2471
+
break;
2472
+
case ETHTOOL_GRXFH:
2473
+
virtnet_get_hashflow(vi, info);
2474
+
break;
2475
+
default:
2476
+
rc = -EOPNOTSUPP;
2477
+
}
2478
+
2479
+
return rc;
2480
+
}
2481
+
2482
+
static int virtnet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info)
2483
+
{
2484
+
struct virtnet_info *vi = netdev_priv(dev);
2485
+
int rc = 0;
2486
+
2487
+
switch (info->cmd) {
2488
+
case ETHTOOL_SRXFH:
2489
+
if (!virtnet_set_hashflow(vi, info))
2490
+
rc = -EINVAL;
2491
+
2492
+
break;
2493
+
default:
2494
+
rc = -EOPNOTSUPP;
2495
+
}
2496
+
2497
+
return rc;
2498
+
}
2499
+
2641
2500
static const struct ethtool_ops virtnet_ethtool_ops = {
2642
2501
.supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES,
2643
2502
.get_drvinfo = virtnet_get_drvinfo,
···
2739
2426
.set_link_ksettings = virtnet_set_link_ksettings,
2740
2427
.set_coalesce = virtnet_set_coalesce,
2741
2428
.get_coalesce = virtnet_get_coalesce,
2429
+
.get_rxfh_key_size = virtnet_get_rxfh_key_size,
2430
+
.get_rxfh_indir_size = virtnet_get_rxfh_indir_size,
2431
+
.get_rxfh = virtnet_get_rxfh,
2432
+
.set_rxfh = virtnet_set_rxfh,
2433
+
.get_rxnfc = virtnet_get_rxnfc,
2434
+
.set_rxnfc = virtnet_set_rxnfc,
2742
2435
};
2743
2436
2744
2437
static void virtnet_freeze_down(struct virtio_device *vdev)
···
2997
2678
vi->guest_offloads = offloads;
2998
2679
}
2999
2680
2681
+
if ((dev->features ^ features) & NETIF_F_RXHASH) {
2682
+
if (features & NETIF_F_RXHASH)
2683
+
vi->ctrl->rss.hash_types = vi->rss_hash_types_saved;
2684
+
else
2685
+
vi->ctrl->rss.hash_types = VIRTIO_NET_HASH_REPORT_NONE;
2686
+
2687
+
if (!virtnet_commit_rss_command(vi))
2688
+
return -EINVAL;
2689
+
}
2690
+
3000
2691
return 0;
3001
2692
}
3002
2693
···
3180
2851
*/
3181
2852
static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq)
3182
2853
{
3183
-
const unsigned int hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2854
+
const unsigned int hdr_len = vi->hdr_len;
3184
2855
unsigned int rq_size = virtqueue_get_vring_size(vq);
3185
2856
unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu;
3186
2857
unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len;
···
3401
3072
"VIRTIO_NET_F_CTRL_VQ") ||
3402
3073
VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") ||
3403
3074
VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR,
3075
+
"VIRTIO_NET_F_CTRL_VQ") ||
3076
+
VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS,
3077
+
"VIRTIO_NET_F_CTRL_VQ") ||
3078
+
VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT,
3404
3079
"VIRTIO_NET_F_CTRL_VQ"))) {
3405
3080
return false;
3406
3081
}
···
3445
3112
u16 max_queue_pairs;
3446
3113
int mtu;
3447
3114
3448
-
/* Find if host supports multiqueue virtio_net device */
3449
-
err = virtio_cread_feature(vdev, VIRTIO_NET_F_MQ,
3450
-
struct virtio_net_config,
3451
-
max_virtqueue_pairs, &max_queue_pairs);
3115
+
/* Find if host supports multiqueue/rss virtio_net device */
3116
+
max_queue_pairs = 1;
3117
+
if (virtio_has_feature(vdev, VIRTIO_NET_F_MQ) || virtio_has_feature(vdev, VIRTIO_NET_F_RSS))
3118
+
max_queue_pairs =
3119
+
virtio_cread16(vdev, offsetof(struct virtio_net_config, max_virtqueue_pairs));
3452
3120
3453
3121
/* We need at least 2 queue's */
3454
-
if (err || max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
3122
+
if (max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN ||
3455
3123
max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX ||
3456
3124
!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ))
3457
3125
max_queue_pairs = 1;
···
3540
3206
if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF))
3541
3207
vi->mergeable_rx_bufs = true;
3542
3208
3543
-
if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
3544
-
virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
3209
+
if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT))
3210
+
vi->has_rss_hash_report = true;
3211
+
3212
+
if (virtio_has_feature(vdev, VIRTIO_NET_F_RSS))
3213
+
vi->has_rss = true;
3214
+
3215
+
if (vi->has_rss || vi->has_rss_hash_report) {
3216
+
vi->rss_indir_table_size =
3217
+
virtio_cread16(vdev, offsetof(struct virtio_net_config,
3218
+
rss_max_indirection_table_length));
3219
+
vi->rss_key_size =
3220
+
virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size));
3221
+
3222
+
vi->rss_hash_types_supported =
3223
+
virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types));
3224
+
vi->rss_hash_types_supported &=
3225
+
~(VIRTIO_NET_RSS_HASH_TYPE_IP_EX |
3226
+
VIRTIO_NET_RSS_HASH_TYPE_TCP_EX |
3227
+
VIRTIO_NET_RSS_HASH_TYPE_UDP_EX);
3228
+
3229
+
dev->hw_features |= NETIF_F_RXHASH;
3230
+
}
3231
+
3232
+
if (vi->has_rss_hash_report)
3233
+
vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash);
3234
+
else if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) ||
3235
+
virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
3545
3236
vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf);
3546
3237
else
3547
3238
vi->hdr_len = sizeof(struct virtio_net_hdr);
···
3632
3273
goto free_vqs;
3633
3274
}
3634
3275
}
3276
+
3277
+
if (vi->has_rss || vi->has_rss_hash_report)
3278
+
virtnet_init_default_rss(vi);
3635
3279
3636
3280
err = register_netdev(dev);
3637
3281
if (err) {
···
3767
3405
VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \
3768
3406
VIRTIO_NET_F_CTRL_MAC_ADDR, \
3769
3407
VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \
3770
-
VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY
3408
+
VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \
3409
+
VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT
3771
3410
3772
3411
static unsigned int features[] = {
3773
3412
VIRTNET_FEATURES,
+52
-88
drivers/vdpa/ifcvf/ifcvf_base.c
+52
-88
drivers/vdpa/ifcvf/ifcvf_base.c
···
10
10
11
11
#include "ifcvf_base.h"
12
12
13
-
static inline u8 ifc_ioread8(u8 __iomem *addr)
14
-
{
15
-
return ioread8(addr);
16
-
}
17
-
static inline u16 ifc_ioread16 (__le16 __iomem *addr)
18
-
{
19
-
return ioread16(addr);
20
-
}
21
-
22
-
static inline u32 ifc_ioread32(__le32 __iomem *addr)
23
-
{
24
-
return ioread32(addr);
25
-
}
26
-
27
-
static inline void ifc_iowrite8(u8 value, u8 __iomem *addr)
28
-
{
29
-
iowrite8(value, addr);
30
-
}
31
-
32
-
static inline void ifc_iowrite16(u16 value, __le16 __iomem *addr)
33
-
{
34
-
iowrite16(value, addr);
35
-
}
36
-
37
-
static inline void ifc_iowrite32(u32 value, __le32 __iomem *addr)
38
-
{
39
-
iowrite32(value, addr);
40
-
}
41
-
42
-
static void ifc_iowrite64_twopart(u64 val,
43
-
__le32 __iomem *lo, __le32 __iomem *hi)
44
-
{
45
-
ifc_iowrite32((u32)val, lo);
46
-
ifc_iowrite32(val >> 32, hi);
47
-
}
48
-
49
13
struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw)
50
14
{
51
15
return container_of(hw, struct ifcvf_adapter, vf);
16
+
}
17
+
18
+
u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector)
19
+
{
20
+
struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
21
+
22
+
vp_iowrite16(qid, &cfg->queue_select);
23
+
vp_iowrite16(vector, &cfg->queue_msix_vector);
24
+
25
+
return vp_ioread16(&cfg->queue_msix_vector);
26
+
}
27
+
28
+
u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector)
29
+
{
30
+
struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
31
+
32
+
cfg = hw->common_cfg;
33
+
vp_iowrite16(vector, &cfg->msix_config);
34
+
35
+
return vp_ioread16(&cfg->msix_config);
52
36
}
53
37
54
38
static void __iomem *get_cap_addr(struct ifcvf_hw *hw,
···
142
158
return -EIO;
143
159
}
144
160
145
-
hw->nr_vring = ifc_ioread16(&hw->common_cfg->num_queues);
161
+
hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues);
146
162
147
163
for (i = 0; i < hw->nr_vring; i++) {
148
-
ifc_iowrite16(i, &hw->common_cfg->queue_select);
149
-
notify_off = ifc_ioread16(&hw->common_cfg->queue_notify_off);
164
+
vp_iowrite16(i, &hw->common_cfg->queue_select);
165
+
notify_off = vp_ioread16(&hw->common_cfg->queue_notify_off);
150
166
hw->vring[i].notify_addr = hw->notify_base +
151
167
notify_off * hw->notify_off_multiplier;
152
168
hw->vring[i].notify_pa = hw->notify_base_pa +
153
169
notify_off * hw->notify_off_multiplier;
170
+
hw->vring[i].irq = -EINVAL;
154
171
}
155
172
156
173
hw->lm_cfg = hw->base[IFCVF_LM_BAR];
···
161
176
hw->common_cfg, hw->notify_base, hw->isr,
162
177
hw->dev_cfg, hw->notify_off_multiplier);
163
178
179
+
hw->vqs_reused_irq = -EINVAL;
180
+
hw->config_irq = -EINVAL;
181
+
164
182
return 0;
165
183
}
166
184
167
185
u8 ifcvf_get_status(struct ifcvf_hw *hw)
168
186
{
169
-
return ifc_ioread8(&hw->common_cfg->device_status);
187
+
return vp_ioread8(&hw->common_cfg->device_status);
170
188
}
171
189
172
190
void ifcvf_set_status(struct ifcvf_hw *hw, u8 status)
173
191
{
174
-
ifc_iowrite8(status, &hw->common_cfg->device_status);
192
+
vp_iowrite8(status, &hw->common_cfg->device_status);
175
193
}
176
194
177
195
void ifcvf_reset(struct ifcvf_hw *hw)
···
202
214
u32 features_lo, features_hi;
203
215
u64 features;
204
216
205
-
ifc_iowrite32(0, &cfg->device_feature_select);
206
-
features_lo = ifc_ioread32(&cfg->device_feature);
217
+
vp_iowrite32(0, &cfg->device_feature_select);
218
+
features_lo = vp_ioread32(&cfg->device_feature);
207
219
208
-
ifc_iowrite32(1, &cfg->device_feature_select);
209
-
features_hi = ifc_ioread32(&cfg->device_feature);
220
+
vp_iowrite32(1, &cfg->device_feature_select);
221
+
features_hi = vp_ioread32(&cfg->device_feature);
210
222
211
223
features = ((u64)features_hi << 32) | features_lo;
212
224
···
259
271
260
272
WARN_ON(offset + length > hw->config_size);
261
273
do {
262
-
old_gen = ifc_ioread8(&hw->common_cfg->config_generation);
274
+
old_gen = vp_ioread8(&hw->common_cfg->config_generation);
263
275
p = dst;
264
276
for (i = 0; i < length; i++)
265
-
*p++ = ifc_ioread8(hw->dev_cfg + offset + i);
277
+
*p++ = vp_ioread8(hw->dev_cfg + offset + i);
266
278
267
-
new_gen = ifc_ioread8(&hw->common_cfg->config_generation);
279
+
new_gen = vp_ioread8(&hw->common_cfg->config_generation);
268
280
} while (old_gen != new_gen);
269
281
}
270
282
···
277
289
p = src;
278
290
WARN_ON(offset + length > hw->config_size);
279
291
for (i = 0; i < length; i++)
280
-
ifc_iowrite8(*p++, hw->dev_cfg + offset + i);
292
+
vp_iowrite8(*p++, hw->dev_cfg + offset + i);
281
293
}
282
294
283
295
static void ifcvf_set_features(struct ifcvf_hw *hw, u64 features)
284
296
{
285
297
struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg;
286
298
287
-
ifc_iowrite32(0, &cfg->guest_feature_select);
288
-
ifc_iowrite32((u32)features, &cfg->guest_feature);
299
+
vp_iowrite32(0, &cfg->guest_feature_select);
300
+
vp_iowrite32((u32)features, &cfg->guest_feature);
289
301
290
-
ifc_iowrite32(1, &cfg->guest_feature_select);
291
-
ifc_iowrite32(features >> 32, &cfg->guest_feature);
302
+
vp_iowrite32(1, &cfg->guest_feature_select);
303
+
vp_iowrite32(features >> 32, &cfg->guest_feature);
292
304
}
293
305
294
306
static int ifcvf_config_features(struct ifcvf_hw *hw)
···
317
329
ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg;
318
330
q_pair_id = qid / hw->nr_vring;
319
331
avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2];
320
-
last_avail_idx = ifc_ioread16(avail_idx_addr);
332
+
last_avail_idx = vp_ioread16(avail_idx_addr);
321
333
322
334
return last_avail_idx;
323
335
}
···
332
344
q_pair_id = qid / hw->nr_vring;
333
345
avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2];
334
346
hw->vring[qid].last_avail_idx = num;
335
-
ifc_iowrite16(num, avail_idx_addr);
347
+
vp_iowrite16(num, avail_idx_addr);
336
348
337
349
return 0;
338
350
}
···
340
352
static int ifcvf_hw_enable(struct ifcvf_hw *hw)
341
353
{
342
354
struct virtio_pci_common_cfg __iomem *cfg;
343
-
struct ifcvf_adapter *ifcvf;
344
355
u32 i;
345
356
346
-
ifcvf = vf_to_adapter(hw);
347
357
cfg = hw->common_cfg;
348
-
ifc_iowrite16(IFCVF_MSI_CONFIG_OFF, &cfg->msix_config);
349
-
350
-
if (ifc_ioread16(&cfg->msix_config) == VIRTIO_MSI_NO_VECTOR) {
351
-
IFCVF_ERR(ifcvf->pdev, "No msix vector for device config\n");
352
-
return -EINVAL;
353
-
}
354
-
355
358
for (i = 0; i < hw->nr_vring; i++) {
356
359
if (!hw->vring[i].ready)
357
360
break;
358
361
359
-
ifc_iowrite16(i, &cfg->queue_select);
360
-
ifc_iowrite64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
362
+
vp_iowrite16(i, &cfg->queue_select);
363
+
vp_iowrite64_twopart(hw->vring[i].desc, &cfg->queue_desc_lo,
361
364
&cfg->queue_desc_hi);
362
-
ifc_iowrite64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
365
+
vp_iowrite64_twopart(hw->vring[i].avail, &cfg->queue_avail_lo,
363
366
&cfg->queue_avail_hi);
364
-
ifc_iowrite64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
367
+
vp_iowrite64_twopart(hw->vring[i].used, &cfg->queue_used_lo,
365
368
&cfg->queue_used_hi);
366
-
ifc_iowrite16(hw->vring[i].size, &cfg->queue_size);
367
-
ifc_iowrite16(i + IFCVF_MSI_QUEUE_OFF, &cfg->queue_msix_vector);
368
-
369
-
if (ifc_ioread16(&cfg->queue_msix_vector) ==
370
-
VIRTIO_MSI_NO_VECTOR) {
371
-
IFCVF_ERR(ifcvf->pdev,
372
-
"No msix vector for queue %u\n", i);
373
-
return -EINVAL;
374
-
}
375
-
369
+
vp_iowrite16(hw->vring[i].size, &cfg->queue_size);
376
370
ifcvf_set_vq_state(hw, i, hw->vring[i].last_avail_idx);
377
-
ifc_iowrite16(1, &cfg->queue_enable);
371
+
vp_iowrite16(1, &cfg->queue_enable);
378
372
}
379
373
380
374
return 0;
···
364
394
365
395
static void ifcvf_hw_disable(struct ifcvf_hw *hw)
366
396
{
367
-
struct virtio_pci_common_cfg __iomem *cfg;
368
397
u32 i;
369
398
370
-
cfg = hw->common_cfg;
371
-
ifc_iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->msix_config);
372
-
399
+
ifcvf_set_config_vector(hw, VIRTIO_MSI_NO_VECTOR);
373
400
for (i = 0; i < hw->nr_vring; i++) {
374
-
ifc_iowrite16(i, &cfg->queue_select);
375
-
ifc_iowrite16(VIRTIO_MSI_NO_VECTOR, &cfg->queue_msix_vector);
401
+
ifcvf_set_vq_vector(hw, i, VIRTIO_MSI_NO_VECTOR);
376
402
}
377
-
378
-
ifc_ioread16(&cfg->queue_msix_vector);
379
403
}
380
404
381
405
int ifcvf_start_hw(struct ifcvf_hw *hw)
···
397
433
398
434
void ifcvf_notify_queue(struct ifcvf_hw *hw, u16 qid)
399
435
{
400
-
ifc_iowrite16(qid, hw->vring[qid].notify_addr);
436
+
vp_iowrite16(qid, hw->vring[qid].notify_addr);
401
437
}
+17
-7
drivers/vdpa/ifcvf/ifcvf_base.h
+17
-7
drivers/vdpa/ifcvf/ifcvf_base.h
···
14
14
#include <linux/pci.h>
15
15
#include <linux/pci_regs.h>
16
16
#include <linux/vdpa.h>
17
+
#include <linux/virtio_pci_modern.h>
17
18
#include <uapi/linux/virtio_net.h>
18
19
#include <uapi/linux/virtio_blk.h>
19
20
#include <uapi/linux/virtio_config.h>
···
28
27
29
28
#define IFCVF_QUEUE_ALIGNMENT PAGE_SIZE
30
29
#define IFCVF_QUEUE_MAX 32768
31
-
#define IFCVF_MSI_CONFIG_OFF 0
32
-
#define IFCVF_MSI_QUEUE_OFF 1
33
30
#define IFCVF_PCI_MAX_RESOURCE 6
34
31
35
32
#define IFCVF_LM_CFG_SIZE 0x40
···
40
41
41
42
#define ifcvf_private_to_vf(adapter) \
42
43
(&((struct ifcvf_adapter *)adapter)->vf)
44
+
45
+
/* all vqs and config interrupt has its own vector */
46
+
#define MSIX_VECTOR_PER_VQ_AND_CONFIG 1
47
+
/* all vqs share a vector, and config interrupt has a separate vector */
48
+
#define MSIX_VECTOR_SHARED_VQ_AND_CONFIG 2
49
+
/* all vqs and config interrupt share a vector */
50
+
#define MSIX_VECTOR_DEV_SHARED 3
43
51
44
52
struct vring_info {
45
53
u64 desc;
···
66
60
u8 __iomem *isr;
67
61
/* Live migration */
68
62
u8 __iomem *lm_cfg;
69
-
u16 nr_vring;
70
63
/* Notification bar number */
71
64
u8 notify_bar;
65
+
u8 msix_vector_status;
66
+
/* virtio-net or virtio-blk device config size */
67
+
u32 config_size;
72
68
/* Notificaiton bar address */
73
69
void __iomem *notify_base;
74
70
phys_addr_t notify_base_pa;
75
71
u32 notify_off_multiplier;
72
+
u32 dev_type;
76
73
u64 req_features;
77
74
u64 hw_features;
78
-
u32 dev_type;
79
75
struct virtio_pci_common_cfg __iomem *common_cfg;
80
76
void __iomem *dev_cfg;
81
77
struct vring_info vring[IFCVF_MAX_QUEUES];
82
78
void __iomem * const *base;
83
79
char config_msix_name[256];
84
80
struct vdpa_callback config_cb;
85
-
unsigned int config_irq;
86
-
/* virtio-net or virtio-blk device config size */
87
-
u32 config_size;
81
+
int config_irq;
82
+
int vqs_reused_irq;
83
+
u16 nr_vring;
88
84
};
89
85
90
86
struct ifcvf_adapter {
···
131
123
struct ifcvf_adapter *vf_to_adapter(struct ifcvf_hw *hw);
132
124
int ifcvf_probed_virtio_net(struct ifcvf_hw *hw);
133
125
u32 ifcvf_get_config_size(struct ifcvf_hw *hw);
126
+
u16 ifcvf_set_vq_vector(struct ifcvf_hw *hw, u16 qid, int vector);
127
+
u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector);
134
128
#endif /* _IFCVF_H_ */
+287
-36
drivers/vdpa/ifcvf/ifcvf_main.c
+287
-36
drivers/vdpa/ifcvf/ifcvf_main.c
···
27
27
return IRQ_HANDLED;
28
28
}
29
29
30
-
static irqreturn_t ifcvf_intr_handler(int irq, void *arg)
30
+
static irqreturn_t ifcvf_vq_intr_handler(int irq, void *arg)
31
31
{
32
32
struct vring_info *vring = arg;
33
33
···
37
37
return IRQ_HANDLED;
38
38
}
39
39
40
+
static irqreturn_t ifcvf_vqs_reused_intr_handler(int irq, void *arg)
41
+
{
42
+
struct ifcvf_hw *vf = arg;
43
+
struct vring_info *vring;
44
+
int i;
45
+
46
+
for (i = 0; i < vf->nr_vring; i++) {
47
+
vring = &vf->vring[i];
48
+
if (vring->cb.callback)
49
+
vring->cb.callback(vring->cb.private);
50
+
}
51
+
52
+
return IRQ_HANDLED;
53
+
}
54
+
55
+
static irqreturn_t ifcvf_dev_intr_handler(int irq, void *arg)
56
+
{
57
+
struct ifcvf_hw *vf = arg;
58
+
u8 isr;
59
+
60
+
isr = vp_ioread8(vf->isr);
61
+
if (isr & VIRTIO_PCI_ISR_CONFIG)
62
+
ifcvf_config_changed(irq, arg);
63
+
64
+
return ifcvf_vqs_reused_intr_handler(irq, arg);
65
+
}
66
+
40
67
static void ifcvf_free_irq_vectors(void *data)
41
68
{
42
69
pci_free_irq_vectors(data);
43
70
}
44
71
45
-
static void ifcvf_free_irq(struct ifcvf_adapter *adapter, int queues)
72
+
static void ifcvf_free_per_vq_irq(struct ifcvf_adapter *adapter)
46
73
{
47
74
struct pci_dev *pdev = adapter->pdev;
48
75
struct ifcvf_hw *vf = &adapter->vf;
49
76
int i;
50
77
51
-
52
-
for (i = 0; i < queues; i++) {
53
-
devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);
54
-
vf->vring[i].irq = -EINVAL;
78
+
for (i = 0; i < vf->nr_vring; i++) {
79
+
if (vf->vring[i].irq != -EINVAL) {
80
+
devm_free_irq(&pdev->dev, vf->vring[i].irq, &vf->vring[i]);
81
+
vf->vring[i].irq = -EINVAL;
82
+
}
55
83
}
56
-
57
-
devm_free_irq(&pdev->dev, vf->config_irq, vf);
58
-
ifcvf_free_irq_vectors(pdev);
59
84
}
60
85
61
-
static int ifcvf_request_irq(struct ifcvf_adapter *adapter)
86
+
static void ifcvf_free_vqs_reused_irq(struct ifcvf_adapter *adapter)
62
87
{
63
88
struct pci_dev *pdev = adapter->pdev;
64
89
struct ifcvf_hw *vf = &adapter->vf;
65
-
int vector, i, ret, irq;
66
-
u16 max_intr;
90
+
91
+
if (vf->vqs_reused_irq != -EINVAL) {
92
+
devm_free_irq(&pdev->dev, vf->vqs_reused_irq, vf);
93
+
vf->vqs_reused_irq = -EINVAL;
94
+
}
95
+
96
+
}
97
+
98
+
static void ifcvf_free_vq_irq(struct ifcvf_adapter *adapter)
99
+
{
100
+
struct ifcvf_hw *vf = &adapter->vf;
101
+
102
+
if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)
103
+
ifcvf_free_per_vq_irq(adapter);
104
+
else
105
+
ifcvf_free_vqs_reused_irq(adapter);
106
+
}
107
+
108
+
static void ifcvf_free_config_irq(struct ifcvf_adapter *adapter)
109
+
{
110
+
struct pci_dev *pdev = adapter->pdev;
111
+
struct ifcvf_hw *vf = &adapter->vf;
112
+
113
+
if (vf->config_irq == -EINVAL)
114
+
return;
115
+
116
+
/* If the irq is shared by all vqs and the config interrupt,
117
+
* it is already freed in ifcvf_free_vq_irq, so here only
118
+
* need to free config irq when msix_vector_status != MSIX_VECTOR_DEV_SHARED
119
+
*/
120
+
if (vf->msix_vector_status != MSIX_VECTOR_DEV_SHARED) {
121
+
devm_free_irq(&pdev->dev, vf->config_irq, vf);
122
+
vf->config_irq = -EINVAL;
123
+
}
124
+
}
125
+
126
+
static void ifcvf_free_irq(struct ifcvf_adapter *adapter)
127
+
{
128
+
struct pci_dev *pdev = adapter->pdev;
129
+
130
+
ifcvf_free_vq_irq(adapter);
131
+
ifcvf_free_config_irq(adapter);
132
+
ifcvf_free_irq_vectors(pdev);
133
+
}
134
+
135
+
/* ifcvf MSIX vectors allocator, this helper tries to allocate
136
+
* vectors for all virtqueues and the config interrupt.
137
+
* It returns the number of allocated vectors, negative
138
+
* return value when fails.
139
+
*/
140
+
static int ifcvf_alloc_vectors(struct ifcvf_adapter *adapter)
141
+
{
142
+
struct pci_dev *pdev = adapter->pdev;
143
+
struct ifcvf_hw *vf = &adapter->vf;
144
+
int max_intr, ret;
67
145
68
146
/* all queues and config interrupt */
69
147
max_intr = vf->nr_vring + 1;
148
+
ret = pci_alloc_irq_vectors(pdev, 1, max_intr, PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
70
149
71
-
ret = pci_alloc_irq_vectors(pdev, max_intr,
72
-
max_intr, PCI_IRQ_MSIX);
73
150
if (ret < 0) {
74
151
IFCVF_ERR(pdev, "Failed to alloc IRQ vectors\n");
75
152
return ret;
76
153
}
77
154
155
+
if (ret < max_intr)
156
+
IFCVF_INFO(pdev,
157
+
"Requested %u vectors, however only %u allocated, lower performance\n",
158
+
max_intr, ret);
159
+
160
+
return ret;
161
+
}
162
+
163
+
static int ifcvf_request_per_vq_irq(struct ifcvf_adapter *adapter)
164
+
{
165
+
struct pci_dev *pdev = adapter->pdev;
166
+
struct ifcvf_hw *vf = &adapter->vf;
167
+
int i, vector, ret, irq;
168
+
169
+
vf->vqs_reused_irq = -EINVAL;
170
+
for (i = 0; i < vf->nr_vring; i++) {
171
+
snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n", pci_name(pdev), i);
172
+
vector = i;
173
+
irq = pci_irq_vector(pdev, vector);
174
+
ret = devm_request_irq(&pdev->dev, irq,
175
+
ifcvf_vq_intr_handler, 0,
176
+
vf->vring[i].msix_name,
177
+
&vf->vring[i]);
178
+
if (ret) {
179
+
IFCVF_ERR(pdev, "Failed to request irq for vq %d\n", i);
180
+
goto err;
181
+
}
182
+
183
+
vf->vring[i].irq = irq;
184
+
ret = ifcvf_set_vq_vector(vf, i, vector);
185
+
if (ret == VIRTIO_MSI_NO_VECTOR) {
186
+
IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);
187
+
goto err;
188
+
}
189
+
}
190
+
191
+
return 0;
192
+
err:
193
+
ifcvf_free_irq(adapter);
194
+
195
+
return -EFAULT;
196
+
}
197
+
198
+
static int ifcvf_request_vqs_reused_irq(struct ifcvf_adapter *adapter)
199
+
{
200
+
struct pci_dev *pdev = adapter->pdev;
201
+
struct ifcvf_hw *vf = &adapter->vf;
202
+
int i, vector, ret, irq;
203
+
204
+
vector = 0;
205
+
snprintf(vf->vring[0].msix_name, 256, "ifcvf[%s]-vqs-reused-irq\n", pci_name(pdev));
206
+
irq = pci_irq_vector(pdev, vector);
207
+
ret = devm_request_irq(&pdev->dev, irq,
208
+
ifcvf_vqs_reused_intr_handler, 0,
209
+
vf->vring[0].msix_name, vf);
210
+
if (ret) {
211
+
IFCVF_ERR(pdev, "Failed to request reused irq for the device\n");
212
+
goto err;
213
+
}
214
+
215
+
vf->vqs_reused_irq = irq;
216
+
for (i = 0; i < vf->nr_vring; i++) {
217
+
vf->vring[i].irq = -EINVAL;
218
+
ret = ifcvf_set_vq_vector(vf, i, vector);
219
+
if (ret == VIRTIO_MSI_NO_VECTOR) {
220
+
IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);
221
+
goto err;
222
+
}
223
+
}
224
+
225
+
return 0;
226
+
err:
227
+
ifcvf_free_irq(adapter);
228
+
229
+
return -EFAULT;
230
+
}
231
+
232
+
static int ifcvf_request_dev_irq(struct ifcvf_adapter *adapter)
233
+
{
234
+
struct pci_dev *pdev = adapter->pdev;
235
+
struct ifcvf_hw *vf = &adapter->vf;
236
+
int i, vector, ret, irq;
237
+
238
+
vector = 0;
239
+
snprintf(vf->vring[0].msix_name, 256, "ifcvf[%s]-dev-irq\n", pci_name(pdev));
240
+
irq = pci_irq_vector(pdev, vector);
241
+
ret = devm_request_irq(&pdev->dev, irq,
242
+
ifcvf_dev_intr_handler, 0,
243
+
vf->vring[0].msix_name, vf);
244
+
if (ret) {
245
+
IFCVF_ERR(pdev, "Failed to request irq for the device\n");
246
+
goto err;
247
+
}
248
+
249
+
vf->vqs_reused_irq = irq;
250
+
for (i = 0; i < vf->nr_vring; i++) {
251
+
vf->vring[i].irq = -EINVAL;
252
+
ret = ifcvf_set_vq_vector(vf, i, vector);
253
+
if (ret == VIRTIO_MSI_NO_VECTOR) {
254
+
IFCVF_ERR(pdev, "No msix vector for vq %u\n", i);
255
+
goto err;
256
+
}
257
+
}
258
+
259
+
vf->config_irq = irq;
260
+
ret = ifcvf_set_config_vector(vf, vector);
261
+
if (ret == VIRTIO_MSI_NO_VECTOR) {
262
+
IFCVF_ERR(pdev, "No msix vector for device config\n");
263
+
goto err;
264
+
}
265
+
266
+
return 0;
267
+
err:
268
+
ifcvf_free_irq(adapter);
269
+
270
+
return -EFAULT;
271
+
272
+
}
273
+
274
+
static int ifcvf_request_vq_irq(struct ifcvf_adapter *adapter)
275
+
{
276
+
struct ifcvf_hw *vf = &adapter->vf;
277
+
int ret;
278
+
279
+
if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)
280
+
ret = ifcvf_request_per_vq_irq(adapter);
281
+
else
282
+
ret = ifcvf_request_vqs_reused_irq(adapter);
283
+
284
+
return ret;
285
+
}
286
+
287
+
static int ifcvf_request_config_irq(struct ifcvf_adapter *adapter)
288
+
{
289
+
struct pci_dev *pdev = adapter->pdev;
290
+
struct ifcvf_hw *vf = &adapter->vf;
291
+
int config_vector, ret;
292
+
293
+
if (vf->msix_vector_status == MSIX_VECTOR_DEV_SHARED)
294
+
return 0;
295
+
296
+
if (vf->msix_vector_status == MSIX_VECTOR_PER_VQ_AND_CONFIG)
297
+
/* vector 0 ~ vf->nr_vring for vqs, num vf->nr_vring vector for config interrupt */
298
+
config_vector = vf->nr_vring;
299
+
300
+
if (vf->msix_vector_status == MSIX_VECTOR_SHARED_VQ_AND_CONFIG)
301
+
/* vector 0 for vqs and 1 for config interrupt */
302
+
config_vector = 1;
303
+
78
304
snprintf(vf->config_msix_name, 256, "ifcvf[%s]-config\n",
79
305
pci_name(pdev));
80
-
vector = 0;
81
-
vf->config_irq = pci_irq_vector(pdev, vector);
306
+
vf->config_irq = pci_irq_vector(pdev, config_vector);
82
307
ret = devm_request_irq(&pdev->dev, vf->config_irq,
83
308
ifcvf_config_changed, 0,
84
309
vf->config_msix_name, vf);
85
310
if (ret) {
86
311
IFCVF_ERR(pdev, "Failed to request config irq\n");
312
+
goto err;
313
+
}
314
+
315
+
ret = ifcvf_set_config_vector(vf, config_vector);
316
+
if (ret == VIRTIO_MSI_NO_VECTOR) {
317
+
IFCVF_ERR(pdev, "No msix vector for device config\n");
318
+
goto err;
319
+
}
320
+
321
+
return 0;
322
+
err:
323
+
ifcvf_free_irq(adapter);
324
+
325
+
return -EFAULT;
326
+
}
327
+
328
+
static int ifcvf_request_irq(struct ifcvf_adapter *adapter)
329
+
{
330
+
struct ifcvf_hw *vf = &adapter->vf;
331
+
int nvectors, ret, max_intr;
332
+
333
+
nvectors = ifcvf_alloc_vectors(adapter);
334
+
if (nvectors <= 0)
335
+
return -EFAULT;
336
+
337
+
vf->msix_vector_status = MSIX_VECTOR_PER_VQ_AND_CONFIG;
338
+
max_intr = vf->nr_vring + 1;
339
+
if (nvectors < max_intr)
340
+
vf->msix_vector_status = MSIX_VECTOR_SHARED_VQ_AND_CONFIG;
341
+
342
+
if (nvectors == 1) {
343
+
vf->msix_vector_status = MSIX_VECTOR_DEV_SHARED;
344
+
ret = ifcvf_request_dev_irq(adapter);
345
+
87
346
return ret;
88
347
}
89
348
90
-
for (i = 0; i < vf->nr_vring; i++) {
91
-
snprintf(vf->vring[i].msix_name, 256, "ifcvf[%s]-%d\n",
92
-
pci_name(pdev), i);
93
-
vector = i + IFCVF_MSI_QUEUE_OFF;
94
-
irq = pci_irq_vector(pdev, vector);
95
-
ret = devm_request_irq(&pdev->dev, irq,
96
-
ifcvf_intr_handler, 0,
97
-
vf->vring[i].msix_name,
98
-
&vf->vring[i]);
99
-
if (ret) {
100
-
IFCVF_ERR(pdev,
101
-
"Failed to request irq for vq %d\n", i);
102
-
ifcvf_free_irq(adapter, i);
349
+
ret = ifcvf_request_vq_irq(adapter);
350
+
if (ret)
351
+
return ret;
103
352
104
-
return ret;
105
-
}
353
+
ret = ifcvf_request_config_irq(adapter);
106
354
107
-
vf->vring[i].irq = irq;
108
-
}
355
+
if (ret)
356
+
return ret;
109
357
110
358
return 0;
111
359
}
···
511
263
512
264
if (status_old & VIRTIO_CONFIG_S_DRIVER_OK) {
513
265
ifcvf_stop_datapath(adapter);
514
-
ifcvf_free_irq(adapter, vf->nr_vring);
266
+
ifcvf_free_irq(adapter);
515
267
}
516
268
517
269
ifcvf_reset_vring(adapter);
···
596
348
{
597
349
struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
598
350
599
-
return ioread8(&vf->common_cfg->config_generation);
351
+
return vp_ioread8(&vf->common_cfg->config_generation);
600
352
}
601
353
602
354
static u32 ifcvf_vdpa_get_device_id(struct vdpa_device *vdpa_dev)
···
658
410
{
659
411
struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
660
412
661
-
return vf->vring[qid].irq;
413
+
if (vf->vqs_reused_irq < 0)
414
+
return vf->vring[qid].irq;
415
+
else
416
+
return -EINVAL;
662
417
}
663
418
664
419
static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
+80
-4
drivers/vdpa/mlx5/net/mlx5_vnet.c
+80
-4
drivers/vdpa/mlx5/net/mlx5_vnet.c
···
1475
1475
virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
1476
1476
struct mlx5_core_dev *pfmdev;
1477
1477
size_t read;
1478
-
u8 mac[ETH_ALEN];
1478
+
u8 mac[ETH_ALEN], mac_back[ETH_ALEN];
1479
1479
1480
1480
pfmdev = pci_get_drvdata(pci_physfn(mvdev->mdev->pdev));
1481
1481
switch (cmd) {
···
1488
1488
status = VIRTIO_NET_OK;
1489
1489
break;
1490
1490
}
1491
+
1492
+
if (is_zero_ether_addr(mac))
1493
+
break;
1491
1494
1492
1495
if (!is_zero_ether_addr(ndev->config.mac)) {
1493
1496
if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) {
···
1506
1503
break;
1507
1504
}
1508
1505
1506
+
/* backup the original mac address so that if failed to add the forward rules
1507
+
* we could restore it
1508
+
*/
1509
+
memcpy(mac_back, ndev->config.mac, ETH_ALEN);
1510
+
1509
1511
memcpy(ndev->config.mac, mac, ETH_ALEN);
1512
+
1513
+
/* Need recreate the flow table entry, so that the packet could forward back
1514
+
*/
1515
+
remove_fwd_to_tir(ndev);
1516
+
1517
+
if (add_fwd_to_tir(ndev)) {
1518
+
mlx5_vdpa_warn(mvdev, "failed to insert forward rules, try to restore\n");
1519
+
1520
+
/* Although it hardly run here, we still need double check */
1521
+
if (is_zero_ether_addr(mac_back)) {
1522
+
mlx5_vdpa_warn(mvdev, "restore mac failed: Original MAC is zero\n");
1523
+
break;
1524
+
}
1525
+
1526
+
/* Try to restore original mac address to MFPS table, and try to restore
1527
+
* the forward rule entry.
1528
+
*/
1529
+
if (mlx5_mpfs_del_mac(pfmdev, ndev->config.mac)) {
1530
+
mlx5_vdpa_warn(mvdev, "restore mac failed: delete MAC %pM from MPFS table failed\n",
1531
+
ndev->config.mac);
1532
+
}
1533
+
1534
+
if (mlx5_mpfs_add_mac(pfmdev, mac_back)) {
1535
+
mlx5_vdpa_warn(mvdev, "restore mac failed: insert old MAC %pM into MPFS table failed\n",
1536
+
mac_back);
1537
+
}
1538
+
1539
+
memcpy(ndev->config.mac, mac_back, ETH_ALEN);
1540
+
1541
+
if (add_fwd_to_tir(ndev))
1542
+
mlx5_vdpa_warn(mvdev, "restore forward rules failed: insert forward rules failed\n");
1543
+
1544
+
break;
1545
+
}
1546
+
1510
1547
status = VIRTIO_NET_OK;
1511
1548
break;
1512
1549
···
1712
1669
return;
1713
1670
1714
1671
if (unlikely(is_ctrl_vq_idx(mvdev, idx))) {
1715
-
if (!mvdev->cvq.ready)
1672
+
if (!mvdev->wq || !mvdev->cvq.ready)
1716
1673
return;
1717
1674
1718
1675
wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC);
···
2608
2565
return ret;
2609
2566
}
2610
2567
2568
+
static int config_func_mtu(struct mlx5_core_dev *mdev, u16 mtu)
2569
+
{
2570
+
int inlen = MLX5_ST_SZ_BYTES(modify_nic_vport_context_in);
2571
+
void *in;
2572
+
int err;
2573
+
2574
+
in = kvzalloc(inlen, GFP_KERNEL);
2575
+
if (!in)
2576
+
return -ENOMEM;
2577
+
2578
+
MLX5_SET(modify_nic_vport_context_in, in, field_select.mtu, 1);
2579
+
MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.mtu,
2580
+
mtu + MLX5V_ETH_HARD_MTU);
2581
+
MLX5_SET(modify_nic_vport_context_in, in, opcode,
2582
+
MLX5_CMD_OP_MODIFY_NIC_VPORT_CONTEXT);
2583
+
2584
+
err = mlx5_cmd_exec_in(mdev, modify_nic_vport_context, in);
2585
+
2586
+
kvfree(in);
2587
+
return err;
2588
+
}
2589
+
2611
2590
static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name,
2612
2591
const struct vdpa_dev_set_config *add_config)
2613
2592
{
···
2689
2624
init_mvqs(ndev);
2690
2625
mutex_init(&ndev->reslock);
2691
2626
config = &ndev->config;
2627
+
2628
+
if (add_config->mask & BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU)) {
2629
+
err = config_func_mtu(mdev, add_config->net.mtu);
2630
+
if (err)
2631
+
goto err_mtu;
2632
+
}
2633
+
2692
2634
err = query_mtu(mdev, &mtu);
2693
2635
if (err)
2694
2636
goto err_mtu;
···
2779
2707
struct mlx5_vdpa_mgmtdev *mgtdev = container_of(v_mdev, struct mlx5_vdpa_mgmtdev, mgtdev);
2780
2708
struct mlx5_vdpa_dev *mvdev = to_mvdev(dev);
2781
2709
struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev);
2710
+
struct workqueue_struct *wq;
2782
2711
2783
2712
mlx5_notifier_unregister(mvdev->mdev, &ndev->nb);
2784
-
destroy_workqueue(mvdev->wq);
2713
+
wq = mvdev->wq;
2714
+
mvdev->wq = NULL;
2715
+
destroy_workqueue(wq);
2785
2716
_vdpa_unregister_device(dev);
2786
2717
mgtdev->ndev = NULL;
2787
2718
}
···
2816
2741
mgtdev->mgtdev.device = mdev->device;
2817
2742
mgtdev->mgtdev.id_table = id_table;
2818
2743
mgtdev->mgtdev.config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR) |
2819
-
BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP);
2744
+
BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP) |
2745
+
BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MTU);
2820
2746
mgtdev->mgtdev.max_supported_vqs =
2821
2747
MLX5_CAP_DEV_VDPA_EMULATION(mdev, max_num_virtio_queues) + 1;
2822
2748
mgtdev->mgtdev.supported_features = get_supported_features(mdev);
+3
-3
drivers/vdpa/vdpa.c
+3
-3
drivers/vdpa/vdpa.c
···
232
232
return (strcmp(dev_name(&vdev->dev), data) == 0);
233
233
}
234
234
235
-
static int __vdpa_register_device(struct vdpa_device *vdev, int nvqs)
235
+
static int __vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
236
236
{
237
237
struct device *dev;
238
238
···
257
257
*
258
258
* Return: Returns an error when fail to add device to vDPA bus
259
259
*/
260
-
int _vdpa_register_device(struct vdpa_device *vdev, int nvqs)
260
+
int _vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
261
261
{
262
262
if (!vdev->mdev)
263
263
return -EINVAL;
···
274
274
*
275
275
* Return: Returns an error when fail to add to vDPA bus
276
276
*/
277
-
int vdpa_register_device(struct vdpa_device *vdev, int nvqs)
277
+
int vdpa_register_device(struct vdpa_device *vdev, u32 nvqs)
278
278
{
279
279
int err;
280
280
+5
-1
drivers/vhost/iotlb.c
+5
-1
drivers/vhost/iotlb.c
···
62
62
*/
63
63
if (start == 0 && last == ULONG_MAX) {
64
64
u64 mid = last / 2;
65
+
int err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr,
66
+
perm, opaque);
65
67
66
-
vhost_iotlb_add_range_ctx(iotlb, start, mid, addr, perm, opaque);
68
+
if (err)
69
+
return err;
70
+
67
71
addr += mid + 1;
68
72
start = mid + 1;
69
73
}
+40
-5
drivers/vhost/vdpa.c
+40
-5
drivers/vhost/vdpa.c
···
42
42
struct device dev;
43
43
struct cdev cdev;
44
44
atomic_t opened;
45
-
int nvqs;
45
+
u32 nvqs;
46
46
int virtio_id;
47
47
int minor;
48
48
struct eventfd_ctx *config_ctx;
···
97
97
return;
98
98
99
99
irq = ops->get_vq_irq(vdpa, qid);
100
+
if (irq < 0)
101
+
return;
102
+
100
103
irq_bypass_unregister_producer(&vq->call_ctx.producer);
101
-
if (!vq->call_ctx.ctx || irq < 0)
104
+
if (!vq->call_ctx.ctx)
102
105
return;
103
106
104
107
vq->call_ctx.producer.token = vq->call_ctx.ctx;
···
161
158
struct vdpa_device *vdpa = v->vdpa;
162
159
const struct vdpa_config_ops *ops = vdpa->config;
163
160
u8 status, status_old;
164
-
int ret, nvqs = v->nvqs;
161
+
u32 nvqs = v->nvqs;
162
+
int ret;
165
163
u16 i;
166
164
167
165
if (copy_from_user(&status, statusp, sizeof(status)))
···
359
355
return 0;
360
356
}
361
357
358
+
static long vhost_vdpa_get_config_size(struct vhost_vdpa *v, u32 __user *argp)
359
+
{
360
+
struct vdpa_device *vdpa = v->vdpa;
361
+
const struct vdpa_config_ops *ops = vdpa->config;
362
+
u32 size;
363
+
364
+
size = ops->get_config_size(vdpa);
365
+
366
+
if (copy_to_user(argp, &size, sizeof(size)))
367
+
return -EFAULT;
368
+
369
+
return 0;
370
+
}
371
+
372
+
static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp)
373
+
{
374
+
struct vdpa_device *vdpa = v->vdpa;
375
+
376
+
if (copy_to_user(argp, &vdpa->nvqs, sizeof(vdpa->nvqs)))
377
+
return -EFAULT;
378
+
379
+
return 0;
380
+
}
381
+
362
382
static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
363
383
void __user *argp)
364
384
{
···
519
491
break;
520
492
case VHOST_VDPA_GET_IOVA_RANGE:
521
493
r = vhost_vdpa_get_iova_range(v, argp);
494
+
break;
495
+
case VHOST_VDPA_GET_CONFIG_SIZE:
496
+
r = vhost_vdpa_get_config_size(v, argp);
497
+
break;
498
+
case VHOST_VDPA_GET_VQS_COUNT:
499
+
r = vhost_vdpa_get_vqs_count(v, argp);
522
500
break;
523
501
default:
524
502
r = vhost_dev_ioctl(&v->vdev, cmd, argp);
···
982
948
struct vhost_vdpa *v;
983
949
struct vhost_dev *dev;
984
950
struct vhost_virtqueue **vqs;
985
-
int nvqs, i, r, opened;
951
+
int r, opened;
952
+
u32 i, nvqs;
986
953
987
954
v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);
988
955
···
1036
1001
1037
1002
static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)
1038
1003
{
1039
-
int i;
1004
+
u32 i;
1040
1005
1041
1006
for (i = 0; i < v->nvqs; i++)
1042
1007
vhost_vdpa_unsetup_vq_irq(v, i);
+2
-1
drivers/vhost/vhost.c
+2
-1
drivers/vhost/vhost.c
···
2550
2550
&vq->avail->idx, r);
2551
2551
return false;
2552
2552
}
2553
+
vq->avail_idx = vhost16_to_cpu(vq, avail_idx);
2553
2554
2554
-
return vhost16_to_cpu(vq, avail_idx) != vq->avail_idx;
2555
+
return vq->avail_idx != vq->last_avail_idx;
2555
2556
}
2556
2557
EXPORT_SYMBOL_GPL(vhost_enable_notify);
2557
2558
+4
-3
drivers/virtio/Kconfig
+4
-3
drivers/virtio/Kconfig
···
105
105
106
106
config VIRTIO_MEM
107
107
tristate "Virtio mem driver"
108
-
depends on X86_64
108
+
depends on X86_64 || ARM64
109
109
depends on VIRTIO
110
110
depends on MEMORY_HOTPLUG
111
111
depends on MEMORY_HOTREMOVE
···
115
115
This driver provides access to virtio-mem paravirtualized memory
116
116
devices, allowing to hotplug and hotunplug memory.
117
117
118
-
This driver was only tested under x86-64, but should theoretically
119
-
work on all architectures that support memory hotplug and hotremove.
118
+
This driver was only tested under x86-64 and arm64, but should
119
+
theoretically work on all architectures that support memory hotplug
120
+
and hotremove.
120
121
121
122
If unsure, say M.
122
123
+3
-2
drivers/virtio/virtio.c
+3
-2
drivers/virtio/virtio.c
···
526
526
goto err;
527
527
}
528
528
529
-
/* Finally, tell the device we're all set */
530
-
virtio_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK);
529
+
/* If restore didn't do it, mark device DRIVER_OK ourselves. */
530
+
if (!(dev->config->get_status(dev) & VIRTIO_CONFIG_S_DRIVER_OK))
531
+
virtio_device_ready(dev);
531
532
532
533
virtio_config_enable(dev);
533
534
+7
-41
drivers/virtio/virtio_pci_common.c
+7
-41
drivers/virtio/virtio_pci_common.c
···
24
24
"Force legacy mode for transitional virtio 1 devices");
25
25
#endif
26
26
27
-
/* disable irq handlers */
28
-
void vp_disable_cbs(struct virtio_device *vdev)
27
+
/* wait for pending irq handlers */
28
+
void vp_synchronize_vectors(struct virtio_device *vdev)
29
29
{
30
30
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
31
31
int i;
32
32
33
-
if (vp_dev->intx_enabled) {
34
-
/*
35
-
* The below synchronize() guarantees that any
36
-
* interrupt for this line arriving after
37
-
* synchronize_irq() has completed is guaranteed to see
38
-
* intx_soft_enabled == false.
39
-
*/
40
-
WRITE_ONCE(vp_dev->intx_soft_enabled, false);
33
+
if (vp_dev->intx_enabled)
41
34
synchronize_irq(vp_dev->pci_dev->irq);
42
-
}
43
35
44
36
for (i = 0; i < vp_dev->msix_vectors; ++i)
45
-
disable_irq(pci_irq_vector(vp_dev->pci_dev, i));
46
-
}
47
-
48
-
/* enable irq handlers */
49
-
void vp_enable_cbs(struct virtio_device *vdev)
50
-
{
51
-
struct virtio_pci_device *vp_dev = to_vp_device(vdev);
52
-
int i;
53
-
54
-
if (vp_dev->intx_enabled) {
55
-
disable_irq(vp_dev->pci_dev->irq);
56
-
/*
57
-
* The above disable_irq() provides TSO ordering and
58
-
* as such promotes the below store to store-release.
59
-
*/
60
-
WRITE_ONCE(vp_dev->intx_soft_enabled, true);
61
-
enable_irq(vp_dev->pci_dev->irq);
62
-
return;
63
-
}
64
-
65
-
for (i = 0; i < vp_dev->msix_vectors; ++i)
66
-
enable_irq(pci_irq_vector(vp_dev->pci_dev, i));
37
+
synchronize_irq(pci_irq_vector(vp_dev->pci_dev, i));
67
38
}
68
39
69
40
/* the notify function used when creating a virt queue */
···
83
112
{
84
113
struct virtio_pci_device *vp_dev = opaque;
85
114
u8 isr;
86
-
87
-
if (!READ_ONCE(vp_dev->intx_soft_enabled))
88
-
return IRQ_NONE;
89
115
90
116
/* reading the ISR has the effect of also clearing it so it's very
91
117
* important to save off the value. */
···
141
173
snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
142
174
"%s-config", name);
143
175
err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
144
-
vp_config_changed, IRQF_NO_AUTOEN,
145
-
vp_dev->msix_names[v],
176
+
vp_config_changed, 0, vp_dev->msix_names[v],
146
177
vp_dev);
147
178
if (err)
148
179
goto error;
···
160
193
snprintf(vp_dev->msix_names[v], sizeof *vp_dev->msix_names,
161
194
"%s-virtqueues", name);
162
195
err = request_irq(pci_irq_vector(vp_dev->pci_dev, v),
163
-
vp_vring_interrupt, IRQF_NO_AUTOEN,
164
-
vp_dev->msix_names[v],
196
+
vp_vring_interrupt, 0, vp_dev->msix_names[v],
165
197
vp_dev);
166
198
if (err)
167
199
goto error;
···
337
371
"%s-%s",
338
372
dev_name(&vp_dev->vdev.dev), names[i]);
339
373
err = request_irq(pci_irq_vector(vp_dev->pci_dev, msix_vec),
340
-
vring_interrupt, IRQF_NO_AUTOEN,
374
+
vring_interrupt, 0,
341
375
vp_dev->msix_names[msix_vec],
342
376
vqs[i]);
343
377
if (err)
+2
-5
drivers/virtio/virtio_pci_common.h
+2
-5
drivers/virtio/virtio_pci_common.h
···
63
63
/* MSI-X support */
64
64
int msix_enabled;
65
65
int intx_enabled;
66
-
bool intx_soft_enabled;
67
66
cpumask_var_t *msix_affinity_masks;
68
67
/* Name strings for interrupts. This size should be enough,
69
68
* and I'm too lazy to allocate each name separately. */
···
101
102
return container_of(vdev, struct virtio_pci_device, vdev);
102
103
}
103
104
104
-
/* disable irq handlers */
105
-
void vp_disable_cbs(struct virtio_device *vdev);
106
-
/* enable irq handlers */
107
-
void vp_enable_cbs(struct virtio_device *vdev);
105
+
/* wait for pending irq handlers */
106
+
void vp_synchronize_vectors(struct virtio_device *vdev);
108
107
/* the notify function used when creating a virt queue */
109
108
bool vp_notify(struct virtqueue *vq);
110
109
/* the config->del_vqs() implementation */
+2
-3
drivers/virtio/virtio_pci_legacy.c
+2
-3
drivers/virtio/virtio_pci_legacy.c
···
98
98
/* Flush out the status write, and flush in device writes,
99
99
* including MSi-X interrupts, if any. */
100
100
vp_legacy_get_status(&vp_dev->ldev);
101
-
/* Disable VQ/configuration callbacks. */
102
-
vp_disable_cbs(vdev);
101
+
/* Flush pending VQ/configuration callbacks. */
102
+
vp_synchronize_vectors(vdev);
103
103
}
104
104
105
105
static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
···
185
185
}
186
186
187
187
static const struct virtio_config_ops virtio_pci_config_ops = {
188
-
.enable_cbs = vp_enable_cbs,
189
188
.get = vp_get,
190
189
.set = vp_set,
191
190
.get_status = vp_get_status,
+11
-7
drivers/virtio/virtio_pci_modern.c
+11
-7
drivers/virtio/virtio_pci_modern.c
···
172
172
*/
173
173
while (vp_modern_get_status(mdev))
174
174
msleep(1);
175
-
/* Disable VQ/configuration callbacks. */
176
-
vp_disable_cbs(vdev);
175
+
/* Flush pending VQ/configuration callbacks. */
176
+
vp_synchronize_vectors(vdev);
177
177
}
178
178
179
179
static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector)
···
293
293
294
294
for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); pos > 0;
295
295
pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) {
296
-
u8 type, cap_len, id;
296
+
u8 type, cap_len, id, res_bar;
297
297
u32 tmp32;
298
298
u64 res_offset, res_length;
299
299
···
315
315
if (id != required_id)
316
316
continue;
317
317
318
-
/* Type, and ID match, looks good */
319
318
pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap,
320
-
bar), bar);
319
+
bar), &res_bar);
320
+
if (res_bar >= PCI_STD_NUM_BARS)
321
+
continue;
322
+
323
+
/* Type and ID match, and the BAR value isn't reserved.
324
+
* Looks good.
325
+
*/
321
326
322
327
/* Read the lower 32bit of length and offset */
323
328
pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap,
···
342
337
length_hi), &tmp32);
343
338
res_length |= ((u64)tmp32) << 32;
344
339
340
+
*bar = res_bar;
345
341
*offset = res_offset;
346
342
*len = res_length;
347
343
···
386
380
}
387
381
388
382
static const struct virtio_config_ops virtio_pci_config_nodev_ops = {
389
-
.enable_cbs = vp_enable_cbs,
390
383
.get = NULL,
391
384
.set = NULL,
392
385
.generation = vp_generation,
···
403
398
};
404
399
405
400
static const struct virtio_config_ops virtio_pci_config_ops = {
406
-
.enable_cbs = vp_enable_cbs,
407
401
.get = vp_get,
408
402
.set = vp_set,
409
403
.generation = vp_generation,
+8
-1
drivers/virtio/virtio_pci_modern_dev.c
+8
-1
drivers/virtio/virtio_pci_modern_dev.c
···
35
35
pci_read_config_dword(dev, off + offsetof(struct virtio_pci_cap, length),
36
36
&length);
37
37
38
+
/* Check if the BAR may have changed since we requested the region. */
39
+
if (bar >= PCI_STD_NUM_BARS || !(mdev->modern_bars & (1 << bar))) {
40
+
dev_err(&dev->dev,
41
+
"virtio_pci: bar unexpectedly changed to %u\n", bar);
42
+
return NULL;
43
+
}
44
+
38
45
if (length <= start) {
39
46
dev_err(&dev->dev,
40
47
"virtio_pci: bad capability len %u (>%u expected)\n",
···
127
120
&bar);
128
121
129
122
/* Ignore structures with reserved BAR values */
130
-
if (bar > 0x5)
123
+
if (bar >= PCI_STD_NUM_BARS)
131
124
continue;
132
125
133
126
if (type == cfg_type) {
+18
-35
drivers/virtio/virtio_ring.c
+18
-35
drivers/virtio/virtio_ring.c
···
379
379
380
380
flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
381
381
382
-
if (flags & VRING_DESC_F_INDIRECT) {
383
-
dma_unmap_single(vring_dma_dev(vq),
384
-
virtio64_to_cpu(vq->vq.vdev, desc->addr),
385
-
virtio32_to_cpu(vq->vq.vdev, desc->len),
386
-
(flags & VRING_DESC_F_WRITE) ?
387
-
DMA_FROM_DEVICE : DMA_TO_DEVICE);
388
-
} else {
389
-
dma_unmap_page(vring_dma_dev(vq),
390
-
virtio64_to_cpu(vq->vq.vdev, desc->addr),
391
-
virtio32_to_cpu(vq->vq.vdev, desc->len),
392
-
(flags & VRING_DESC_F_WRITE) ?
393
-
DMA_FROM_DEVICE : DMA_TO_DEVICE);
394
-
}
382
+
dma_unmap_page(vring_dma_dev(vq),
383
+
virtio64_to_cpu(vq->vq.vdev, desc->addr),
384
+
virtio32_to_cpu(vq->vq.vdev, desc->len),
385
+
(flags & VRING_DESC_F_WRITE) ?
386
+
DMA_FROM_DEVICE : DMA_TO_DEVICE);
395
387
}
396
388
397
389
static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
···
976
984
* Packed ring specific functions - *_packed().
977
985
*/
978
986
979
-
static void vring_unmap_state_packed(const struct vring_virtqueue *vq,
980
-
struct vring_desc_extra *state)
987
+
static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
988
+
struct vring_desc_extra *extra)
981
989
{
982
990
u16 flags;
983
991
984
992
if (!vq->use_dma_api)
985
993
return;
986
994
987
-
flags = state->flags;
995
+
flags = extra->flags;
988
996
989
997
if (flags & VRING_DESC_F_INDIRECT) {
990
998
dma_unmap_single(vring_dma_dev(vq),
991
-
state->addr, state->len,
999
+
extra->addr, extra->len,
992
1000
(flags & VRING_DESC_F_WRITE) ?
993
1001
DMA_FROM_DEVICE : DMA_TO_DEVICE);
994
1002
} else {
995
1003
dma_unmap_page(vring_dma_dev(vq),
996
-
state->addr, state->len,
1004
+
extra->addr, extra->len,
997
1005
(flags & VRING_DESC_F_WRITE) ?
998
1006
DMA_FROM_DEVICE : DMA_TO_DEVICE);
999
1007
}
···
1009
1017
1010
1018
flags = le16_to_cpu(desc->flags);
1011
1019
1012
-
if (flags & VRING_DESC_F_INDIRECT) {
1013
-
dma_unmap_single(vring_dma_dev(vq),
1014
-
le64_to_cpu(desc->addr),
1015
-
le32_to_cpu(desc->len),
1016
-
(flags & VRING_DESC_F_WRITE) ?
1017
-
DMA_FROM_DEVICE : DMA_TO_DEVICE);
1018
-
} else {
1019
-
dma_unmap_page(vring_dma_dev(vq),
1020
-
le64_to_cpu(desc->addr),
1021
-
le32_to_cpu(desc->len),
1022
-
(flags & VRING_DESC_F_WRITE) ?
1023
-
DMA_FROM_DEVICE : DMA_TO_DEVICE);
1024
-
}
1020
+
dma_unmap_page(vring_dma_dev(vq),
1021
+
le64_to_cpu(desc->addr),
1022
+
le32_to_cpu(desc->len),
1023
+
(flags & VRING_DESC_F_WRITE) ?
1024
+
DMA_FROM_DEVICE : DMA_TO_DEVICE);
1025
1025
}
1026
1026
1027
1027
static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
···
1287
1303
for (n = 0; n < total_sg; n++) {
1288
1304
if (i == err_idx)
1289
1305
break;
1290
-
vring_unmap_state_packed(vq,
1291
-
&vq->packed.desc_extra[curr]);
1306
+
vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]);
1292
1307
curr = vq->packed.desc_extra[curr].next;
1293
1308
i++;
1294
1309
if (i >= vq->packed.vring.num)
···
1366
1383
if (unlikely(vq->use_dma_api)) {
1367
1384
curr = id;
1368
1385
for (i = 0; i < state->num; i++) {
1369
-
vring_unmap_state_packed(vq,
1370
-
&vq->packed.desc_extra[curr]);
1386
+
vring_unmap_extra_packed(vq,
1387
+
&vq->packed.desc_extra[curr]);
1371
1388
curr = vq->packed.desc_extra[curr].next;
1372
1389
}
1373
1390
}
-22
include/linux/balloon_compaction.h
-22
include/linux/balloon_compaction.h
···
80
80
81
81
#ifdef CONFIG_BALLOON_COMPACTION
82
82
extern const struct address_space_operations balloon_aops;
83
-
extern bool balloon_page_isolate(struct page *page,
84
-
isolate_mode_t mode);
85
-
extern void balloon_page_putback(struct page *page);
86
-
extern int balloon_page_migrate(struct address_space *mapping,
87
-
struct page *newpage,
88
-
struct page *page, enum migrate_mode mode);
89
83
90
84
/*
91
85
* balloon_page_insert - insert a page into the balloon's page list and make
···
147
153
{
148
154
__ClearPageOffline(page);
149
155
list_del(&page->lru);
150
-
}
151
-
152
-
static inline bool balloon_page_isolate(struct page *page)
153
-
{
154
-
return false;
155
-
}
156
-
157
-
static inline void balloon_page_putback(struct page *page)
158
-
{
159
-
return;
160
-
}
161
-
162
-
static inline int balloon_page_migrate(struct page *newpage,
163
-
struct page *page, enum migrate_mode mode)
164
-
{
165
-
return 0;
166
156
}
167
157
168
158
static inline gfp_t balloon_mapping_gfp_mask(void)
+5
-4
include/linux/vdpa.h
+5
-4
include/linux/vdpa.h
···
83
83
unsigned int index;
84
84
bool features_valid;
85
85
bool use_va;
86
-
int nvqs;
86
+
u32 nvqs;
87
87
struct vdpa_mgmt_dev *mdev;
88
88
};
89
89
···
207
207
* @reset: Reset device
208
208
* @vdev: vdpa device
209
209
* Returns integer: success (0) or error (< 0)
210
-
* @get_config_size: Get the size of the configuration space
210
+
* @get_config_size: Get the size of the configuration space includes
211
+
* fields that are conditional on feature bits.
211
212
* @vdev: vdpa device
212
213
* Returns size_t: configuration size
213
214
* @get_config: Read from device specific configuration space
···
338
337
dev_struct, member)), name, use_va), \
339
338
dev_struct, member)
340
339
341
-
int vdpa_register_device(struct vdpa_device *vdev, int nvqs);
340
+
int vdpa_register_device(struct vdpa_device *vdev, u32 nvqs);
342
341
void vdpa_unregister_device(struct vdpa_device *vdev);
343
342
344
-
int _vdpa_register_device(struct vdpa_device *vdev, int nvqs);
343
+
int _vdpa_register_device(struct vdpa_device *vdev, u32 nvqs);
345
344
void _vdpa_unregister_device(struct vdpa_device *vdev);
346
345
347
346
/**
+7
include/uapi/linux/vhost.h
+7
include/uapi/linux/vhost.h
···
150
150
/* Get the valid iova range */
151
151
#define VHOST_VDPA_GET_IOVA_RANGE _IOR(VHOST_VIRTIO, 0x78, \
152
152
struct vhost_vdpa_iova_range)
153
+
154
+
/* Get the config size */
155
+
#define VHOST_VDPA_GET_CONFIG_SIZE _IOR(VHOST_VIRTIO, 0x79, __u32)
156
+
157
+
/* Get the count of all virtqueues */
158
+
#define VHOST_VDPA_GET_VQS_COUNT _IOR(VHOST_VIRTIO, 0x80, __u32)
159
+
153
160
#endif
+6
include/uapi/linux/virtio_config.h
+6
include/uapi/linux/virtio_config.h
···
83
83
#define VIRTIO_F_RING_PACKED 34
84
84
85
85
/*
86
+
* Inorder feature indicates that all buffers are used by the device
87
+
* in the same order in which they have been made available.
88
+
*/
89
+
#define VIRTIO_F_IN_ORDER 35
90
+
91
+
/*
86
92
* This feature indicates that memory accesses by the driver and the
87
93
* device are ordered in a way described by the platform.
88
94
*/
+81
-1
include/uapi/linux/virtio_crypto.h
+81
-1
include/uapi/linux/virtio_crypto.h
···
37
37
#define VIRTIO_CRYPTO_SERVICE_HASH 1
38
38
#define VIRTIO_CRYPTO_SERVICE_MAC 2
39
39
#define VIRTIO_CRYPTO_SERVICE_AEAD 3
40
+
#define VIRTIO_CRYPTO_SERVICE_AKCIPHER 4
40
41
41
42
#define VIRTIO_CRYPTO_OPCODE(service, op) (((service) << 8) | (op))
42
43
···
58
57
VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x02)
59
58
#define VIRTIO_CRYPTO_AEAD_DESTROY_SESSION \
60
59
VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x03)
60
+
#define VIRTIO_CRYPTO_AKCIPHER_CREATE_SESSION \
61
+
VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x04)
62
+
#define VIRTIO_CRYPTO_AKCIPHER_DESTROY_SESSION \
63
+
VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x05)
61
64
__le32 opcode;
62
65
__le32 algo;
63
66
__le32 flag;
···
185
180
__u8 padding[32];
186
181
};
187
182
183
+
struct virtio_crypto_rsa_session_para {
184
+
#define VIRTIO_CRYPTO_RSA_RAW_PADDING 0
185
+
#define VIRTIO_CRYPTO_RSA_PKCS1_PADDING 1
186
+
__le32 padding_algo;
187
+
188
+
#define VIRTIO_CRYPTO_RSA_NO_HASH 0
189
+
#define VIRTIO_CRYPTO_RSA_MD2 1
190
+
#define VIRTIO_CRYPTO_RSA_MD3 2
191
+
#define VIRTIO_CRYPTO_RSA_MD4 3
192
+
#define VIRTIO_CRYPTO_RSA_MD5 4
193
+
#define VIRTIO_CRYPTO_RSA_SHA1 5
194
+
#define VIRTIO_CRYPTO_RSA_SHA256 6
195
+
#define VIRTIO_CRYPTO_RSA_SHA384 7
196
+
#define VIRTIO_CRYPTO_RSA_SHA512 8
197
+
#define VIRTIO_CRYPTO_RSA_SHA224 9
198
+
__le32 hash_algo;
199
+
};
200
+
201
+
struct virtio_crypto_ecdsa_session_para {
202
+
#define VIRTIO_CRYPTO_CURVE_UNKNOWN 0
203
+
#define VIRTIO_CRYPTO_CURVE_NIST_P192 1
204
+
#define VIRTIO_CRYPTO_CURVE_NIST_P224 2
205
+
#define VIRTIO_CRYPTO_CURVE_NIST_P256 3
206
+
#define VIRTIO_CRYPTO_CURVE_NIST_P384 4
207
+
#define VIRTIO_CRYPTO_CURVE_NIST_P521 5
208
+
__le32 curve_id;
209
+
__le32 padding;
210
+
};
211
+
212
+
struct virtio_crypto_akcipher_session_para {
213
+
#define VIRTIO_CRYPTO_NO_AKCIPHER 0
214
+
#define VIRTIO_CRYPTO_AKCIPHER_RSA 1
215
+
#define VIRTIO_CRYPTO_AKCIPHER_DSA 2
216
+
#define VIRTIO_CRYPTO_AKCIPHER_ECDSA 3
217
+
__le32 algo;
218
+
219
+
#define VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PUBLIC 1
220
+
#define VIRTIO_CRYPTO_AKCIPHER_KEY_TYPE_PRIVATE 2
221
+
__le32 keytype;
222
+
__le32 keylen;
223
+
224
+
union {
225
+
struct virtio_crypto_rsa_session_para rsa;
226
+
struct virtio_crypto_ecdsa_session_para ecdsa;
227
+
} u;
228
+
};
229
+
230
+
struct virtio_crypto_akcipher_create_session_req {
231
+
struct virtio_crypto_akcipher_session_para para;
232
+
__u8 padding[36];
233
+
};
234
+
188
235
struct virtio_crypto_alg_chain_session_para {
189
236
#define VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER 1
190
237
#define VIRTIO_CRYPTO_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH 2
···
304
247
mac_create_session;
305
248
struct virtio_crypto_aead_create_session_req
306
249
aead_create_session;
250
+
struct virtio_crypto_akcipher_create_session_req
251
+
akcipher_create_session;
307
252
struct virtio_crypto_destroy_session_req
308
253
destroy_session;
309
254
__u8 padding[56];
···
325
266
VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x00)
326
267
#define VIRTIO_CRYPTO_AEAD_DECRYPT \
327
268
VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AEAD, 0x01)
269
+
#define VIRTIO_CRYPTO_AKCIPHER_ENCRYPT \
270
+
VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x00)
271
+
#define VIRTIO_CRYPTO_AKCIPHER_DECRYPT \
272
+
VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x01)
273
+
#define VIRTIO_CRYPTO_AKCIPHER_SIGN \
274
+
VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x02)
275
+
#define VIRTIO_CRYPTO_AKCIPHER_VERIFY \
276
+
VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x03)
328
277
__le32 opcode;
329
278
/* algo should be service-specific algorithms */
330
279
__le32 algo;
···
457
390
__u8 padding[32];
458
391
};
459
392
393
+
struct virtio_crypto_akcipher_para {
394
+
__le32 src_data_len;
395
+
__le32 dst_data_len;
396
+
};
397
+
398
+
struct virtio_crypto_akcipher_data_req {
399
+
struct virtio_crypto_akcipher_para para;
400
+
__u8 padding[40];
401
+
};
402
+
460
403
/* The request of the data virtqueue's packet */
461
404
struct virtio_crypto_op_data_req {
462
405
struct virtio_crypto_op_header header;
···
476
399
struct virtio_crypto_hash_data_req hash_req;
477
400
struct virtio_crypto_mac_data_req mac_req;
478
401
struct virtio_crypto_aead_data_req aead_req;
402
+
struct virtio_crypto_akcipher_data_req akcipher_req;
479
403
__u8 padding[48];
480
404
} u;
481
405
};
···
486
408
#define VIRTIO_CRYPTO_BADMSG 2
487
409
#define VIRTIO_CRYPTO_NOTSUPP 3
488
410
#define VIRTIO_CRYPTO_INVSESS 4 /* Invalid session id */
411
+
#define VIRTIO_CRYPTO_NOSPC 5 /* no free session ID */
412
+
#define VIRTIO_CRYPTO_KEY_REJECTED 6 /* Signature verification failed */
489
413
490
414
/* The accelerator hardware is ready */
491
415
#define VIRTIO_CRYPTO_S_HW_READY (1 << 0)
···
518
438
__le32 max_cipher_key_len;
519
439
/* Maximum length of authenticated key */
520
440
__le32 max_auth_key_len;
521
-
__le32 reserve;
441
+
__le32 akcipher_algo;
522
442
/* Maximum size of each crypto request's content */
523
443
__le64 max_size;
524
444
};
+3
-3
mm/balloon_compaction.c
+3
-3
mm/balloon_compaction.c
···
203
203
204
204
#ifdef CONFIG_BALLOON_COMPACTION
205
205
206
-
bool balloon_page_isolate(struct page *page, isolate_mode_t mode)
206
+
static bool balloon_page_isolate(struct page *page, isolate_mode_t mode)
207
207
208
208
{
209
209
struct balloon_dev_info *b_dev_info = balloon_page_device(page);
···
217
217
return true;
218
218
}
219
219
220
-
void balloon_page_putback(struct page *page)
220
+
static void balloon_page_putback(struct page *page)
221
221
{
222
222
struct balloon_dev_info *b_dev_info = balloon_page_device(page);
223
223
unsigned long flags;
···
230
230
231
231
232
232
/* move_to_new_page() counterpart for a ballooned page */
233
-
int balloon_page_migrate(struct address_space *mapping,
233
+
static int balloon_page_migrate(struct address_space *mapping,
234
234
struct page *newpage, struct page *page,
235
235
enum migrate_mode mode)
236
236
{
+2
-1
tools/virtio/Makefile
+2
-1
tools/virtio/Makefile
···
5
5
vringh_test: vringh_test.o vringh.o virtio_ring.o
6
6
7
7
CFLAGS += -g -O2 -Werror -Wno-maybe-uninitialized -Wall -I. -I../include/ -I ../../usr/include/ -Wno-pointer-sign -fno-strict-overflow -fno-strict-aliasing -fno-common -MMD -U_FORTIFY_SOURCE -include ../../include/linux/kconfig.h
8
-
LDFLAGS += -lpthread
8
+
CFLAGS += -pthread
9
+
LDFLAGS += -pthread
9
10
vpath %.c ../../drivers/virtio ../../drivers/vhost
10
11
mod:
11
12
${MAKE} -C `pwd`/../.. M=`pwd`/vhost_test V=${V}
+2
-2
tools/virtio/linux/dma-mapping.h
+2
-2
tools/virtio/linux/dma-mapping.h
···
26
26
#define dma_map_single(d, p, s, dir) (virt_to_phys(p))
27
27
#define dma_mapping_error(...) (0)
28
28
29
-
#define dma_unmap_single(...) do { } while (0)
30
-
#define dma_unmap_page(...) do { } while (0)
29
+
#define dma_unmap_single(d, a, s, r) do { (void)(d); (void)(a); (void)(s); (void)(r); } while (0)
30
+
#define dma_unmap_page(d, a, s, r) do { (void)(d); (void)(a); (void)(s); (void)(r); } while (0)
31
31
32
32
#define dma_max_mapping_size(...) SIZE_MAX
33
33