Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'enic-sr-iov-v2-preparatory-infrastructure'
Satish Kharat says:
====================
enic: SR-IOV V2 preparatory infrastructure
This is the first of four series adding SR-IOV V2 support to the enic
driver for Cisco VIC 14xx/15xx adapters.
The existing V1 SR-IOV implementation has VFs that interact directly
with the VIC firmware, leaving the PF driver with no visibility or
control over VF behavior. V2 introduces a PF-mediated model where VFs
communicate with the PF through a mailbox over a dedicated admin
channel. This brings enic in line with the standard Linux SR-IOV
model, enabling full PF management of VFs via ip link (MAC, VLAN,
link state, spoofchk, trust, and per-VF statistics).
This preparatory series adds detection and resource helper code with
no functional change to existing driver behavior:
- Extend BAR resource discovery for admin channel resources
- Register the V2 VF PCI device ID
- Detect VF type (V1/V2/usNIC) from SR-IOV PCI capability
- Make enic_dev_enable/disable ref-counted for shared use by data
path and admin channel
- Add type-aware resource allocation for admin WQ/RQ/CQ/INTR
- Detect presence of admin channel resources at probe time
Tested on VIC 14xx and 15xx series adapters with V2 VFs under KVM
(sriov_numvfs, VF passthrough, ip link VF configuration, VF traffic).
Based in part on initial work by Christian Benvenuti.
====================
Link: https://patch.msgid.link/20260401-enic-sriov-v2-prep-v4-0-d5834b2ef1b9@cisco.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+165
-20
+18
drivers/net/ethernet/cisco/enic/enic.h
+18
drivers/net/ethernet/cisco/enic/enic.h
···
225
225
struct page_pool *pool;
226
226
} ____cacheline_aligned;
227
227
228
+
enum enic_vf_type {
229
+
ENIC_VF_TYPE_NONE,
230
+
ENIC_VF_TYPE_V1,
231
+
ENIC_VF_TYPE_USNIC,
232
+
ENIC_VF_TYPE_V2,
233
+
};
234
+
228
235
/* Per-instance private data structure */
229
236
struct enic {
230
237
struct net_device *netdev;
···
259
252
#ifdef CONFIG_PCI_IOV
260
253
u16 num_vfs;
261
254
#endif
255
+
enum enic_vf_type vf_type;
256
+
unsigned int enable_count;
262
257
spinlock_t enic_api_lock;
263
258
bool enic_api_busy;
264
259
struct enic_port_profile *pp;
···
289
280
u8 rss_key[ENIC_RSS_LEN];
290
281
struct vnic_gen_stats gen_stats;
291
282
enum ext_cq ext_cq;
283
+
284
+
/* Admin channel resources for SR-IOV MBOX */
285
+
bool has_admin_channel;
286
+
struct vnic_wq admin_wq;
287
+
struct vnic_rq admin_rq;
288
+
struct vnic_cq admin_cq[2];
289
+
struct vnic_intr admin_intr;
292
290
};
293
291
294
292
static inline struct net_device *vnic_get_netdev(struct vnic_dev *vdev)
···
313
297
dev_warn(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)
314
298
#define vdev_info(vdev, fmt, ...) \
315
299
dev_info(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)
300
+
#define vdev_dbg(vdev, fmt, ...) \
301
+
dev_dbg(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__)
316
302
317
303
#define vdev_neterr(vdev, fmt, ...) \
318
304
netdev_err(vnic_get_netdev(vdev), fmt, ##__VA_ARGS__)
+13
-4
drivers/net/ethernet/cisco/enic/enic_dev.c
+13
-4
drivers/net/ethernet/cisco/enic/enic_dev.c
···
131
131
132
132
int enic_dev_enable(struct enic *enic)
133
133
{
134
-
int err;
134
+
int err = 0;
135
135
136
136
spin_lock_bh(&enic->devcmd_lock);
137
-
err = vnic_dev_enable_wait(enic->vdev);
137
+
if (enic->enable_count == 0)
138
+
err = vnic_dev_enable_wait(enic->vdev);
139
+
if (!err)
140
+
enic->enable_count++;
138
141
spin_unlock_bh(&enic->devcmd_lock);
139
142
140
143
return err;
···
145
142
146
143
int enic_dev_disable(struct enic *enic)
147
144
{
148
-
int err;
145
+
int err = 0;
149
146
150
147
spin_lock_bh(&enic->devcmd_lock);
151
-
err = vnic_dev_disable(enic->vdev);
148
+
if (enic->enable_count == 0) {
149
+
spin_unlock_bh(&enic->devcmd_lock);
150
+
return 0;
151
+
}
152
+
enic->enable_count--;
153
+
if (enic->enable_count == 0)
154
+
err = vnic_dev_disable(enic->vdev);
152
155
spin_unlock_bh(&enic->devcmd_lock);
153
156
154
157
return err;
+57
-2
drivers/net/ethernet/cisco/enic/enic_main.c
+57
-2
drivers/net/ethernet/cisco/enic/enic_main.c
···
66
66
#define PCI_DEVICE_ID_CISCO_VIC_ENET 0x0043 /* ethernet vnic */
67
67
#define PCI_DEVICE_ID_CISCO_VIC_ENET_DYN 0x0044 /* enet dynamic vnic */
68
68
#define PCI_DEVICE_ID_CISCO_VIC_ENET_VF 0x0071 /* enet SRIOV VF */
69
+
#define PCI_DEVICE_ID_CISCO_VIC_ENET_VF_V2 0x02b7 /* enet SRIOV V2 VF */
70
+
#define PCI_DEVICE_ID_CISCO_VIC_ENET_VF_USNIC 0x00cf /* enet USNIC VF */
69
71
70
72
/* Supported devices */
71
73
static const struct pci_device_id enic_id_table[] = {
72
74
{ PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET) },
73
75
{ PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_DYN) },
74
76
{ PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_VF) },
77
+
{ PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_VF_V2) },
75
78
{ 0, } /* end of table */
76
79
};
77
80
···
310
307
311
308
static int enic_is_sriov_vf(struct enic *enic)
312
309
{
313
-
return enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
310
+
return enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_VF ||
311
+
enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_VF_V2;
314
312
}
315
313
316
314
int enic_is_valid_vf(struct enic *enic, int vf)
···
1750
1746
if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX)
1751
1747
for (i = 0; i < enic->wq_count; i++)
1752
1748
napi_enable(&enic->napi[enic_cq_wq(enic, i)]);
1753
-
enic_dev_enable(enic);
1749
+
err = enic_dev_enable(enic);
1750
+
if (err) {
1751
+
netdev_err(netdev, "Failed to enable device: %d\n", err);
1752
+
goto err_out_dev_enable;
1753
+
}
1754
1754
1755
1755
for (i = 0; i < enic->intr_count; i++)
1756
1756
vnic_intr_unmask(&enic->intr[i]);
···
1764
1756
1765
1757
return 0;
1766
1758
1759
+
err_out_dev_enable:
1760
+
for (i = 0; i < enic->rq_count; i++)
1761
+
napi_disable(&enic->napi[i]);
1762
+
if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX)
1763
+
for (i = 0; i < enic->wq_count; i++)
1764
+
napi_disable(&enic->napi[enic_cq_wq(enic, i)]);
1765
+
netif_tx_disable(netdev);
1766
+
if (!enic_is_dynamic(enic) && !enic_is_sriov_vf(enic))
1767
+
enic_dev_del_station_addr(enic);
1768
+
for (i = 0; i < enic->wq_count; i++)
1769
+
vnic_wq_disable(&enic->wq[i].vwq);
1767
1770
err_out_free_rq:
1768
1771
for (i = 0; i < enic->rq_count; i++) {
1769
1772
ret = vnic_rq_disable(&enic->rq[i].vrq);
···
2637
2618
iounmap(enic->bar[i].vaddr);
2638
2619
}
2639
2620
2621
+
#ifdef CONFIG_PCI_IOV
2622
+
static void enic_sriov_detect_vf_type(struct enic *enic)
2623
+
{
2624
+
struct pci_dev *pdev = enic->pdev;
2625
+
int pos;
2626
+
u16 vf_dev_id;
2627
+
2628
+
if (enic_is_sriov_vf(enic) || enic_is_dynamic(enic))
2629
+
return;
2630
+
2631
+
pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
2632
+
if (!pos) {
2633
+
enic->vf_type = ENIC_VF_TYPE_NONE;
2634
+
return;
2635
+
}
2636
+
2637
+
pci_read_config_word(pdev, pos + PCI_SRIOV_VF_DID, &vf_dev_id);
2638
+
2639
+
switch (vf_dev_id) {
2640
+
case PCI_DEVICE_ID_CISCO_VIC_ENET_VF:
2641
+
enic->vf_type = ENIC_VF_TYPE_V1;
2642
+
break;
2643
+
case PCI_DEVICE_ID_CISCO_VIC_ENET_VF_USNIC:
2644
+
enic->vf_type = ENIC_VF_TYPE_USNIC;
2645
+
break;
2646
+
case PCI_DEVICE_ID_CISCO_VIC_ENET_VF_V2:
2647
+
enic->vf_type = ENIC_VF_TYPE_V2;
2648
+
break;
2649
+
default:
2650
+
enic->vf_type = ENIC_VF_TYPE_NONE;
2651
+
break;
2652
+
}
2653
+
}
2654
+
#endif
2655
+
2640
2656
static int enic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2641
2657
{
2642
2658
struct device *dev = &pdev->dev;
···
2785
2731
num_pps = enic->num_vfs;
2786
2732
}
2787
2733
}
2734
+
enic_sriov_detect_vf_type(enic);
2788
2735
#endif
2789
2736
2790
2737
/* Allocate structure for port profiles */
+10
-2
drivers/net/ethernet/cisco/enic/enic_res.c
+10
-2
drivers/net/ethernet/cisco/enic/enic_res.c
···
205
205
enic->cq_count = enic->cq_avail;
206
206
enic->intr_count = enic->intr_avail;
207
207
208
+
enic->has_admin_channel =
209
+
vnic_dev_get_res_count(enic->vdev, RES_TYPE_ADMIN_WQ) >= 1 &&
210
+
vnic_dev_get_res_count(enic->vdev, RES_TYPE_ADMIN_RQ) >= 1 &&
211
+
vnic_dev_get_res_count(enic->vdev, RES_TYPE_ADMIN_CQ) >=
212
+
ARRAY_SIZE(enic->admin_cq) &&
213
+
vnic_dev_get_res_count(enic->vdev, RES_TYPE_SRIOV_INTR) >= 1;
214
+
208
215
dev_info(enic_get_dev(enic),
209
-
"vNIC resources avail: wq %d rq %d cq %d intr %d\n",
216
+
"vNIC resources avail: wq %d rq %d cq %d intr %d admin %s\n",
210
217
enic->wq_avail, enic->rq_avail,
211
-
enic->cq_avail, enic->intr_avail);
218
+
enic->cq_avail, enic->intr_avail,
219
+
enic->has_admin_channel ? "yes" : "no");
212
220
}
213
221
214
222
void enic_init_vnic_resources(struct enic *enic)
+11
-3
drivers/net/ethernet/cisco/enic/vnic_cq.c
+11
-3
drivers/net/ethernet/cisco/enic/vnic_cq.c
···
20
20
cq->ctrl = NULL;
21
21
}
22
22
23
-
int vnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, unsigned int index,
24
-
unsigned int desc_count, unsigned int desc_size)
23
+
int vnic_cq_alloc_with_type(struct vnic_dev *vdev, struct vnic_cq *cq,
24
+
unsigned int index, unsigned int desc_count,
25
+
unsigned int desc_size, unsigned int res_type)
25
26
{
26
27
cq->index = index;
27
28
cq->vdev = vdev;
28
29
29
-
cq->ctrl = vnic_dev_get_res(vdev, RES_TYPE_CQ, index);
30
+
cq->ctrl = vnic_dev_get_res(vdev, res_type, index);
30
31
if (!cq->ctrl) {
31
32
vdev_err(vdev, "Failed to hook CQ[%d] resource\n", index);
32
33
return -EINVAL;
33
34
}
34
35
35
36
return vnic_dev_alloc_desc_ring(vdev, &cq->ring, desc_count, desc_size);
37
+
}
38
+
39
+
int vnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, unsigned int index,
40
+
unsigned int desc_count, unsigned int desc_size)
41
+
{
42
+
return vnic_cq_alloc_with_type(vdev, cq, index, desc_count, desc_size,
43
+
RES_TYPE_CQ);
36
44
}
37
45
38
46
void vnic_cq_init(struct vnic_cq *cq, unsigned int flow_control_enable,
+3
drivers/net/ethernet/cisco/enic/vnic_cq.h
+3
drivers/net/ethernet/cisco/enic/vnic_cq.h
···
73
73
void vnic_cq_free(struct vnic_cq *cq);
74
74
int vnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, unsigned int index,
75
75
unsigned int desc_count, unsigned int desc_size);
76
+
int vnic_cq_alloc_with_type(struct vnic_dev *vdev, struct vnic_cq *cq,
77
+
unsigned int index, unsigned int desc_count,
78
+
unsigned int desc_size, unsigned int res_type);
76
79
void vnic_cq_init(struct vnic_cq *cq, unsigned int flow_control_enable,
77
80
unsigned int color_enable, unsigned int cq_head, unsigned int cq_tail,
78
81
unsigned int cq_tail_color, unsigned int interrupt_enable,
+10
drivers/net/ethernet/cisco/enic/vnic_dev.c
+10
drivers/net/ethernet/cisco/enic/vnic_dev.c
···
77
77
u32 count = ioread32(&r->count);
78
78
u32 len;
79
79
80
+
vdev_dbg(vdev, "res type %u bar %u offset 0x%x count %u\n",
81
+
type, bar_num, bar_offset, count);
82
+
80
83
r++;
81
84
82
85
if (bar_num >= num_bars)
···
93
90
case RES_TYPE_RQ:
94
91
case RES_TYPE_CQ:
95
92
case RES_TYPE_INTR_CTRL:
93
+
case RES_TYPE_ADMIN_WQ:
94
+
case RES_TYPE_ADMIN_RQ:
95
+
case RES_TYPE_ADMIN_CQ:
96
96
/* each count is stride bytes long */
97
97
len = count * VNIC_RES_STRIDE;
98
98
if (len + bar_offset > bar[bar_num].len) {
···
108
102
case RES_TYPE_INTR_PBA_LEGACY:
109
103
case RES_TYPE_DEVCMD:
110
104
case RES_TYPE_DEVCMD2:
105
+
case RES_TYPE_SRIOV_INTR:
111
106
len = count;
112
107
break;
113
108
default:
···
142
135
case RES_TYPE_RQ:
143
136
case RES_TYPE_CQ:
144
137
case RES_TYPE_INTR_CTRL:
138
+
case RES_TYPE_ADMIN_WQ:
139
+
case RES_TYPE_ADMIN_RQ:
140
+
case RES_TYPE_ADMIN_CQ:
145
141
return (char __iomem *)vdev->res[type].vaddr +
146
142
index * VNIC_RES_STRIDE;
147
143
default:
+9
-3
drivers/net/ethernet/cisco/enic/vnic_intr.c
+9
-3
drivers/net/ethernet/cisco/enic/vnic_intr.c
···
19
19
intr->ctrl = NULL;
20
20
}
21
21
22
-
int vnic_intr_alloc(struct vnic_dev *vdev, struct vnic_intr *intr,
23
-
unsigned int index)
22
+
int vnic_intr_alloc_with_type(struct vnic_dev *vdev, struct vnic_intr *intr,
23
+
unsigned int index, unsigned int res_type)
24
24
{
25
25
intr->index = index;
26
26
intr->vdev = vdev;
27
27
28
-
intr->ctrl = vnic_dev_get_res(vdev, RES_TYPE_INTR_CTRL, index);
28
+
intr->ctrl = vnic_dev_get_res(vdev, res_type, index);
29
29
if (!intr->ctrl) {
30
30
vdev_err(vdev, "Failed to hook INTR[%d].ctrl resource\n",
31
31
index);
···
33
33
}
34
34
35
35
return 0;
36
+
}
37
+
38
+
int vnic_intr_alloc(struct vnic_dev *vdev, struct vnic_intr *intr,
39
+
unsigned int index)
40
+
{
41
+
return vnic_intr_alloc_with_type(vdev, intr, index, RES_TYPE_INTR_CTRL);
36
42
}
37
43
38
44
void vnic_intr_init(struct vnic_intr *intr, u32 coalescing_timer,
+2
drivers/net/ethernet/cisco/enic/vnic_intr.h
+2
drivers/net/ethernet/cisco/enic/vnic_intr.h
···
89
89
void vnic_intr_free(struct vnic_intr *intr);
90
90
int vnic_intr_alloc(struct vnic_dev *vdev, struct vnic_intr *intr,
91
91
unsigned int index);
92
+
int vnic_intr_alloc_with_type(struct vnic_dev *vdev, struct vnic_intr *intr,
93
+
unsigned int index, unsigned int res_type);
92
94
void vnic_intr_init(struct vnic_intr *intr, u32 coalescing_timer,
93
95
unsigned int coalescing_type, unsigned int mask_on_assertion);
94
96
void vnic_intr_coalescing_timer_set(struct vnic_intr *intr,
+4
drivers/net/ethernet/cisco/enic/vnic_resource.h
+4
drivers/net/ethernet/cisco/enic/vnic_resource.h
···
42
42
RES_TYPE_DEPRECATED1, /* Old version of devcmd 2 */
43
43
RES_TYPE_DEPRECATED2, /* Old version of devcmd 2 */
44
44
RES_TYPE_DEVCMD2, /* Device control region */
45
+
RES_TYPE_SRIOV_INTR = 45, /* SR-IOV VF interrupt */
46
+
RES_TYPE_ADMIN_WQ = 49, /* Admin channel WQ */
47
+
RES_TYPE_ADMIN_RQ, /* Admin channel RQ */
48
+
RES_TYPE_ADMIN_CQ, /* Admin channel CQ */
45
49
46
50
RES_TYPE_MAX, /* Count of resource types */
47
51
};
+11
-3
drivers/net/ethernet/cisco/enic/vnic_rq.c
+11
-3
drivers/net/ethernet/cisco/enic/vnic_rq.c
···
69
69
rq->ctrl = NULL;
70
70
}
71
71
72
-
int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
73
-
unsigned int desc_count, unsigned int desc_size)
72
+
int vnic_rq_alloc_with_type(struct vnic_dev *vdev, struct vnic_rq *rq,
73
+
unsigned int index, unsigned int desc_count,
74
+
unsigned int desc_size, unsigned int res_type)
74
75
{
75
76
int err;
76
77
77
78
rq->index = index;
78
79
rq->vdev = vdev;
79
80
80
-
rq->ctrl = vnic_dev_get_res(vdev, RES_TYPE_RQ, index);
81
+
rq->ctrl = vnic_dev_get_res(vdev, res_type, index);
81
82
if (!rq->ctrl) {
82
83
vdev_err(vdev, "Failed to hook RQ[%d] resource\n", index);
83
84
return -EINVAL;
···
97
96
}
98
97
99
98
return 0;
99
+
}
100
+
101
+
int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
102
+
unsigned int desc_count, unsigned int desc_size)
103
+
{
104
+
return vnic_rq_alloc_with_type(vdev, rq, index, desc_count, desc_size,
105
+
RES_TYPE_RQ);
100
106
}
101
107
102
108
static void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index,
+3
drivers/net/ethernet/cisco/enic/vnic_rq.h
+3
drivers/net/ethernet/cisco/enic/vnic_rq.h
···
196
196
void vnic_rq_free(struct vnic_rq *rq);
197
197
int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index,
198
198
unsigned int desc_count, unsigned int desc_size);
199
+
int vnic_rq_alloc_with_type(struct vnic_dev *vdev, struct vnic_rq *rq,
200
+
unsigned int index, unsigned int desc_count,
201
+
unsigned int desc_size, unsigned int res_type);
199
202
void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index,
200
203
unsigned int error_interrupt_enable,
201
204
unsigned int error_interrupt_offset);
+11
-3
drivers/net/ethernet/cisco/enic/vnic_wq.c
+11
-3
drivers/net/ethernet/cisco/enic/vnic_wq.c
···
72
72
wq->ctrl = NULL;
73
73
}
74
74
75
-
int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index,
76
-
unsigned int desc_count, unsigned int desc_size)
75
+
int vnic_wq_alloc_with_type(struct vnic_dev *vdev, struct vnic_wq *wq,
76
+
unsigned int index, unsigned int desc_count,
77
+
unsigned int desc_size, unsigned int res_type)
77
78
{
78
79
int err;
79
80
80
81
wq->index = index;
81
82
wq->vdev = vdev;
82
83
83
-
wq->ctrl = vnic_dev_get_res(vdev, RES_TYPE_WQ, index);
84
+
wq->ctrl = vnic_dev_get_res(vdev, res_type, index);
84
85
if (!wq->ctrl) {
85
86
vdev_err(vdev, "Failed to hook WQ[%d] resource\n", index);
86
87
return -EINVAL;
···
100
99
}
101
100
102
101
return 0;
102
+
}
103
+
104
+
int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index,
105
+
unsigned int desc_count, unsigned int desc_size)
106
+
{
107
+
return vnic_wq_alloc_with_type(vdev, wq, index, desc_count, desc_size,
108
+
RES_TYPE_WQ);
103
109
}
104
110
105
111
int enic_wq_devcmd2_alloc(struct vnic_dev *vdev, struct vnic_wq *wq,
+3
drivers/net/ethernet/cisco/enic/vnic_wq.h
+3
drivers/net/ethernet/cisco/enic/vnic_wq.h
···
165
165
void vnic_wq_free(struct vnic_wq *wq);
166
166
int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index,
167
167
unsigned int desc_count, unsigned int desc_size);
168
+
int vnic_wq_alloc_with_type(struct vnic_dev *vdev, struct vnic_wq *wq,
169
+
unsigned int index, unsigned int desc_count,
170
+
unsigned int desc_size, unsigned int res_type);
168
171
void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
169
172
unsigned int error_interrupt_enable,
170
173
unsigned int error_interrupt_offset);