Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'ionic-rework-fix-for-doorbell-miss'
Shannon Nelson says:
====================
ionic: rework fix for doorbell miss
A latency test in a scaled out setting (many VMs with many queues)
has uncovered an issue with our missed doorbell fix from
commit b69585bfcece ("ionic: missed doorbell workaround")
As a refresher, the Elba ASIC has an issue where once in a blue
moon it might miss/drop a queue doorbell notification from
the driver. This can result in Tx timeouts and potential Rx
buffer misses.
The basic problem with the original solution is that
we're delaying things with a timer for every single queue,
periodically using mod_timer() to reset to reset the alarm, and
mod_timer() becomes a more and more expensive thing as there
are more and more VFs and queues each with their own timer.
A ping-pong latency test tends to exacerbate the effect such
that every napi is doing a mod_timer() in every cycle.
An alternative has been worked out to replace this using
periodic workqueue items outside the napi cycle to request a
napi_schedule driven by a single delayed-workqueue per device
rather than a timer for every queue. Also, now that newer
firmware is actually reporting its ASIC type, we can restrict
this to the appropriate chip.
The testing scenario used 128 VFs in UP state, 16 queues per
VF, and latency tests were done using TCP_RR with adaptive
interrupt coalescing enabled, running on 1 VF. We would see
99th percentile latencies of up to 900us range, with some max
fliers as much as 4ms.
With these fixes the 99th percentile latencies are typically well
under 50us with the occasional max under 500us.
v1:
https://lore.kernel.org/netdev/20240610230706.34883-1-shannon.nelson@amd.com/
====================
Link: https://lore.kernel.org/r/20240619003257.6138-1-shannon.nelson@amd.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+264
-76
+7
drivers/net/ethernet/pensando/ionic/ionic.h
+7
drivers/net/ethernet/pensando/ionic/ionic.h
···
18
18
#define PCI_DEVICE_ID_PENSANDO_IONIC_ETH_PF 0x1002
19
19
#define PCI_DEVICE_ID_PENSANDO_IONIC_ETH_VF 0x1003
20
20
21
+
#define IONIC_ASIC_TYPE_ELBA 2
22
+
21
23
#define DEVCMD_TIMEOUT 5
22
24
#define IONIC_ADMINQ_TIME_SLICE msecs_to_jiffies(100)
23
25
···
49
47
struct ionic_dev_bar bars[IONIC_BARS_MAX];
50
48
unsigned int num_bars;
51
49
struct ionic_identity ident;
50
+
struct workqueue_struct *wq;
52
51
struct ionic_lif *lif;
53
52
unsigned int nnqs_per_lif;
54
53
unsigned int neqs_per_lif;
···
57
54
unsigned int nrxqs_per_lif;
58
55
unsigned int nintrs;
59
56
DECLARE_BITMAP(intrs, IONIC_INTR_CTRL_REGS_MAX);
57
+
cpumask_var_t *affinity_masks;
58
+
struct delayed_work doorbell_check_dwork;
60
59
struct work_struct nb_work;
61
60
struct notifier_block nb;
62
61
struct rw_semaphore vf_op_lock; /* lock for VF operations */
···
97
92
int ionic_port_identify(struct ionic *ionic);
98
93
int ionic_port_init(struct ionic *ionic);
99
94
int ionic_port_reset(struct ionic *ionic);
95
+
96
+
bool ionic_doorbell_wa(struct ionic *ionic);
100
97
101
98
#endif /* _IONIC_H_ */
+3
drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
+3
drivers/net/ethernet/pensando/ionic/ionic_bus_pci.c
···
377
377
378
378
mod_timer(&ionic->watchdog_timer,
379
379
round_jiffies(jiffies + ionic->watchdog_period));
380
+
ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
380
381
381
382
return 0;
382
383
···
412
411
if (test_and_clear_bit(IONIC_LIF_F_FW_RESET, ionic->lif->state))
413
412
set_bit(IONIC_LIF_F_FW_STOPPING, ionic->lif->state);
414
413
414
+
if (ionic->lif->doorbell_wa)
415
+
cancel_delayed_work_sync(&ionic->doorbell_check_dwork);
415
416
ionic_lif_unregister(ionic->lif);
416
417
ionic_devlink_unregister(ionic);
417
418
ionic_lif_deinit(ionic->lif);
+121
-8
drivers/net/ethernet/pensando/ionic/ionic_dev.c
+121
-8
drivers/net/ethernet/pensando/ionic/ionic_dev.c
···
43
43
44
44
work->type = IONIC_DW_TYPE_RX_MODE;
45
45
netdev_dbg(lif->netdev, "deferred: rx_mode\n");
46
-
ionic_lif_deferred_enqueue(&lif->deferred, work);
46
+
ionic_lif_deferred_enqueue(lif, work);
47
47
}
48
48
}
49
49
50
-
static void ionic_watchdog_init(struct ionic *ionic)
50
+
static void ionic_napi_schedule_do_softirq(struct napi_struct *napi)
51
+
{
52
+
local_bh_disable();
53
+
napi_schedule(napi);
54
+
local_bh_enable();
55
+
}
56
+
57
+
void ionic_doorbell_napi_work(struct work_struct *work)
58
+
{
59
+
struct ionic_qcq *qcq = container_of(work, struct ionic_qcq,
60
+
doorbell_napi_work);
61
+
unsigned long now, then, dif;
62
+
63
+
now = READ_ONCE(jiffies);
64
+
then = qcq->q.dbell_jiffies;
65
+
dif = now - then;
66
+
67
+
if (dif > qcq->q.dbell_deadline)
68
+
ionic_napi_schedule_do_softirq(&qcq->napi);
69
+
}
70
+
71
+
static int ionic_get_preferred_cpu(struct ionic *ionic,
72
+
struct ionic_intr_info *intr)
73
+
{
74
+
int cpu;
75
+
76
+
cpu = cpumask_first_and(*intr->affinity_mask, cpu_online_mask);
77
+
if (cpu >= nr_cpu_ids)
78
+
cpu = cpumask_local_spread(0, dev_to_node(ionic->dev));
79
+
80
+
return cpu;
81
+
}
82
+
83
+
static void ionic_queue_dbell_napi_work(struct ionic *ionic,
84
+
struct ionic_qcq *qcq)
85
+
{
86
+
int cpu;
87
+
88
+
if (!(qcq->flags & IONIC_QCQ_F_INTR))
89
+
return;
90
+
91
+
cpu = ionic_get_preferred_cpu(ionic, &qcq->intr);
92
+
queue_work_on(cpu, ionic->wq, &qcq->doorbell_napi_work);
93
+
}
94
+
95
+
static void ionic_doorbell_check_dwork(struct work_struct *work)
96
+
{
97
+
struct ionic *ionic = container_of(work, struct ionic,
98
+
doorbell_check_dwork.work);
99
+
struct ionic_lif *lif = ionic->lif;
100
+
101
+
mutex_lock(&lif->queue_lock);
102
+
103
+
if (test_bit(IONIC_LIF_F_FW_STOPPING, lif->state) ||
104
+
test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
105
+
mutex_unlock(&lif->queue_lock);
106
+
return;
107
+
}
108
+
109
+
ionic_napi_schedule_do_softirq(&lif->adminqcq->napi);
110
+
111
+
if (test_bit(IONIC_LIF_F_UP, lif->state)) {
112
+
int i;
113
+
114
+
for (i = 0; i < lif->nxqs; i++) {
115
+
ionic_queue_dbell_napi_work(ionic, lif->txqcqs[i]);
116
+
ionic_queue_dbell_napi_work(ionic, lif->rxqcqs[i]);
117
+
}
118
+
119
+
if (lif->hwstamp_txq &&
120
+
lif->hwstamp_txq->flags & IONIC_QCQ_F_INTR)
121
+
ionic_napi_schedule_do_softirq(&lif->hwstamp_txq->napi);
122
+
if (lif->hwstamp_rxq &&
123
+
lif->hwstamp_rxq->flags & IONIC_QCQ_F_INTR)
124
+
ionic_napi_schedule_do_softirq(&lif->hwstamp_rxq->napi);
125
+
}
126
+
mutex_unlock(&lif->queue_lock);
127
+
128
+
ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
129
+
}
130
+
131
+
bool ionic_doorbell_wa(struct ionic *ionic)
132
+
{
133
+
u8 asic_type = ionic->idev.dev_info.asic_type;
134
+
135
+
return !asic_type || asic_type == IONIC_ASIC_TYPE_ELBA;
136
+
}
137
+
138
+
static int ionic_watchdog_init(struct ionic *ionic)
51
139
{
52
140
struct ionic_dev *idev = &ionic->idev;
53
141
···
151
63
idev->fw_status_ready = true;
152
64
idev->fw_generation = IONIC_FW_STS_F_GENERATION &
153
65
ioread8(&idev->dev_info_regs->fw_status);
66
+
67
+
ionic->wq = alloc_workqueue("%s-wq", WQ_UNBOUND, 0,
68
+
dev_name(ionic->dev));
69
+
if (!ionic->wq) {
70
+
dev_err(ionic->dev, "alloc_workqueue failed");
71
+
return -ENOMEM;
72
+
}
73
+
74
+
if (ionic_doorbell_wa(ionic))
75
+
INIT_DELAYED_WORK(&ionic->doorbell_check_dwork,
76
+
ionic_doorbell_check_dwork);
77
+
78
+
return 0;
79
+
}
80
+
81
+
void ionic_queue_doorbell_check(struct ionic *ionic, int delay)
82
+
{
83
+
int cpu;
84
+
85
+
if (!ionic->lif->doorbell_wa)
86
+
return;
87
+
88
+
cpu = ionic_get_preferred_cpu(ionic, &ionic->lif->adminqcq->intr);
89
+
queue_delayed_work_on(cpu, ionic->wq, &ionic->doorbell_check_dwork,
90
+
delay);
154
91
}
155
92
156
93
void ionic_init_devinfo(struct ionic *ionic)
···
207
94
struct device *dev = ionic->dev;
208
95
int size;
209
96
u32 sig;
97
+
int err;
210
98
211
99
/* BAR0: dev_cmd and interrupts */
212
100
if (num_bars < 1) {
···
243
129
return -EFAULT;
244
130
}
245
131
246
-
ionic_watchdog_init(ionic);
132
+
err = ionic_watchdog_init(ionic);
133
+
if (err)
134
+
return err;
247
135
248
136
idev->db_pages = bar->vaddr;
249
137
idev->phy_db_pages = bar->bus_addr;
···
277
161
idev->phy_cmb_pages = 0;
278
162
idev->cmb_npages = 0;
279
163
164
+
destroy_workqueue(ionic->wq);
280
165
mutex_destroy(&idev->cmb_inuse_lock);
281
166
}
282
167
···
390
273
if (work) {
391
274
work->type = IONIC_DW_TYPE_LIF_RESET;
392
275
work->fw_status = fw_status_ready;
393
-
ionic_lif_deferred_enqueue(&lif->deferred, work);
276
+
ionic_lif_deferred_enqueue(lif, work);
394
277
}
395
278
}
396
279
}
···
820
703
q->dbval | q->head_idx);
821
704
822
705
q->dbell_jiffies = jiffies;
823
-
824
-
if (q_to_qcq(q)->napi_qcq)
825
-
mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
826
-
jiffies + IONIC_NAPI_DEADLINE);
827
706
}
828
707
}
829
708
+5
-3
drivers/net/ethernet/pensando/ionic/ionic_dev.h
+5
-3
drivers/net/ethernet/pensando/ionic/ionic_dev.h
···
28
28
#define IONIC_DEV_INFO_REG_COUNT 32
29
29
#define IONIC_DEV_CMD_REG_COUNT 32
30
30
31
-
#define IONIC_NAPI_DEADLINE (HZ / 200) /* 5ms */
31
+
#define IONIC_NAPI_DEADLINE (HZ) /* 1 sec */
32
32
#define IONIC_ADMIN_DOORBELL_DEADLINE (HZ / 2) /* 500ms */
33
33
#define IONIC_TX_DOORBELL_DEADLINE (HZ / 100) /* 10ms */
34
34
#define IONIC_RX_MIN_DOORBELL_DEADLINE (HZ / 100) /* 10ms */
···
280
280
u64 rearm_count;
281
281
unsigned int index;
282
282
unsigned int vector;
283
-
unsigned int cpu;
284
283
u32 dim_coal_hw;
285
-
cpumask_t affinity_mask;
284
+
cpumask_var_t *affinity_mask;
285
+
struct irq_affinity_notify aff_notify;
286
286
};
287
287
288
288
struct ionic_cq {
···
386
386
387
387
int ionic_heartbeat_check(struct ionic *ionic);
388
388
bool ionic_is_fw_running(struct ionic_dev *idev);
389
+
void ionic_doorbell_napi_work(struct work_struct *work);
390
+
void ionic_queue_doorbell_check(struct ionic *ionic, int delay);
389
391
390
392
bool ionic_adminq_poke_doorbell(struct ionic_queue *q);
391
393
bool ionic_txq_poke_doorbell(struct ionic_queue *q);
+10
-1
drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
+10
-1
drivers/net/ethernet/pensando/ionic/ionic_ethtool.c
···
11
11
#include "ionic_ethtool.h"
12
12
#include "ionic_stats.h"
13
13
14
+
#define IONIC_MAX_RX_COPYBREAK min(U16_MAX, IONIC_MAX_BUF_LEN)
15
+
14
16
static void ionic_get_stats_strings(struct ionic_lif *lif, u8 *buf)
15
17
{
16
18
u32 i;
···
874
872
const void *data)
875
873
{
876
874
struct ionic_lif *lif = netdev_priv(dev);
875
+
u32 rx_copybreak;
877
876
878
877
switch (tuna->id) {
879
878
case ETHTOOL_RX_COPYBREAK:
880
-
lif->rx_copybreak = *(u32 *)data;
879
+
rx_copybreak = *(u32 *)data;
880
+
if (rx_copybreak > IONIC_MAX_RX_COPYBREAK) {
881
+
netdev_err(dev, "Max supported rx_copybreak size: %u\n",
882
+
IONIC_MAX_RX_COPYBREAK);
883
+
return -EINVAL;
884
+
}
885
+
lif->rx_copybreak = (u16)rx_copybreak;
881
886
break;
882
887
default:
883
888
return -EOPNOTSUPP;
+101
-43
drivers/net/ethernet/pensando/ionic/ionic_lif.c
+101
-43
drivers/net/ethernet/pensando/ionic/ionic_lif.c
···
126
126
} while (true);
127
127
}
128
128
129
-
void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
129
+
void ionic_lif_deferred_enqueue(struct ionic_lif *lif,
130
130
struct ionic_deferred_work *work)
131
131
{
132
-
spin_lock_bh(&def->lock);
133
-
list_add_tail(&work->list, &def->list);
134
-
spin_unlock_bh(&def->lock);
135
-
schedule_work(&def->work);
132
+
spin_lock_bh(&lif->deferred.lock);
133
+
list_add_tail(&work->list, &lif->deferred.list);
134
+
spin_unlock_bh(&lif->deferred.lock);
135
+
queue_work(lif->ionic->wq, &lif->deferred.work);
136
136
}
137
137
138
138
static void ionic_link_status_check(struct ionic_lif *lif)
···
207
207
}
208
208
209
209
work->type = IONIC_DW_TYPE_LINK_STATUS;
210
-
ionic_lif_deferred_enqueue(&lif->deferred, work);
210
+
ionic_lif_deferred_enqueue(lif, work);
211
211
} else {
212
212
ionic_link_status_check(lif);
213
213
}
214
-
}
215
-
216
-
static void ionic_napi_deadline(struct timer_list *timer)
217
-
{
218
-
struct ionic_qcq *qcq = container_of(timer, struct ionic_qcq, napi_deadline);
219
-
220
-
napi_schedule(&qcq->napi);
221
214
}
222
215
223
216
static irqreturn_t ionic_isr(int irq, void *data)
···
265
272
clear_bit(index, ionic->intrs);
266
273
}
267
274
275
+
static void ionic_irq_aff_notify(struct irq_affinity_notify *notify,
276
+
const cpumask_t *mask)
277
+
{
278
+
struct ionic_intr_info *intr = container_of(notify, struct ionic_intr_info, aff_notify);
279
+
280
+
cpumask_copy(*intr->affinity_mask, mask);
281
+
}
282
+
283
+
static void ionic_irq_aff_release(struct kref __always_unused *ref)
284
+
{
285
+
}
286
+
268
287
static int ionic_qcq_enable(struct ionic_qcq *qcq)
269
288
{
270
289
struct ionic_queue *q = &qcq->q;
···
311
306
312
307
if (qcq->flags & IONIC_QCQ_F_INTR) {
313
308
napi_enable(&qcq->napi);
309
+
irq_set_affinity_notifier(qcq->intr.vector,
310
+
&qcq->intr.aff_notify);
314
311
irq_set_affinity_hint(qcq->intr.vector,
315
-
&qcq->intr.affinity_mask);
312
+
*qcq->intr.affinity_mask);
316
313
ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
317
314
IONIC_INTR_MASK_CLEAR);
318
315
}
···
344
337
if (qcq->flags & IONIC_QCQ_F_INTR) {
345
338
struct ionic_dev *idev = &lif->ionic->idev;
346
339
340
+
if (lif->doorbell_wa)
341
+
cancel_work_sync(&qcq->doorbell_napi_work);
347
342
cancel_work_sync(&qcq->dim.work);
348
343
ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
349
344
IONIC_INTR_MASK_SET);
350
345
synchronize_irq(qcq->intr.vector);
346
+
irq_set_affinity_notifier(qcq->intr.vector, NULL);
351
347
irq_set_affinity_hint(qcq->intr.vector, NULL);
352
348
napi_disable(&qcq->napi);
353
-
del_timer_sync(&qcq->napi_deadline);
354
349
}
355
350
356
351
/* If there was a previous fw communcation error, don't bother with
···
487
478
{
488
479
n_qcq->intr.vector = src_qcq->intr.vector;
489
480
n_qcq->intr.index = src_qcq->intr.index;
490
-
n_qcq->napi_qcq = src_qcq->napi_qcq;
491
481
}
492
482
493
483
static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
494
484
{
485
+
cpumask_var_t *affinity_mask;
495
486
int err;
496
487
497
488
if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
···
523
514
}
524
515
525
516
/* try to get the irq on the local numa node first */
526
-
qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
527
-
dev_to_node(lif->ionic->dev));
528
-
if (qcq->intr.cpu != -1)
529
-
cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
517
+
affinity_mask = &lif->ionic->affinity_masks[qcq->intr.index];
518
+
if (cpumask_empty(*affinity_mask)) {
519
+
unsigned int cpu;
520
+
521
+
cpu = cpumask_local_spread(qcq->intr.index,
522
+
dev_to_node(lif->ionic->dev));
523
+
if (cpu != -1)
524
+
cpumask_set_cpu(cpu, *affinity_mask);
525
+
}
526
+
527
+
qcq->intr.affinity_mask = affinity_mask;
528
+
qcq->intr.aff_notify.notify = ionic_irq_aff_notify;
529
+
qcq->intr.aff_notify.release = ionic_irq_aff_release;
530
530
531
531
netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
532
532
return 0;
···
692
674
693
675
INIT_WORK(&new->dim.work, ionic_dim_work);
694
676
new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
677
+
if (lif->doorbell_wa)
678
+
INIT_WORK(&new->doorbell_napi_work, ionic_doorbell_napi_work);
695
679
696
680
*qcq = new;
697
681
···
852
832
q->dbell_deadline = IONIC_TX_DOORBELL_DEADLINE;
853
833
q->dbell_jiffies = jiffies;
854
834
855
-
if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) {
835
+
if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
856
836
netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi);
857
-
qcq->napi_qcq = qcq;
858
-
timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
859
-
}
860
837
861
838
qcq->flags |= IONIC_QCQ_F_INITED;
862
839
···
925
908
netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi);
926
909
else
927
910
netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi);
928
-
929
-
qcq->napi_qcq = qcq;
930
-
timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
931
911
932
912
qcq->flags |= IONIC_QCQ_F_INITED;
933
913
···
1180
1166
struct ionic_dev *idev = &lif->ionic->idev;
1181
1167
unsigned long irqflags;
1182
1168
unsigned int flags = 0;
1183
-
bool resched = false;
1184
1169
int rx_work = 0;
1185
1170
int tx_work = 0;
1186
1171
int n_work = 0;
···
1195
1182
if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1196
1183
a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1197
1184
ionic_adminq_service, NULL, NULL);
1185
+
1198
1186
spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1199
1187
1200
1188
if (lif->hwstamp_rxq)
···
1217
1203
ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1218
1204
}
1219
1205
1220
-
if (!a_work && ionic_adminq_poke_doorbell(&lif->adminqcq->q))
1221
-
resched = true;
1222
-
if (lif->hwstamp_rxq && !rx_work && ionic_rxq_poke_doorbell(&lif->hwstamp_rxq->q))
1223
-
resched = true;
1224
-
if (lif->hwstamp_txq && !tx_work && ionic_txq_poke_doorbell(&lif->hwstamp_txq->q))
1225
-
resched = true;
1226
-
if (resched)
1227
-
mod_timer(&lif->adminqcq->napi_deadline,
1228
-
jiffies + IONIC_NAPI_DEADLINE);
1206
+
if (lif->doorbell_wa) {
1207
+
if (!a_work)
1208
+
ionic_adminq_poke_doorbell(&lif->adminqcq->q);
1209
+
if (lif->hwstamp_rxq && !rx_work)
1210
+
ionic_rxq_poke_doorbell(&lif->hwstamp_rxq->q);
1211
+
if (lif->hwstamp_txq && !tx_work)
1212
+
ionic_txq_poke_doorbell(&lif->hwstamp_txq->q);
1213
+
}
1229
1214
1230
1215
return work_done;
1231
1216
}
···
1396
1383
}
1397
1384
work->type = IONIC_DW_TYPE_RX_MODE;
1398
1385
netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1399
-
ionic_lif_deferred_enqueue(&lif->deferred, work);
1386
+
ionic_lif_deferred_enqueue(lif, work);
1400
1387
}
1401
1388
1402
1389
static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
···
3152
3139
return err;
3153
3140
}
3154
3141
3142
+
static int ionic_affinity_masks_alloc(struct ionic *ionic)
3143
+
{
3144
+
cpumask_var_t *affinity_masks;
3145
+
int nintrs = ionic->nintrs;
3146
+
int i;
3147
+
3148
+
affinity_masks = kcalloc(nintrs, sizeof(cpumask_var_t), GFP_KERNEL);
3149
+
if (!affinity_masks)
3150
+
return -ENOMEM;
3151
+
3152
+
for (i = 0; i < nintrs; i++) {
3153
+
if (!zalloc_cpumask_var_node(&affinity_masks[i], GFP_KERNEL,
3154
+
dev_to_node(ionic->dev)))
3155
+
goto err_out;
3156
+
}
3157
+
3158
+
ionic->affinity_masks = affinity_masks;
3159
+
3160
+
return 0;
3161
+
3162
+
err_out:
3163
+
for (--i; i >= 0; i--)
3164
+
free_cpumask_var(affinity_masks[i]);
3165
+
kfree(affinity_masks);
3166
+
3167
+
return -ENOMEM;
3168
+
}
3169
+
3170
+
static void ionic_affinity_masks_free(struct ionic *ionic)
3171
+
{
3172
+
int i;
3173
+
3174
+
for (i = 0; i < ionic->nintrs; i++)
3175
+
free_cpumask_var(ionic->affinity_masks[i]);
3176
+
kfree(ionic->affinity_masks);
3177
+
ionic->affinity_masks = NULL;
3178
+
}
3179
+
3155
3180
int ionic_lif_alloc(struct ionic *ionic)
3156
3181
{
3157
3182
struct device *dev = ionic->dev;
···
3281
3230
3282
3231
ionic_debugfs_add_lif(lif);
3283
3232
3233
+
err = ionic_affinity_masks_alloc(ionic);
3234
+
if (err)
3235
+
goto err_out_free_lif_info;
3236
+
3284
3237
/* allocate control queues and txrx queue arrays */
3285
3238
ionic_lif_queue_identify(lif);
3286
3239
err = ionic_qcqs_alloc(lif);
3287
3240
if (err)
3288
-
goto err_out_free_lif_info;
3241
+
goto err_out_free_affinity_masks;
3289
3242
3290
3243
/* allocate rss indirection table */
3291
3244
tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
···
3311
3256
3312
3257
err_out_free_qcqs:
3313
3258
ionic_qcqs_free(lif);
3259
+
err_out_free_affinity_masks:
3260
+
ionic_affinity_masks_free(lif->ionic);
3314
3261
err_out_free_lif_info:
3315
3262
dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3316
3263
lif->info = NULL;
···
3413
3356
clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
3414
3357
ionic_link_status_check_request(lif, CAN_SLEEP);
3415
3358
netif_device_attach(lif->netdev);
3359
+
ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
3416
3360
3417
3361
return 0;
3418
3362
···
3486
3428
ionic_qcqs_free(lif);
3487
3429
if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3488
3430
ionic_lif_reset(lif);
3431
+
3432
+
ionic_affinity_masks_free(lif->ionic);
3489
3433
3490
3434
/* free lif info */
3491
3435
kfree(lif->identity);
···
3562
3502
3563
3503
netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi);
3564
3504
3565
-
qcq->napi_qcq = qcq;
3566
-
timer_setup(&qcq->napi_deadline, ionic_napi_deadline, 0);
3567
-
3568
3505
napi_enable(&qcq->napi);
3569
3506
3570
3507
if (qcq->flags & IONIC_QCQ_F_INTR) {
3571
3508
irq_set_affinity_hint(qcq->intr.vector,
3572
-
&qcq->intr.affinity_mask);
3509
+
*qcq->intr.affinity_mask);
3573
3510
ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3574
3511
IONIC_INTR_MASK_CLEAR);
3575
3512
}
···
3753
3696
goto err_out_notifyq_deinit;
3754
3697
3755
3698
lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3699
+
lif->doorbell_wa = ionic_doorbell_wa(lif->ionic);
3756
3700
3757
3701
set_bit(IONIC_LIF_F_INITED, lif->state);
3758
3702
+6
-6
drivers/net/ethernet/pensando/ionic/ionic_lif.h
+6
-6
drivers/net/ethernet/pensando/ionic/ionic_lif.h
···
84
84
u32 cmb_pgid;
85
85
u32 cmb_order;
86
86
struct dim dim;
87
-
struct timer_list napi_deadline;
88
87
struct ionic_queue q;
89
88
struct ionic_cq cq;
90
89
struct napi_struct napi;
91
-
struct ionic_qcq *napi_qcq;
92
90
struct ionic_intr_info intr;
91
+
struct work_struct doorbell_napi_work;
93
92
struct dentry *dentry;
94
93
};
95
94
···
206
207
unsigned int nxqs;
207
208
unsigned int ntxq_descs;
208
209
unsigned int nrxq_descs;
209
-
u32 rx_copybreak;
210
210
u64 rxq_features;
211
-
u16 rx_mode;
212
211
u64 hw_features;
212
+
u16 rx_copybreak;
213
+
u16 rx_mode;
213
214
bool registered;
215
+
bool doorbell_wa;
214
216
u16 lif_type;
215
217
unsigned int link_down_count;
216
218
unsigned int nmcast;
···
226
226
u32 info_sz;
227
227
struct ionic_qtype_info qtype_info[IONIC_QTYPE_MAX];
228
228
229
-
u16 rss_types;
230
229
u8 rss_hash_key[IONIC_RSS_HASH_KEY_SIZE];
231
230
u8 *rss_ind_tbl;
232
231
dma_addr_t rss_ind_tbl_pa;
233
232
u32 rss_ind_tbl_sz;
233
+
u16 rss_types;
234
234
235
235
struct ionic_rx_filters rx_filters;
236
236
u32 rx_coalesce_usecs; /* what the user asked for */
···
333
333
void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep);
334
334
void ionic_get_stats64(struct net_device *netdev,
335
335
struct rtnl_link_stats64 *ns);
336
-
void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
336
+
void ionic_lif_deferred_enqueue(struct ionic_lif *lif,
337
337
struct ionic_deferred_work *work);
338
338
int ionic_lif_alloc(struct ionic *ionic);
339
339
int ionic_lif_init(struct ionic_lif *lif);
+1
-1
drivers/net/ethernet/pensando/ionic/ionic_main.c
+1
-1
drivers/net/ethernet/pensando/ionic/ionic_main.c
+10
-14
drivers/net/ethernet/pensando/ionic/ionic_txrx.c
+10
-14
drivers/net/ethernet/pensando/ionic/ionic_txrx.c
···
868
868
869
869
q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
870
870
q->dbell_jiffies = jiffies;
871
-
872
-
mod_timer(&q_to_qcq(q)->napi_qcq->napi_deadline,
873
-
jiffies + IONIC_NAPI_DEADLINE);
874
871
}
875
872
876
873
void ionic_rx_empty(struct ionic_queue *q)
···
950
953
work_done, flags);
951
954
}
952
955
953
-
if (!work_done && ionic_txq_poke_doorbell(&qcq->q))
954
-
mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
956
+
if (!work_done && cq->bound_q->lif->doorbell_wa)
957
+
ionic_txq_poke_doorbell(&qcq->q);
955
958
956
959
return work_done;
957
960
}
···
993
996
work_done, flags);
994
997
}
995
998
996
-
if (!work_done && ionic_rxq_poke_doorbell(&qcq->q))
997
-
mod_timer(&qcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
999
+
if (!work_done && cq->bound_q->lif->doorbell_wa)
1000
+
ionic_rxq_poke_doorbell(&qcq->q);
998
1001
999
1002
return work_done;
1000
1003
}
···
1007
1010
struct ionic_qcq *txqcq;
1008
1011
struct ionic_lif *lif;
1009
1012
struct ionic_cq *txcq;
1010
-
bool resched = false;
1011
1013
u32 rx_work_done = 0;
1012
1014
u32 tx_work_done = 0;
1013
1015
u32 flags = 0;
···
1038
1042
tx_work_done + rx_work_done, flags);
1039
1043
}
1040
1044
1041
-
if (!rx_work_done && ionic_rxq_poke_doorbell(&rxqcq->q))
1042
-
resched = true;
1043
-
if (!tx_work_done && ionic_txq_poke_doorbell(&txqcq->q))
1044
-
resched = true;
1045
-
if (resched)
1046
-
mod_timer(&rxqcq->napi_deadline, jiffies + IONIC_NAPI_DEADLINE);
1045
+
if (lif->doorbell_wa) {
1046
+
if (!rx_work_done)
1047
+
ionic_rxq_poke_doorbell(&rxqcq->q);
1048
+
if (!tx_work_done)
1049
+
ionic_txq_poke_doorbell(&txqcq->q);
1050
+
}
1047
1051
1048
1052
return rx_work_done;
1049
1053
}