Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'ibmveth-rr-performance'
Nick Child says:
====================
ibmveth RR performance
This patchset aims to increase the ibmveth drivers small packet
request response rate.
These 2 patches address:
1. NAPI rescheduling technique
2. Driver-side processing of small packets
Testing over several netperf tcp_rr connections, we saw a
30% increase in transactions per second. No regressions
were observed in other workloads.
====================
Link: https://patch.msgid.link/20240801211215.128101-1-nnac123@linux.ibm.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+77
-99
+77
-99
drivers/net/ethernet/ibm/ibmveth.c
+77
-99
drivers/net/ethernet/ibm/ibmveth.c
···
39
39
#include "ibmveth.h"
40
40
41
41
static irqreturn_t ibmveth_interrupt(int irq, void *dev_instance);
42
-
static void ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter);
42
+
static void ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter,
43
+
bool reuse);
43
44
static unsigned long ibmveth_get_desired_dma(struct vio_dev *vdev);
44
45
45
46
static struct kobj_type ktype_veth_pool;
···
227
226
for (i = 0; i < count; ++i) {
228
227
union ibmveth_buf_desc desc;
229
228
229
+
free_index = pool->consumer_index;
230
+
index = pool->free_map[free_index];
231
+
skb = NULL;
232
+
233
+
BUG_ON(index == IBM_VETH_INVALID_MAP);
234
+
235
+
/* are we allocating a new buffer or recycling an old one */
236
+
if (pool->skbuff[index])
237
+
goto reuse;
238
+
230
239
skb = netdev_alloc_skb(adapter->netdev, pool->buff_size);
231
240
232
241
if (!skb) {
···
246
235
break;
247
236
}
248
237
249
-
free_index = pool->consumer_index;
250
-
pool->consumer_index++;
251
-
if (pool->consumer_index >= pool->size)
252
-
pool->consumer_index = 0;
253
-
index = pool->free_map[free_index];
254
-
255
-
BUG_ON(index == IBM_VETH_INVALID_MAP);
256
-
BUG_ON(pool->skbuff[index] != NULL);
257
-
258
238
dma_addr = dma_map_single(&adapter->vdev->dev, skb->data,
259
239
pool->buff_size, DMA_FROM_DEVICE);
260
240
261
241
if (dma_mapping_error(&adapter->vdev->dev, dma_addr))
262
242
goto failure;
263
243
264
-
pool->free_map[free_index] = IBM_VETH_INVALID_MAP;
265
244
pool->dma_addr[index] = dma_addr;
266
245
pool->skbuff[index] = skb;
267
246
268
-
correlator = ((u64)pool->index << 32) | index;
269
-
*(u64 *)skb->data = correlator;
270
-
247
+
if (rx_flush) {
248
+
unsigned int len = min(pool->buff_size,
249
+
adapter->netdev->mtu +
250
+
IBMVETH_BUFF_OH);
251
+
ibmveth_flush_buffer(skb->data, len);
252
+
}
253
+
reuse:
254
+
dma_addr = pool->dma_addr[index];
271
255
desc.fields.flags_len = IBMVETH_BUF_VALID | pool->buff_size;
272
256
desc.fields.address = dma_addr;
273
257
274
-
if (rx_flush) {
275
-
unsigned int len = min(pool->buff_size,
276
-
adapter->netdev->mtu +
277
-
IBMVETH_BUFF_OH);
278
-
ibmveth_flush_buffer(skb->data, len);
279
-
}
258
+
correlator = ((u64)pool->index << 32) | index;
259
+
*(u64 *)pool->skbuff[index]->data = correlator;
260
+
280
261
lpar_rc = h_add_logical_lan_buffer(adapter->vdev->unit_address,
281
262
desc.desc);
282
263
283
264
if (lpar_rc != H_SUCCESS) {
265
+
netdev_warn(adapter->netdev,
266
+
"%sadd_logical_lan failed %lu\n",
267
+
skb ? "" : "When recycling: ", lpar_rc);
284
268
goto failure;
285
-
} else {
286
-
buffers_added++;
287
-
adapter->replenish_add_buff_success++;
288
269
}
270
+
271
+
pool->free_map[free_index] = IBM_VETH_INVALID_MAP;
272
+
pool->consumer_index++;
273
+
if (pool->consumer_index >= pool->size)
274
+
pool->consumer_index = 0;
275
+
276
+
buffers_added++;
277
+
adapter->replenish_add_buff_success++;
289
278
}
290
279
291
280
mb();
···
293
282
return;
294
283
295
284
failure:
296
-
pool->free_map[free_index] = index;
297
-
pool->skbuff[index] = NULL;
298
-
if (pool->consumer_index == 0)
299
-
pool->consumer_index = pool->size - 1;
300
-
else
301
-
pool->consumer_index--;
302
-
if (!dma_mapping_error(&adapter->vdev->dev, dma_addr))
285
+
286
+
if (dma_addr && !dma_mapping_error(&adapter->vdev->dev, dma_addr))
303
287
dma_unmap_single(&adapter->vdev->dev,
304
288
pool->dma_addr[index], pool->buff_size,
305
289
DMA_FROM_DEVICE);
306
-
dev_kfree_skb_any(skb);
290
+
dev_kfree_skb_any(pool->skbuff[index]);
291
+
pool->skbuff[index] = NULL;
307
292
adapter->replenish_add_buff_failure++;
308
293
309
294
mb();
···
372
365
373
366
/* remove a buffer from a pool */
374
367
static void ibmveth_remove_buffer_from_pool(struct ibmveth_adapter *adapter,
375
-
u64 correlator)
368
+
u64 correlator, bool reuse)
376
369
{
377
370
unsigned int pool = correlator >> 32;
378
371
unsigned int index = correlator & 0xffffffffUL;
···
383
376
BUG_ON(index >= adapter->rx_buff_pool[pool].size);
384
377
385
378
skb = adapter->rx_buff_pool[pool].skbuff[index];
386
-
387
379
BUG_ON(skb == NULL);
388
380
389
-
adapter->rx_buff_pool[pool].skbuff[index] = NULL;
381
+
/* if we are going to reuse the buffer then keep the pointers around
382
+
* but mark index as available. replenish will see the skb pointer and
383
+
* assume it is to be recycled.
384
+
*/
385
+
if (!reuse) {
386
+
/* remove the skb pointer to mark free. actual freeing is done
387
+
* by upper level networking after gro_recieve
388
+
*/
389
+
adapter->rx_buff_pool[pool].skbuff[index] = NULL;
390
390
391
-
dma_unmap_single(&adapter->vdev->dev,
392
-
adapter->rx_buff_pool[pool].dma_addr[index],
393
-
adapter->rx_buff_pool[pool].buff_size,
394
-
DMA_FROM_DEVICE);
391
+
dma_unmap_single(&adapter->vdev->dev,
392
+
adapter->rx_buff_pool[pool].dma_addr[index],
393
+
adapter->rx_buff_pool[pool].buff_size,
394
+
DMA_FROM_DEVICE);
395
+
}
395
396
396
397
free_index = adapter->rx_buff_pool[pool].producer_index;
397
398
adapter->rx_buff_pool[pool].producer_index++;
···
426
411
return adapter->rx_buff_pool[pool].skbuff[index];
427
412
}
428
413
429
-
/* recycle the current buffer on the rx queue */
430
-
static int ibmveth_rxq_recycle_buffer(struct ibmveth_adapter *adapter)
414
+
static void ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter,
415
+
bool reuse)
431
416
{
432
-
u32 q_index = adapter->rx_queue.index;
433
-
u64 correlator = adapter->rx_queue.queue_addr[q_index].correlator;
434
-
unsigned int pool = correlator >> 32;
435
-
unsigned int index = correlator & 0xffffffffUL;
436
-
union ibmveth_buf_desc desc;
437
-
unsigned long lpar_rc;
438
-
int ret = 1;
417
+
u64 cor;
439
418
440
-
BUG_ON(pool >= IBMVETH_NUM_BUFF_POOLS);
441
-
BUG_ON(index >= adapter->rx_buff_pool[pool].size);
442
-
443
-
if (!adapter->rx_buff_pool[pool].active) {
444
-
ibmveth_rxq_harvest_buffer(adapter);
445
-
ibmveth_free_buffer_pool(adapter, &adapter->rx_buff_pool[pool]);
446
-
goto out;
447
-
}
448
-
449
-
desc.fields.flags_len = IBMVETH_BUF_VALID |
450
-
adapter->rx_buff_pool[pool].buff_size;
451
-
desc.fields.address = adapter->rx_buff_pool[pool].dma_addr[index];
452
-
453
-
lpar_rc = h_add_logical_lan_buffer(adapter->vdev->unit_address, desc.desc);
454
-
455
-
if (lpar_rc != H_SUCCESS) {
456
-
netdev_dbg(adapter->netdev, "h_add_logical_lan_buffer failed "
457
-
"during recycle rc=%ld", lpar_rc);
458
-
ibmveth_remove_buffer_from_pool(adapter, adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator);
459
-
ret = 0;
460
-
}
461
-
462
-
if (++adapter->rx_queue.index == adapter->rx_queue.num_slots) {
463
-
adapter->rx_queue.index = 0;
464
-
adapter->rx_queue.toggle = !adapter->rx_queue.toggle;
465
-
}
466
-
467
-
out:
468
-
return ret;
469
-
}
470
-
471
-
static void ibmveth_rxq_harvest_buffer(struct ibmveth_adapter *adapter)
472
-
{
473
-
ibmveth_remove_buffer_from_pool(adapter, adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator);
419
+
cor = adapter->rx_queue.queue_addr[adapter->rx_queue.index].correlator;
420
+
ibmveth_remove_buffer_from_pool(adapter, cor, reuse);
474
421
475
422
if (++adapter->rx_queue.index == adapter->rx_queue.num_slots) {
476
423
adapter->rx_queue.index = 0;
···
1314
1337
unsigned long lpar_rc;
1315
1338
u16 mss = 0;
1316
1339
1340
+
restart_poll:
1317
1341
while (frames_processed < budget) {
1318
1342
if (!ibmveth_rxq_pending_buffer(adapter))
1319
1343
break;
···
1324
1346
wmb(); /* suggested by larson1 */
1325
1347
adapter->rx_invalid_buffer++;
1326
1348
netdev_dbg(netdev, "recycling invalid buffer\n");
1327
-
ibmveth_rxq_recycle_buffer(adapter);
1349
+
ibmveth_rxq_harvest_buffer(adapter, true);
1328
1350
} else {
1329
1351
struct sk_buff *skb, *new_skb;
1330
1352
int length = ibmveth_rxq_frame_length(adapter);
···
1357
1379
if (rx_flush)
1358
1380
ibmveth_flush_buffer(skb->data,
1359
1381
length + offset);
1360
-
if (!ibmveth_rxq_recycle_buffer(adapter))
1361
-
kfree_skb(skb);
1382
+
ibmveth_rxq_harvest_buffer(adapter, true);
1362
1383
skb = new_skb;
1363
1384
} else {
1364
-
ibmveth_rxq_harvest_buffer(adapter);
1385
+
ibmveth_rxq_harvest_buffer(adapter, false);
1365
1386
skb_reserve(skb, offset);
1366
1387
}
1367
1388
···
1397
1420
1398
1421
ibmveth_replenish_task(adapter);
1399
1422
1400
-
if (frames_processed < budget) {
1401
-
napi_complete_done(napi, frames_processed);
1423
+
if (frames_processed == budget)
1424
+
goto out;
1402
1425
1403
-
/* We think we are done - reenable interrupts,
1404
-
* then check once more to make sure we are done.
1405
-
*/
1426
+
if (!napi_complete_done(napi, frames_processed))
1427
+
goto out;
1428
+
1429
+
/* We think we are done - reenable interrupts,
1430
+
* then check once more to make sure we are done.
1431
+
*/
1432
+
lpar_rc = h_vio_signal(adapter->vdev->unit_address, VIO_IRQ_ENABLE);
1433
+
BUG_ON(lpar_rc != H_SUCCESS);
1434
+
1435
+
if (ibmveth_rxq_pending_buffer(adapter) && napi_schedule(napi)) {
1406
1436
lpar_rc = h_vio_signal(adapter->vdev->unit_address,
1407
-
VIO_IRQ_ENABLE);
1408
-
1409
-
BUG_ON(lpar_rc != H_SUCCESS);
1410
-
1411
-
if (ibmveth_rxq_pending_buffer(adapter) &&
1412
-
napi_schedule(napi)) {
1413
-
lpar_rc = h_vio_signal(adapter->vdev->unit_address,
1414
-
VIO_IRQ_DISABLE);
1415
-
}
1437
+
VIO_IRQ_DISABLE);
1438
+
goto restart_poll;
1416
1439
}
1417
1440
1441
+
out:
1418
1442
return frames_processed;
1419
1443
}
1420
1444