Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
gve: Implement queue api
The new netdev queue api is implemented for gve.
Tested-by: Mina Almasry <almasrymina@google.com>
Reviewed-by: Mina Almasry <almasrymina@google.com>
Reviewed-by: Praveen Kaligineedi <pkaligineedi@google.com>
Reviewed-by: Harshitha Ramamurthy <hramamurthy@google.com>
Signed-off-by: Shailend Chand <shailend@google.com>
Link: https://lore.kernel.org/all/20240501232549.1327174-11-shailend@google.com/
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
authored by
Shailend Chand
and committed by
Jakub Kicinski
c93462b9
8c4e4798
+189
-24
+6
drivers/net/ethernet/google/gve/gve.h
+6
drivers/net/ethernet/google/gve/gve.h
···
1096
1096
void gve_rx_write_doorbell(struct gve_priv *priv, struct gve_rx_ring *rx);
1097
1097
int gve_rx_poll(struct gve_notify_block *block, int budget);
1098
1098
bool gve_rx_work_pending(struct gve_rx_ring *rx);
1099
+
int gve_rx_alloc_ring_gqi(struct gve_priv *priv,
1100
+
struct gve_rx_alloc_rings_cfg *cfg,
1101
+
struct gve_rx_ring *rx,
1102
+
int idx);
1103
+
void gve_rx_free_ring_gqi(struct gve_priv *priv, struct gve_rx_ring *rx,
1104
+
struct gve_rx_alloc_rings_cfg *cfg);
1099
1105
int gve_rx_alloc_rings(struct gve_priv *priv);
1100
1106
int gve_rx_alloc_rings_gqi(struct gve_priv *priv,
1101
1107
struct gve_rx_alloc_rings_cfg *cfg);
+6
drivers/net/ethernet/google/gve/gve_dqo.h
+6
drivers/net/ethernet/google/gve/gve_dqo.h
···
44
44
struct gve_tx_alloc_rings_cfg *cfg);
45
45
void gve_tx_start_ring_dqo(struct gve_priv *priv, int idx);
46
46
void gve_tx_stop_ring_dqo(struct gve_priv *priv, int idx);
47
+
int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
48
+
struct gve_rx_alloc_rings_cfg *cfg,
49
+
struct gve_rx_ring *rx,
50
+
int idx);
51
+
void gve_rx_free_ring_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
52
+
struct gve_rx_alloc_rings_cfg *cfg);
47
53
int gve_rx_alloc_rings_dqo(struct gve_priv *priv,
48
54
struct gve_rx_alloc_rings_cfg *cfg);
49
55
void gve_rx_free_rings_dqo(struct gve_priv *priv,
+165
-12
drivers/net/ethernet/google/gve/gve_main.c
+165
-12
drivers/net/ethernet/google/gve/gve_main.c
···
17
17
#include <linux/workqueue.h>
18
18
#include <linux/utsname.h>
19
19
#include <linux/version.h>
20
+
#include <net/netdev_queues.h>
20
21
#include <net/sch_generic.h>
21
22
#include <net/xdp_sock_drv.h>
22
23
#include "gve.h"
···
1239
1238
gve_rx_get_curr_alloc_cfg(priv, rx_alloc_cfg);
1240
1239
}
1241
1240
1241
+
static void gve_rx_start_ring(struct gve_priv *priv, int i)
1242
+
{
1243
+
if (gve_is_gqi(priv))
1244
+
gve_rx_start_ring_gqi(priv, i);
1245
+
else
1246
+
gve_rx_start_ring_dqo(priv, i);
1247
+
}
1248
+
1242
1249
static void gve_rx_start_rings(struct gve_priv *priv, int num_rings)
1243
1250
{
1244
1251
int i;
1245
1252
1246
-
for (i = 0; i < num_rings; i++) {
1247
-
if (gve_is_gqi(priv))
1248
-
gve_rx_start_ring_gqi(priv, i);
1249
-
else
1250
-
gve_rx_start_ring_dqo(priv, i);
1251
-
}
1253
+
for (i = 0; i < num_rings; i++)
1254
+
gve_rx_start_ring(priv, i);
1255
+
}
1256
+
1257
+
static void gve_rx_stop_ring(struct gve_priv *priv, int i)
1258
+
{
1259
+
if (gve_is_gqi(priv))
1260
+
gve_rx_stop_ring_gqi(priv, i);
1261
+
else
1262
+
gve_rx_stop_ring_dqo(priv, i);
1252
1263
}
1253
1264
1254
1265
static void gve_rx_stop_rings(struct gve_priv *priv, int num_rings)
···
1270
1257
if (!priv->rx)
1271
1258
return;
1272
1259
1273
-
for (i = 0; i < num_rings; i++) {
1274
-
if (gve_is_gqi(priv))
1275
-
gve_rx_stop_ring_gqi(priv, i);
1276
-
else
1277
-
gve_rx_stop_ring_dqo(priv, i);
1278
-
}
1260
+
for (i = 0; i < num_rings; i++)
1261
+
gve_rx_stop_ring(priv, i);
1279
1262
}
1280
1263
1281
1264
static void gve_queues_mem_remove(struct gve_priv *priv)
···
1886
1877
gve_set_napi_enabled(priv);
1887
1878
}
1888
1879
1880
+
static void gve_turnup_and_check_status(struct gve_priv *priv)
1881
+
{
1882
+
u32 status;
1883
+
1884
+
gve_turnup(priv);
1885
+
status = ioread32be(&priv->reg_bar0->device_status);
1886
+
gve_handle_link_status(priv, GVE_DEVICE_STATUS_LINK_STATUS_MASK & status);
1887
+
}
1888
+
1889
1889
static void gve_tx_timeout(struct net_device *dev, unsigned int txqueue)
1890
1890
{
1891
1891
struct gve_notify_block *block;
···
2341
2323
writeb('\n', driver_version_register);
2342
2324
}
2343
2325
2326
+
static int gve_rx_queue_stop(struct net_device *dev, void *per_q_mem, int idx)
2327
+
{
2328
+
struct gve_priv *priv = netdev_priv(dev);
2329
+
struct gve_rx_ring *gve_per_q_mem;
2330
+
int err;
2331
+
2332
+
if (!priv->rx)
2333
+
return -EAGAIN;
2334
+
2335
+
/* Destroying queue 0 while other queues exist is not supported in DQO */
2336
+
if (!gve_is_gqi(priv) && idx == 0)
2337
+
return -ERANGE;
2338
+
2339
+
/* Single-queue destruction requires quiescence on all queues */
2340
+
gve_turndown(priv);
2341
+
2342
+
/* This failure will trigger a reset - no need to clean up */
2343
+
err = gve_adminq_destroy_single_rx_queue(priv, idx);
2344
+
if (err)
2345
+
return err;
2346
+
2347
+
if (gve_is_qpl(priv)) {
2348
+
/* This failure will trigger a reset - no need to clean up */
2349
+
err = gve_unregister_qpl(priv, gve_rx_get_qpl(priv, idx));
2350
+
if (err)
2351
+
return err;
2352
+
}
2353
+
2354
+
gve_rx_stop_ring(priv, idx);
2355
+
2356
+
/* Turn the unstopped queues back up */
2357
+
gve_turnup_and_check_status(priv);
2358
+
2359
+
gve_per_q_mem = (struct gve_rx_ring *)per_q_mem;
2360
+
*gve_per_q_mem = priv->rx[idx];
2361
+
memset(&priv->rx[idx], 0, sizeof(priv->rx[idx]));
2362
+
return 0;
2363
+
}
2364
+
2365
+
static void gve_rx_queue_mem_free(struct net_device *dev, void *per_q_mem)
2366
+
{
2367
+
struct gve_priv *priv = netdev_priv(dev);
2368
+
struct gve_rx_alloc_rings_cfg cfg = {0};
2369
+
struct gve_rx_ring *gve_per_q_mem;
2370
+
2371
+
gve_per_q_mem = (struct gve_rx_ring *)per_q_mem;
2372
+
gve_rx_get_curr_alloc_cfg(priv, &cfg);
2373
+
2374
+
if (gve_is_gqi(priv))
2375
+
gve_rx_free_ring_gqi(priv, gve_per_q_mem, &cfg);
2376
+
else
2377
+
gve_rx_free_ring_dqo(priv, gve_per_q_mem, &cfg);
2378
+
}
2379
+
2380
+
static int gve_rx_queue_mem_alloc(struct net_device *dev, void *per_q_mem,
2381
+
int idx)
2382
+
{
2383
+
struct gve_priv *priv = netdev_priv(dev);
2384
+
struct gve_rx_alloc_rings_cfg cfg = {0};
2385
+
struct gve_rx_ring *gve_per_q_mem;
2386
+
int err;
2387
+
2388
+
if (!priv->rx)
2389
+
return -EAGAIN;
2390
+
2391
+
gve_per_q_mem = (struct gve_rx_ring *)per_q_mem;
2392
+
gve_rx_get_curr_alloc_cfg(priv, &cfg);
2393
+
2394
+
if (gve_is_gqi(priv))
2395
+
err = gve_rx_alloc_ring_gqi(priv, &cfg, gve_per_q_mem, idx);
2396
+
else
2397
+
err = gve_rx_alloc_ring_dqo(priv, &cfg, gve_per_q_mem, idx);
2398
+
2399
+
return err;
2400
+
}
2401
+
2402
+
static int gve_rx_queue_start(struct net_device *dev, void *per_q_mem, int idx)
2403
+
{
2404
+
struct gve_priv *priv = netdev_priv(dev);
2405
+
struct gve_rx_ring *gve_per_q_mem;
2406
+
int err;
2407
+
2408
+
if (!priv->rx)
2409
+
return -EAGAIN;
2410
+
2411
+
gve_per_q_mem = (struct gve_rx_ring *)per_q_mem;
2412
+
priv->rx[idx] = *gve_per_q_mem;
2413
+
2414
+
/* Single-queue creation requires quiescence on all queues */
2415
+
gve_turndown(priv);
2416
+
2417
+
gve_rx_start_ring(priv, idx);
2418
+
2419
+
if (gve_is_qpl(priv)) {
2420
+
/* This failure will trigger a reset - no need to clean up */
2421
+
err = gve_register_qpl(priv, gve_rx_get_qpl(priv, idx));
2422
+
if (err)
2423
+
goto abort;
2424
+
}
2425
+
2426
+
/* This failure will trigger a reset - no need to clean up */
2427
+
err = gve_adminq_create_single_rx_queue(priv, idx);
2428
+
if (err)
2429
+
goto abort;
2430
+
2431
+
if (gve_is_gqi(priv))
2432
+
gve_rx_write_doorbell(priv, &priv->rx[idx]);
2433
+
else
2434
+
gve_rx_post_buffers_dqo(&priv->rx[idx]);
2435
+
2436
+
/* Turn the unstopped queues back up */
2437
+
gve_turnup_and_check_status(priv);
2438
+
return 0;
2439
+
2440
+
abort:
2441
+
gve_rx_stop_ring(priv, idx);
2442
+
2443
+
/* All failures in this func result in a reset, by clearing the struct
2444
+
* at idx, we prevent a double free when that reset runs. The reset,
2445
+
* which needs the rtnl lock, will not run till this func returns and
2446
+
* its caller gives up the lock.
2447
+
*/
2448
+
memset(&priv->rx[idx], 0, sizeof(priv->rx[idx]));
2449
+
return err;
2450
+
}
2451
+
2452
+
static const struct netdev_queue_mgmt_ops gve_queue_mgmt_ops = {
2453
+
.ndo_queue_mem_size = sizeof(struct gve_rx_ring),
2454
+
.ndo_queue_mem_alloc = gve_rx_queue_mem_alloc,
2455
+
.ndo_queue_mem_free = gve_rx_queue_mem_free,
2456
+
.ndo_queue_start = gve_rx_queue_start,
2457
+
.ndo_queue_stop = gve_rx_queue_stop,
2458
+
};
2459
+
2344
2460
static int gve_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2345
2461
{
2346
2462
int max_tx_queues, max_rx_queues;
···
2529
2377
pci_set_drvdata(pdev, dev);
2530
2378
dev->ethtool_ops = &gve_ethtool_ops;
2531
2379
dev->netdev_ops = &gve_netdev_ops;
2380
+
dev->queue_mgmt_ops = &gve_queue_mgmt_ops;
2532
2381
2533
2382
/* Set default and supported features.
2534
2383
*
+6
-6
drivers/net/ethernet/google/gve/gve_rx.c
+6
-6
drivers/net/ethernet/google/gve/gve_rx.c
···
99
99
gve_rx_reset_ring_gqi(priv, idx);
100
100
}
101
101
102
-
static void gve_rx_free_ring_gqi(struct gve_priv *priv, struct gve_rx_ring *rx,
103
-
struct gve_rx_alloc_rings_cfg *cfg)
102
+
void gve_rx_free_ring_gqi(struct gve_priv *priv, struct gve_rx_ring *rx,
103
+
struct gve_rx_alloc_rings_cfg *cfg)
104
104
{
105
105
struct device *dev = &priv->pdev->dev;
106
106
u32 slots = rx->mask + 1;
···
264
264
gve_add_napi(priv, ntfy_idx, gve_napi_poll);
265
265
}
266
266
267
-
static int gve_rx_alloc_ring_gqi(struct gve_priv *priv,
268
-
struct gve_rx_alloc_rings_cfg *cfg,
269
-
struct gve_rx_ring *rx,
270
-
int idx)
267
+
int gve_rx_alloc_ring_gqi(struct gve_priv *priv,
268
+
struct gve_rx_alloc_rings_cfg *cfg,
269
+
struct gve_rx_ring *rx,
270
+
int idx)
271
271
{
272
272
struct device *hdev = &priv->pdev->dev;
273
273
u32 slots = cfg->ring_size;
+6
-6
drivers/net/ethernet/google/gve/gve_rx_dqo.c
+6
-6
drivers/net/ethernet/google/gve/gve_rx_dqo.c
···
299
299
gve_rx_reset_ring_dqo(priv, idx);
300
300
}
301
301
302
-
static void gve_rx_free_ring_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
303
-
struct gve_rx_alloc_rings_cfg *cfg)
302
+
void gve_rx_free_ring_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
303
+
struct gve_rx_alloc_rings_cfg *cfg)
304
304
{
305
305
struct device *hdev = &priv->pdev->dev;
306
306
size_t completion_queue_slots;
···
376
376
gve_add_napi(priv, ntfy_idx, gve_napi_poll_dqo);
377
377
}
378
378
379
-
static int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
380
-
struct gve_rx_alloc_rings_cfg *cfg,
381
-
struct gve_rx_ring *rx,
382
-
int idx)
379
+
int gve_rx_alloc_ring_dqo(struct gve_priv *priv,
380
+
struct gve_rx_alloc_rings_cfg *cfg,
381
+
struct gve_rx_ring *rx,
382
+
int idx)
383
383
{
384
384
struct device *hdev = &priv->pdev->dev;
385
385
int qpl_page_cnt;