Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'device-memory-tcp'
Prep patches for Device Memory TCP
Pick up a couple of prep patches for Device Memory TCP which
stand on their own.
Link: https://patch.msgid.link/20240628003253.1694510-1-almasrymina@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
+317
-187
+2
-2
include/linux/skbuff_ref.h
+2
-2
include/linux/skbuff_ref.h
···
32
32
__skb_frag_ref(&skb_shinfo(skb)->frags[f]);
33
33
}
34
34
35
-
bool napi_pp_put_page(struct page *page);
35
+
bool napi_pp_put_page(netmem_ref netmem);
36
36
37
37
static inline void
38
38
skb_page_unref(struct page *page, bool recycle)
39
39
{
40
40
#ifdef CONFIG_PAGE_POOL
41
-
if (recycle && napi_pp_put_page(page))
41
+
if (recycle && napi_pp_put_page(page_to_netmem(page)))
42
42
return;
43
43
#endif
44
44
put_page(page);
+15
include/net/netmem.h
+15
include/net/netmem.h
···
38
38
return (__force netmem_ref)page;
39
39
}
40
40
41
+
static inline int netmem_ref_count(netmem_ref netmem)
42
+
{
43
+
return page_ref_count(netmem_to_page(netmem));
44
+
}
45
+
46
+
static inline unsigned long netmem_to_pfn(netmem_ref netmem)
47
+
{
48
+
return page_to_pfn(netmem_to_page(netmem));
49
+
}
50
+
51
+
static inline netmem_ref netmem_compound_head(netmem_ref netmem)
52
+
{
53
+
return page_to_netmem(compound_head(netmem_to_page(netmem)));
54
+
}
55
+
41
56
#endif /* _NET_NETMEM_H */
+71
-22
include/net/page_pool/helpers.h
+71
-22
include/net/page_pool/helpers.h
···
55
55
#include <linux/dma-mapping.h>
56
56
57
57
#include <net/page_pool/types.h>
58
+
#include <net/net_debug.h>
59
+
#include <net/netmem.h>
58
60
59
61
#ifdef CONFIG_PAGE_POOL_STATS
60
62
/* Deprecated driver-facing API, use netlink instead */
···
214
212
return pool->p.dma_dir;
215
213
}
216
214
215
+
static inline void page_pool_fragment_netmem(netmem_ref netmem, long nr)
216
+
{
217
+
atomic_long_set(&netmem_to_page(netmem)->pp_ref_count, nr);
218
+
}
219
+
217
220
/**
218
221
* page_pool_fragment_page() - split a fresh page into fragments
219
222
* @page: page to split
···
239
232
*/
240
233
static inline void page_pool_fragment_page(struct page *page, long nr)
241
234
{
242
-
atomic_long_set(&page->pp_ref_count, nr);
235
+
page_pool_fragment_netmem(page_to_netmem(page), nr);
243
236
}
244
237
245
-
static inline long page_pool_unref_page(struct page *page, long nr)
238
+
static inline long page_pool_unref_netmem(netmem_ref netmem, long nr)
246
239
{
240
+
struct page *page = netmem_to_page(netmem);
247
241
long ret;
248
242
249
243
/* If nr == pp_ref_count then we have cleared all remaining
···
287
279
return ret;
288
280
}
289
281
290
-
static inline void page_pool_ref_page(struct page *page)
282
+
static inline long page_pool_unref_page(struct page *page, long nr)
291
283
{
292
-
atomic_long_inc(&page->pp_ref_count);
284
+
return page_pool_unref_netmem(page_to_netmem(page), nr);
293
285
}
294
286
295
-
static inline bool page_pool_is_last_ref(struct page *page)
287
+
static inline void page_pool_ref_netmem(netmem_ref netmem)
288
+
{
289
+
atomic_long_inc(&netmem_to_page(netmem)->pp_ref_count);
290
+
}
291
+
292
+
static inline void page_pool_ref_page(struct page *page)
293
+
{
294
+
page_pool_ref_netmem(page_to_netmem(page));
295
+
}
296
+
297
+
static inline bool page_pool_is_last_ref(netmem_ref netmem)
296
298
{
297
299
/* If page_pool_unref_page() returns 0, we were the last user */
298
-
return page_pool_unref_page(page, 1) == 0;
300
+
return page_pool_unref_netmem(netmem, 1) == 0;
301
+
}
302
+
303
+
static inline void page_pool_put_netmem(struct page_pool *pool,
304
+
netmem_ref netmem,
305
+
unsigned int dma_sync_size,
306
+
bool allow_direct)
307
+
{
308
+
/* When page_pool isn't compiled-in, net/core/xdp.c doesn't
309
+
* allow registering MEM_TYPE_PAGE_POOL, but shield linker.
310
+
*/
311
+
#ifdef CONFIG_PAGE_POOL
312
+
if (!page_pool_is_last_ref(netmem))
313
+
return;
314
+
315
+
page_pool_put_unrefed_netmem(pool, netmem, dma_sync_size, allow_direct);
316
+
#endif
299
317
}
300
318
301
319
/**
···
342
308
unsigned int dma_sync_size,
343
309
bool allow_direct)
344
310
{
345
-
/* When page_pool isn't compiled-in, net/core/xdp.c doesn't
346
-
* allow registering MEM_TYPE_PAGE_POOL, but shield linker.
347
-
*/
348
-
#ifdef CONFIG_PAGE_POOL
349
-
if (!page_pool_is_last_ref(page))
350
-
return;
311
+
page_pool_put_netmem(pool, page_to_netmem(page), dma_sync_size,
312
+
allow_direct);
313
+
}
351
314
352
-
page_pool_put_unrefed_page(pool, page, dma_sync_size, allow_direct);
353
-
#endif
315
+
static inline void page_pool_put_full_netmem(struct page_pool *pool,
316
+
netmem_ref netmem,
317
+
bool allow_direct)
318
+
{
319
+
page_pool_put_netmem(pool, netmem, -1, allow_direct);
354
320
}
355
321
356
322
/**
···
365
331
static inline void page_pool_put_full_page(struct page_pool *pool,
366
332
struct page *page, bool allow_direct)
367
333
{
368
-
page_pool_put_page(pool, page, -1, allow_direct);
334
+
page_pool_put_netmem(pool, page_to_netmem(page), -1, allow_direct);
369
335
}
370
336
371
337
/**
···
399
365
page_pool_put_page(pool, virt_to_head_page(va), -1, allow_direct);
400
366
}
401
367
368
+
static inline dma_addr_t page_pool_get_dma_addr_netmem(netmem_ref netmem)
369
+
{
370
+
struct page *page = netmem_to_page(netmem);
371
+
372
+
dma_addr_t ret = page->dma_addr;
373
+
374
+
if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA)
375
+
ret <<= PAGE_SHIFT;
376
+
377
+
return ret;
378
+
}
379
+
402
380
/**
403
381
* page_pool_get_dma_addr() - Retrieve the stored DMA address.
404
382
* @page: page allocated from a page pool
···
420
374
*/
421
375
static inline dma_addr_t page_pool_get_dma_addr(const struct page *page)
422
376
{
423
-
dma_addr_t ret = page->dma_addr;
424
-
425
-
if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA)
426
-
ret <<= PAGE_SHIFT;
427
-
428
-
return ret;
377
+
return page_pool_get_dma_addr_netmem(page_to_netmem((struct page *)page));
429
378
}
430
379
431
-
static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr)
380
+
static inline bool page_pool_set_dma_addr_netmem(netmem_ref netmem,
381
+
dma_addr_t addr)
432
382
{
383
+
struct page *page = netmem_to_page(netmem);
384
+
433
385
if (PAGE_POOL_32BIT_ARCH_WITH_64BIT_DMA) {
434
386
page->dma_addr = addr >> PAGE_SHIFT;
435
387
···
461
417
page_pool_get_dma_addr(page),
462
418
offset + pool->p.offset, dma_sync_size,
463
419
page_pool_get_dma_dir(pool));
420
+
}
421
+
422
+
static inline bool page_pool_set_dma_addr(struct page *page, dma_addr_t addr)
423
+
{
424
+
return page_pool_set_dma_addr_netmem(page_to_netmem(page), addr);
464
425
}
465
426
466
427
static inline bool page_pool_put(struct page_pool *pool)
+11
-3
include/net/page_pool/types.h
+11
-3
include/net/page_pool/types.h
···
6
6
#include <linux/dma-direction.h>
7
7
#include <linux/ptr_ring.h>
8
8
#include <linux/types.h>
9
+
#include <net/netmem.h>
9
10
10
11
#define PP_FLAG_DMA_MAP BIT(0) /* Should page_pool do the DMA
11
12
* map/unmap
···
41
40
#define PP_ALLOC_CACHE_REFILL 64
42
41
struct pp_alloc_cache {
43
42
u32 count;
44
-
struct page *cache[PP_ALLOC_CACHE_SIZE];
43
+
netmem_ref cache[PP_ALLOC_CACHE_SIZE];
45
44
};
46
45
47
46
/**
···
74
73
struct net_device *netdev;
75
74
unsigned int flags;
76
75
/* private: used by test code only */
77
-
void (*init_callback)(struct page *page, void *arg);
76
+
void (*init_callback)(netmem_ref netmem, void *arg);
78
77
void *init_arg;
79
78
);
80
79
};
···
152
151
*/
153
152
__cacheline_group_begin(frag) __aligned(4 * sizeof(long));
154
153
long frag_users;
155
-
struct page *frag_page;
154
+
netmem_ref frag_page;
156
155
unsigned int frag_offset;
157
156
__cacheline_group_end(frag);
158
157
···
221
220
};
222
221
223
222
struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);
223
+
netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp);
224
224
struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
225
225
unsigned int size, gfp_t gfp);
226
+
netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool,
227
+
unsigned int *offset, unsigned int size,
228
+
gfp_t gfp);
226
229
struct page_pool *page_pool_create(const struct page_pool_params *params);
227
230
struct page_pool *page_pool_create_percpu(const struct page_pool_params *params,
228
231
int cpuid);
···
257
252
}
258
253
#endif
259
254
255
+
void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem,
256
+
unsigned int dma_sync_size,
257
+
bool allow_direct);
260
258
void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
261
259
unsigned int dma_sync_size,
262
260
bool allow_direct);
+16
-14
include/trace/events/page_pool.h
+16
-14
include/trace/events/page_pool.h
···
42
42
TRACE_EVENT(page_pool_state_release,
43
43
44
44
TP_PROTO(const struct page_pool *pool,
45
-
const struct page *page, u32 release),
45
+
netmem_ref netmem, u32 release),
46
46
47
-
TP_ARGS(pool, page, release),
47
+
TP_ARGS(pool, netmem, release),
48
48
49
49
TP_STRUCT__entry(
50
50
__field(const struct page_pool *, pool)
51
-
__field(const struct page *, page)
51
+
__field(unsigned long, netmem)
52
52
__field(u32, release)
53
53
__field(unsigned long, pfn)
54
54
),
55
55
56
56
TP_fast_assign(
57
57
__entry->pool = pool;
58
-
__entry->page = page;
58
+
__entry->netmem = (__force unsigned long)netmem;
59
59
__entry->release = release;
60
-
__entry->pfn = page_to_pfn(page);
60
+
__entry->pfn = netmem_to_pfn(netmem);
61
61
),
62
62
63
-
TP_printk("page_pool=%p page=%p pfn=0x%lx release=%u",
64
-
__entry->pool, __entry->page, __entry->pfn, __entry->release)
63
+
TP_printk("page_pool=%p netmem=%p pfn=0x%lx release=%u",
64
+
__entry->pool, (void *)__entry->netmem,
65
+
__entry->pfn, __entry->release)
65
66
);
66
67
67
68
TRACE_EVENT(page_pool_state_hold,
68
69
69
70
TP_PROTO(const struct page_pool *pool,
70
-
const struct page *page, u32 hold),
71
+
netmem_ref netmem, u32 hold),
71
72
72
-
TP_ARGS(pool, page, hold),
73
+
TP_ARGS(pool, netmem, hold),
73
74
74
75
TP_STRUCT__entry(
75
76
__field(const struct page_pool *, pool)
76
-
__field(const struct page *, page)
77
+
__field(unsigned long, netmem)
77
78
__field(u32, hold)
78
79
__field(unsigned long, pfn)
79
80
),
80
81
81
82
TP_fast_assign(
82
83
__entry->pool = pool;
83
-
__entry->page = page;
84
+
__entry->netmem = (__force unsigned long)netmem;
84
85
__entry->hold = hold;
85
-
__entry->pfn = page_to_pfn(page);
86
+
__entry->pfn = netmem_to_pfn(netmem);
86
87
),
87
88
88
-
TP_printk("page_pool=%p page=%p pfn=0x%lx hold=%u",
89
-
__entry->pool, __entry->page, __entry->pfn, __entry->hold)
89
+
TP_printk("page_pool=%p netmem=%p pfn=0x%lx hold=%u",
90
+
__entry->pool, (void *)__entry->netmem,
91
+
__entry->pfn, __entry->hold)
90
92
);
91
93
92
94
TRACE_EVENT(page_pool_update_nid,
+3
-2
net/bpf/test_run.c
+3
-2
net/bpf/test_run.c
···
127
127
#define TEST_XDP_FRAME_SIZE (PAGE_SIZE - sizeof(struct xdp_page_head))
128
128
#define TEST_XDP_MAX_BATCH 256
129
129
130
-
static void xdp_test_run_init_page(struct page *page, void *arg)
130
+
static void xdp_test_run_init_page(netmem_ref netmem, void *arg)
131
131
{
132
-
struct xdp_page_head *head = phys_to_virt(page_to_phys(page));
132
+
struct xdp_page_head *head =
133
+
phys_to_virt(page_to_phys(netmem_to_page(netmem)));
133
134
struct xdp_buff *new_ctx, *orig_ctx;
134
135
u32 headroom = XDP_PACKET_HEADROOM;
135
136
struct xdp_test_data *xdp = arg;
+165
-139
net/core/page_pool.c
+165
-139
net/core/page_pool.c
···
327
327
}
328
328
EXPORT_SYMBOL(page_pool_create);
329
329
330
-
static void page_pool_return_page(struct page_pool *pool, struct page *page);
330
+
static void page_pool_return_page(struct page_pool *pool, netmem_ref netmem);
331
331
332
-
noinline
333
-
static struct page *page_pool_refill_alloc_cache(struct page_pool *pool)
332
+
static noinline netmem_ref page_pool_refill_alloc_cache(struct page_pool *pool)
334
333
{
335
334
struct ptr_ring *r = &pool->ring;
336
-
struct page *page;
335
+
netmem_ref netmem;
337
336
int pref_nid; /* preferred NUMA node */
338
337
339
338
/* Quicker fallback, avoid locks when ring is empty */
340
339
if (__ptr_ring_empty(r)) {
341
340
alloc_stat_inc(pool, empty);
342
-
return NULL;
341
+
return 0;
343
342
}
344
343
345
344
/* Softirq guarantee CPU and thus NUMA node is stable. This,
···
353
354
354
355
/* Refill alloc array, but only if NUMA match */
355
356
do {
356
-
page = __ptr_ring_consume(r);
357
-
if (unlikely(!page))
357
+
netmem = (__force netmem_ref)__ptr_ring_consume(r);
358
+
if (unlikely(!netmem))
358
359
break;
359
360
360
-
if (likely(page_to_nid(page) == pref_nid)) {
361
-
pool->alloc.cache[pool->alloc.count++] = page;
361
+
if (likely(page_to_nid(netmem_to_page(netmem)) == pref_nid)) {
362
+
pool->alloc.cache[pool->alloc.count++] = netmem;
362
363
} else {
363
364
/* NUMA mismatch;
364
365
* (1) release 1 page to page-allocator and
365
366
* (2) break out to fallthrough to alloc_pages_node.
366
367
* This limit stress on page buddy alloactor.
367
368
*/
368
-
page_pool_return_page(pool, page);
369
+
page_pool_return_page(pool, netmem);
369
370
alloc_stat_inc(pool, waive);
370
-
page = NULL;
371
+
netmem = 0;
371
372
break;
372
373
}
373
374
} while (pool->alloc.count < PP_ALLOC_CACHE_REFILL);
374
375
375
376
/* Return last page */
376
377
if (likely(pool->alloc.count > 0)) {
377
-
page = pool->alloc.cache[--pool->alloc.count];
378
+
netmem = pool->alloc.cache[--pool->alloc.count];
378
379
alloc_stat_inc(pool, refill);
379
380
}
380
381
381
-
return page;
382
+
return netmem;
382
383
}
383
384
384
385
/* fast path */
385
-
static struct page *__page_pool_get_cached(struct page_pool *pool)
386
+
static netmem_ref __page_pool_get_cached(struct page_pool *pool)
386
387
{
387
-
struct page *page;
388
+
netmem_ref netmem;
388
389
389
390
/* Caller MUST guarantee safe non-concurrent access, e.g. softirq */
390
391
if (likely(pool->alloc.count)) {
391
392
/* Fast-path */
392
-
page = pool->alloc.cache[--pool->alloc.count];
393
+
netmem = pool->alloc.cache[--pool->alloc.count];
393
394
alloc_stat_inc(pool, fast);
394
395
} else {
395
-
page = page_pool_refill_alloc_cache(pool);
396
+
netmem = page_pool_refill_alloc_cache(pool);
396
397
}
397
398
398
-
return page;
399
+
return netmem;
399
400
}
400
401
401
402
static void __page_pool_dma_sync_for_device(const struct page_pool *pool,
402
-
const struct page *page,
403
+
netmem_ref netmem,
403
404
u32 dma_sync_size)
404
405
{
405
406
#if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC)
406
-
dma_addr_t dma_addr = page_pool_get_dma_addr(page);
407
+
dma_addr_t dma_addr = page_pool_get_dma_addr_netmem(netmem);
407
408
408
409
dma_sync_size = min(dma_sync_size, pool->p.max_len);
409
410
__dma_sync_single_for_device(pool->p.dev, dma_addr + pool->p.offset,
···
413
414
414
415
static __always_inline void
415
416
page_pool_dma_sync_for_device(const struct page_pool *pool,
416
-
const struct page *page,
417
+
netmem_ref netmem,
417
418
u32 dma_sync_size)
418
419
{
419
420
if (pool->dma_sync && dma_dev_need_sync(pool->p.dev))
420
-
__page_pool_dma_sync_for_device(pool, page, dma_sync_size);
421
+
__page_pool_dma_sync_for_device(pool, netmem, dma_sync_size);
421
422
}
422
423
423
-
static bool page_pool_dma_map(struct page_pool *pool, struct page *page)
424
+
static bool page_pool_dma_map(struct page_pool *pool, netmem_ref netmem)
424
425
{
425
426
dma_addr_t dma;
426
427
···
429
430
* into page private data (i.e 32bit cpu with 64bit DMA caps)
430
431
* This mapping is kept for lifetime of page, until leaving pool.
431
432
*/
432
-
dma = dma_map_page_attrs(pool->p.dev, page, 0,
433
-
(PAGE_SIZE << pool->p.order),
434
-
pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC |
435
-
DMA_ATTR_WEAK_ORDERING);
433
+
dma = dma_map_page_attrs(pool->p.dev, netmem_to_page(netmem), 0,
434
+
(PAGE_SIZE << pool->p.order), pool->p.dma_dir,
435
+
DMA_ATTR_SKIP_CPU_SYNC |
436
+
DMA_ATTR_WEAK_ORDERING);
436
437
if (dma_mapping_error(pool->p.dev, dma))
437
438
return false;
438
439
439
-
if (page_pool_set_dma_addr(page, dma))
440
+
if (page_pool_set_dma_addr_netmem(netmem, dma))
440
441
goto unmap_failed;
441
442
442
-
page_pool_dma_sync_for_device(pool, page, pool->p.max_len);
443
+
page_pool_dma_sync_for_device(pool, netmem, pool->p.max_len);
443
444
444
445
return true;
445
446
···
451
452
return false;
452
453
}
453
454
454
-
static void page_pool_set_pp_info(struct page_pool *pool,
455
-
struct page *page)
455
+
static void page_pool_set_pp_info(struct page_pool *pool, netmem_ref netmem)
456
456
{
457
+
struct page *page = netmem_to_page(netmem);
458
+
457
459
page->pp = pool;
458
460
page->pp_magic |= PP_SIGNATURE;
459
461
···
464
464
* is dirtying the same cache line as the page->pp_magic above, so
465
465
* the overhead is negligible.
466
466
*/
467
-
page_pool_fragment_page(page, 1);
467
+
page_pool_fragment_netmem(netmem, 1);
468
468
if (pool->has_init_callback)
469
-
pool->slow.init_callback(page, pool->slow.init_arg);
469
+
pool->slow.init_callback(netmem, pool->slow.init_arg);
470
470
}
471
471
472
-
static void page_pool_clear_pp_info(struct page *page)
472
+
static void page_pool_clear_pp_info(netmem_ref netmem)
473
473
{
474
+
struct page *page = netmem_to_page(netmem);
475
+
474
476
page->pp_magic = 0;
475
477
page->pp = NULL;
476
478
}
···
487
485
if (unlikely(!page))
488
486
return NULL;
489
487
490
-
if (pool->dma_map && unlikely(!page_pool_dma_map(pool, page))) {
488
+
if (pool->dma_map && unlikely(!page_pool_dma_map(pool, page_to_netmem(page)))) {
491
489
put_page(page);
492
490
return NULL;
493
491
}
494
492
495
493
alloc_stat_inc(pool, slow_high_order);
496
-
page_pool_set_pp_info(pool, page);
494
+
page_pool_set_pp_info(pool, page_to_netmem(page));
497
495
498
496
/* Track how many pages are held 'in-flight' */
499
497
pool->pages_state_hold_cnt++;
500
-
trace_page_pool_state_hold(pool, page, pool->pages_state_hold_cnt);
498
+
trace_page_pool_state_hold(pool, page_to_netmem(page),
499
+
pool->pages_state_hold_cnt);
501
500
return page;
502
501
}
503
502
504
503
/* slow path */
505
-
noinline
506
-
static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
507
-
gfp_t gfp)
504
+
static noinline netmem_ref __page_pool_alloc_pages_slow(struct page_pool *pool,
505
+
gfp_t gfp)
508
506
{
509
507
const int bulk = PP_ALLOC_CACHE_REFILL;
510
508
unsigned int pp_order = pool->p.order;
511
509
bool dma_map = pool->dma_map;
512
-
struct page *page;
510
+
netmem_ref netmem;
513
511
int i, nr_pages;
514
512
515
513
/* Don't support bulk alloc for high-order pages */
516
514
if (unlikely(pp_order))
517
-
return __page_pool_alloc_page_order(pool, gfp);
515
+
return page_to_netmem(__page_pool_alloc_page_order(pool, gfp));
518
516
519
517
/* Unnecessary as alloc cache is empty, but guarantees zero count */
520
518
if (unlikely(pool->alloc.count > 0))
···
523
521
/* Mark empty alloc.cache slots "empty" for alloc_pages_bulk_array */
524
522
memset(&pool->alloc.cache, 0, sizeof(void *) * bulk);
525
523
526
-
nr_pages = alloc_pages_bulk_array_node(gfp, pool->p.nid, bulk,
527
-
pool->alloc.cache);
524
+
nr_pages = alloc_pages_bulk_array_node(gfp,
525
+
pool->p.nid, bulk,
526
+
(struct page **)pool->alloc.cache);
528
527
if (unlikely(!nr_pages))
529
-
return NULL;
528
+
return 0;
530
529
531
530
/* Pages have been filled into alloc.cache array, but count is zero and
532
531
* page element have not been (possibly) DMA mapped.
533
532
*/
534
533
for (i = 0; i < nr_pages; i++) {
535
-
page = pool->alloc.cache[i];
536
-
if (dma_map && unlikely(!page_pool_dma_map(pool, page))) {
537
-
put_page(page);
534
+
netmem = pool->alloc.cache[i];
535
+
if (dma_map && unlikely(!page_pool_dma_map(pool, netmem))) {
536
+
put_page(netmem_to_page(netmem));
538
537
continue;
539
538
}
540
539
541
-
page_pool_set_pp_info(pool, page);
542
-
pool->alloc.cache[pool->alloc.count++] = page;
540
+
page_pool_set_pp_info(pool, netmem);
541
+
pool->alloc.cache[pool->alloc.count++] = netmem;
543
542
/* Track how many pages are held 'in-flight' */
544
543
pool->pages_state_hold_cnt++;
545
-
trace_page_pool_state_hold(pool, page,
544
+
trace_page_pool_state_hold(pool, netmem,
546
545
pool->pages_state_hold_cnt);
547
546
}
548
547
549
548
/* Return last page */
550
549
if (likely(pool->alloc.count > 0)) {
551
-
page = pool->alloc.cache[--pool->alloc.count];
550
+
netmem = pool->alloc.cache[--pool->alloc.count];
552
551
alloc_stat_inc(pool, slow);
553
552
} else {
554
-
page = NULL;
553
+
netmem = 0;
555
554
}
556
555
557
556
/* When page just alloc'ed is should/must have refcnt 1. */
558
-
return page;
557
+
return netmem;
559
558
}
560
559
561
560
/* For using page_pool replace: alloc_pages() API calls, but provide
562
561
* synchronization guarantee for allocation side.
563
562
*/
564
-
struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)
563
+
netmem_ref page_pool_alloc_netmem(struct page_pool *pool, gfp_t gfp)
565
564
{
566
-
struct page *page;
565
+
netmem_ref netmem;
567
566
568
567
/* Fast-path: Get a page from cache */
569
-
page = __page_pool_get_cached(pool);
570
-
if (page)
571
-
return page;
568
+
netmem = __page_pool_get_cached(pool);
569
+
if (netmem)
570
+
return netmem;
572
571
573
572
/* Slow-path: cache empty, do real allocation */
574
-
page = __page_pool_alloc_pages_slow(pool, gfp);
575
-
return page;
573
+
netmem = __page_pool_alloc_pages_slow(pool, gfp);
574
+
return netmem;
575
+
}
576
+
EXPORT_SYMBOL(page_pool_alloc_netmem);
577
+
578
+
struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)
579
+
{
580
+
return netmem_to_page(page_pool_alloc_netmem(pool, gfp));
576
581
}
577
582
EXPORT_SYMBOL(page_pool_alloc_pages);
578
583
ALLOW_ERROR_INJECTION(page_pool_alloc_pages, NULL);
···
608
599
return inflight;
609
600
}
610
601
611
-
static __always_inline
612
-
void __page_pool_release_page_dma(struct page_pool *pool, struct page *page)
602
+
static __always_inline void __page_pool_release_page_dma(struct page_pool *pool,
603
+
netmem_ref netmem)
613
604
{
614
605
dma_addr_t dma;
615
606
···
619
610
*/
620
611
return;
621
612
622
-
dma = page_pool_get_dma_addr(page);
613
+
dma = page_pool_get_dma_addr_netmem(netmem);
623
614
624
615
/* When page is unmapped, it cannot be returned to our pool */
625
616
dma_unmap_page_attrs(pool->p.dev, dma,
626
617
PAGE_SIZE << pool->p.order, pool->p.dma_dir,
627
618
DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING);
628
-
page_pool_set_dma_addr(page, 0);
619
+
page_pool_set_dma_addr_netmem(netmem, 0);
629
620
}
630
621
631
622
/* Disconnects a page (from a page_pool). API users can have a need
···
633
624
* a regular page (that will eventually be returned to the normal
634
625
* page-allocator via put_page).
635
626
*/
636
-
void page_pool_return_page(struct page_pool *pool, struct page *page)
627
+
void page_pool_return_page(struct page_pool *pool, netmem_ref netmem)
637
628
{
638
629
int count;
639
630
640
-
__page_pool_release_page_dma(pool, page);
641
-
642
-
page_pool_clear_pp_info(page);
631
+
__page_pool_release_page_dma(pool, netmem);
643
632
644
633
/* This may be the last page returned, releasing the pool, so
645
634
* it is not safe to reference pool afterwards.
646
635
*/
647
636
count = atomic_inc_return_relaxed(&pool->pages_state_release_cnt);
648
-
trace_page_pool_state_release(pool, page, count);
637
+
trace_page_pool_state_release(pool, netmem, count);
649
638
650
-
put_page(page);
639
+
page_pool_clear_pp_info(netmem);
640
+
put_page(netmem_to_page(netmem));
651
641
/* An optimization would be to call __free_pages(page, pool->p.order)
652
642
* knowing page is not part of page-cache (thus avoiding a
653
643
* __page_cache_release() call).
654
644
*/
655
645
}
656
646
657
-
static bool page_pool_recycle_in_ring(struct page_pool *pool, struct page *page)
647
+
static bool page_pool_recycle_in_ring(struct page_pool *pool, netmem_ref netmem)
658
648
{
659
649
int ret;
660
650
/* BH protection not needed if current is softirq */
661
651
if (in_softirq())
662
-
ret = ptr_ring_produce(&pool->ring, page);
652
+
ret = ptr_ring_produce(&pool->ring, (__force void *)netmem);
663
653
else
664
-
ret = ptr_ring_produce_bh(&pool->ring, page);
654
+
ret = ptr_ring_produce_bh(&pool->ring, (__force void *)netmem);
665
655
666
656
if (!ret) {
667
657
recycle_stat_inc(pool, ring);
···
675
667
*
676
668
* Caller must provide appropriate safe context.
677
669
*/
678
-
static bool page_pool_recycle_in_cache(struct page *page,
670
+
static bool page_pool_recycle_in_cache(netmem_ref netmem,
679
671
struct page_pool *pool)
680
672
{
681
673
if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE)) {
···
684
676
}
685
677
686
678
/* Caller MUST have verified/know (page_ref_count(page) == 1) */
687
-
pool->alloc.cache[pool->alloc.count++] = page;
679
+
pool->alloc.cache[pool->alloc.count++] = netmem;
688
680
recycle_stat_inc(pool, cached);
689
681
return true;
690
682
}
691
683
692
-
static bool __page_pool_page_can_be_recycled(const struct page *page)
684
+
static bool __page_pool_page_can_be_recycled(netmem_ref netmem)
693
685
{
694
-
return page_ref_count(page) == 1 && !page_is_pfmemalloc(page);
686
+
return page_ref_count(netmem_to_page(netmem)) == 1 &&
687
+
!page_is_pfmemalloc(netmem_to_page(netmem));
695
688
}
696
689
697
690
/* If the page refcnt == 1, this will try to recycle the page.
···
701
692
* If the page refcnt != 1, then the page will be returned to memory
702
693
* subsystem.
703
694
*/
704
-
static __always_inline struct page *
705
-
__page_pool_put_page(struct page_pool *pool, struct page *page,
695
+
static __always_inline netmem_ref
696
+
__page_pool_put_page(struct page_pool *pool, netmem_ref netmem,
706
697
unsigned int dma_sync_size, bool allow_direct)
707
698
{
708
699
lockdep_assert_no_hardirq();
···
716
707
* page is NOT reusable when allocated when system is under
717
708
* some pressure. (page_is_pfmemalloc)
718
709
*/
719
-
if (likely(__page_pool_page_can_be_recycled(page))) {
710
+
if (likely(__page_pool_page_can_be_recycled(netmem))) {
720
711
/* Read barrier done in page_ref_count / READ_ONCE */
721
712
722
-
page_pool_dma_sync_for_device(pool, page, dma_sync_size);
713
+
page_pool_dma_sync_for_device(pool, netmem, dma_sync_size);
723
714
724
-
if (allow_direct && page_pool_recycle_in_cache(page, pool))
725
-
return NULL;
715
+
if (allow_direct && page_pool_recycle_in_cache(netmem, pool))
716
+
return 0;
726
717
727
718
/* Page found as candidate for recycling */
728
-
return page;
719
+
return netmem;
729
720
}
730
721
/* Fallback/non-XDP mode: API user have elevated refcnt.
731
722
*
···
741
732
* will be invoking put_page.
742
733
*/
743
734
recycle_stat_inc(pool, released_refcnt);
744
-
page_pool_return_page(pool, page);
735
+
page_pool_return_page(pool, netmem);
745
736
746
-
return NULL;
737
+
return 0;
747
738
}
748
739
749
740
static bool page_pool_napi_local(const struct page_pool *pool)
···
769
760
return napi && READ_ONCE(napi->list_owner) == cpuid;
770
761
}
771
762
772
-
void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
773
-
unsigned int dma_sync_size, bool allow_direct)
763
+
void page_pool_put_unrefed_netmem(struct page_pool *pool, netmem_ref netmem,
764
+
unsigned int dma_sync_size, bool allow_direct)
774
765
{
775
766
if (!allow_direct)
776
767
allow_direct = page_pool_napi_local(pool);
777
768
778
-
page = __page_pool_put_page(pool, page, dma_sync_size, allow_direct);
779
-
if (page && !page_pool_recycle_in_ring(pool, page)) {
769
+
netmem =
770
+
__page_pool_put_page(pool, netmem, dma_sync_size, allow_direct);
771
+
if (netmem && !page_pool_recycle_in_ring(pool, netmem)) {
780
772
/* Cache full, fallback to free pages */
781
773
recycle_stat_inc(pool, ring_full);
782
-
page_pool_return_page(pool, page);
774
+
page_pool_return_page(pool, netmem);
783
775
}
776
+
}
777
+
EXPORT_SYMBOL(page_pool_put_unrefed_netmem);
778
+
779
+
void page_pool_put_unrefed_page(struct page_pool *pool, struct page *page,
780
+
unsigned int dma_sync_size, bool allow_direct)
781
+
{
782
+
page_pool_put_unrefed_netmem(pool, page_to_netmem(page), dma_sync_size,
783
+
allow_direct);
784
784
}
785
785
EXPORT_SYMBOL(page_pool_put_unrefed_page);
786
786
···
818
800
allow_direct = page_pool_napi_local(pool);
819
801
820
802
for (i = 0; i < count; i++) {
821
-
struct page *page = virt_to_head_page(data[i]);
803
+
netmem_ref netmem = page_to_netmem(virt_to_head_page(data[i]));
822
804
823
805
/* It is not the last user for the page frag case */
824
-
if (!page_pool_is_last_ref(page))
806
+
if (!page_pool_is_last_ref(netmem))
825
807
continue;
826
808
827
-
page = __page_pool_put_page(pool, page, -1, allow_direct);
809
+
netmem = __page_pool_put_page(pool, netmem, -1, allow_direct);
828
810
/* Approved for bulk recycling in ptr_ring cache */
829
-
if (page)
830
-
data[bulk_len++] = page;
811
+
if (netmem)
812
+
data[bulk_len++] = (__force void *)netmem;
831
813
}
832
814
833
815
if (!bulk_len)
···
853
835
* since put_page() with refcnt == 1 can be an expensive operation
854
836
*/
855
837
for (; i < bulk_len; i++)
856
-
page_pool_return_page(pool, data[i]);
838
+
page_pool_return_page(pool, (__force netmem_ref)data[i]);
857
839
}
858
840
EXPORT_SYMBOL(page_pool_put_page_bulk);
859
841
860
-
static struct page *page_pool_drain_frag(struct page_pool *pool,
861
-
struct page *page)
842
+
static netmem_ref page_pool_drain_frag(struct page_pool *pool,
843
+
netmem_ref netmem)
862
844
{
863
845
long drain_count = BIAS_MAX - pool->frag_users;
864
846
865
847
/* Some user is still using the page frag */
866
-
if (likely(page_pool_unref_page(page, drain_count)))
867
-
return NULL;
848
+
if (likely(page_pool_unref_netmem(netmem, drain_count)))
849
+
return 0;
868
850
869
-
if (__page_pool_page_can_be_recycled(page)) {
870
-
page_pool_dma_sync_for_device(pool, page, -1);
871
-
return page;
851
+
if (__page_pool_page_can_be_recycled(netmem)) {
852
+
page_pool_dma_sync_for_device(pool, netmem, -1);
853
+
return netmem;
872
854
}
873
855
874
-
page_pool_return_page(pool, page);
875
-
return NULL;
856
+
page_pool_return_page(pool, netmem);
857
+
return 0;
876
858
}
877
859
878
860
static void page_pool_free_frag(struct page_pool *pool)
879
861
{
880
862
long drain_count = BIAS_MAX - pool->frag_users;
881
-
struct page *page = pool->frag_page;
863
+
netmem_ref netmem = pool->frag_page;
882
864
883
-
pool->frag_page = NULL;
865
+
pool->frag_page = 0;
884
866
885
-
if (!page || page_pool_unref_page(page, drain_count))
867
+
if (!netmem || page_pool_unref_netmem(netmem, drain_count))
886
868
return;
887
869
888
-
page_pool_return_page(pool, page);
870
+
page_pool_return_page(pool, netmem);
889
871
}
890
872
891
-
struct page *page_pool_alloc_frag(struct page_pool *pool,
892
-
unsigned int *offset,
893
-
unsigned int size, gfp_t gfp)
873
+
netmem_ref page_pool_alloc_frag_netmem(struct page_pool *pool,
874
+
unsigned int *offset, unsigned int size,
875
+
gfp_t gfp)
894
876
{
895
877
unsigned int max_size = PAGE_SIZE << pool->p.order;
896
-
struct page *page = pool->frag_page;
878
+
netmem_ref netmem = pool->frag_page;
897
879
898
880
if (WARN_ON(size > max_size))
899
-
return NULL;
881
+
return 0;
900
882
901
883
size = ALIGN(size, dma_get_cache_alignment());
902
884
*offset = pool->frag_offset;
903
885
904
-
if (page && *offset + size > max_size) {
905
-
page = page_pool_drain_frag(pool, page);
906
-
if (page) {
886
+
if (netmem && *offset + size > max_size) {
887
+
netmem = page_pool_drain_frag(pool, netmem);
888
+
if (netmem) {
907
889
alloc_stat_inc(pool, fast);
908
890
goto frag_reset;
909
891
}
910
892
}
911
893
912
-
if (!page) {
913
-
page = page_pool_alloc_pages(pool, gfp);
914
-
if (unlikely(!page)) {
915
-
pool->frag_page = NULL;
916
-
return NULL;
894
+
if (!netmem) {
895
+
netmem = page_pool_alloc_netmem(pool, gfp);
896
+
if (unlikely(!netmem)) {
897
+
pool->frag_page = 0;
898
+
return 0;
917
899
}
918
900
919
-
pool->frag_page = page;
901
+
pool->frag_page = netmem;
920
902
921
903
frag_reset:
922
904
pool->frag_users = 1;
923
905
*offset = 0;
924
906
pool->frag_offset = size;
925
-
page_pool_fragment_page(page, BIAS_MAX);
926
-
return page;
907
+
page_pool_fragment_netmem(netmem, BIAS_MAX);
908
+
return netmem;
927
909
}
928
910
929
911
pool->frag_users++;
930
912
pool->frag_offset = *offset + size;
931
913
alloc_stat_inc(pool, fast);
932
-
return page;
914
+
return netmem;
915
+
}
916
+
EXPORT_SYMBOL(page_pool_alloc_frag_netmem);
917
+
918
+
struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
919
+
unsigned int size, gfp_t gfp)
920
+
{
921
+
return netmem_to_page(page_pool_alloc_frag_netmem(pool, offset, size,
922
+
gfp));
933
923
}
934
924
EXPORT_SYMBOL(page_pool_alloc_frag);
935
925
936
926
static void page_pool_empty_ring(struct page_pool *pool)
937
927
{
938
-
struct page *page;
928
+
netmem_ref netmem;
939
929
940
930
/* Empty recycle ring */
941
-
while ((page = ptr_ring_consume_bh(&pool->ring))) {
931
+
while ((netmem = (__force netmem_ref)ptr_ring_consume_bh(&pool->ring))) {
942
932
/* Verify the refcnt invariant of cached pages */
943
-
if (!(page_ref_count(page) == 1))
933
+
if (!(page_ref_count(netmem_to_page(netmem)) == 1))
944
934
pr_crit("%s() page_pool refcnt %d violation\n",
945
-
__func__, page_ref_count(page));
935
+
__func__, netmem_ref_count(netmem));
946
936
947
-
page_pool_return_page(pool, page);
937
+
page_pool_return_page(pool, netmem);
948
938
}
949
939
}
950
940
···
968
942
969
943
static void page_pool_empty_alloc_cache_once(struct page_pool *pool)
970
944
{
971
-
struct page *page;
945
+
netmem_ref netmem;
972
946
973
947
if (pool->destroy_cnt)
974
948
return;
···
978
952
* call concurrently.
979
953
*/
980
954
while (pool->alloc.count) {
981
-
page = pool->alloc.cache[--pool->alloc.count];
982
-
page_pool_return_page(pool, page);
955
+
netmem = pool->alloc.cache[--pool->alloc.count];
956
+
page_pool_return_page(pool, netmem);
983
957
}
984
958
}
985
959
···
1086
1060
/* Caller must provide appropriate safe context, e.g. NAPI. */
1087
1061
void page_pool_update_nid(struct page_pool *pool, int new_nid)
1088
1062
{
1089
-
struct page *page;
1063
+
netmem_ref netmem;
1090
1064
1091
1065
trace_page_pool_update_nid(pool, new_nid);
1092
1066
pool->p.nid = new_nid;
1093
1067
1094
1068
/* Flush pool alloc cache, as refill will check NUMA node */
1095
1069
while (pool->alloc.count) {
1096
-
page = pool->alloc.cache[--pool->alloc.count];
1097
-
page_pool_return_page(pool, page);
1070
+
netmem = pool->alloc.cache[--pool->alloc.count];
1071
+
page_pool_return_page(pool, netmem);
1098
1072
}
1099
1073
}
1100
1074
EXPORT_SYMBOL(page_pool_update_nid);
+5
-3
net/core/skbuff.c
+5
-3
net/core/skbuff.c
···
1015
1015
EXPORT_SYMBOL(skb_cow_data_for_xdp);
1016
1016
1017
1017
#if IS_ENABLED(CONFIG_PAGE_POOL)
1018
-
bool napi_pp_put_page(struct page *page)
1018
+
bool napi_pp_put_page(netmem_ref netmem)
1019
1019
{
1020
+
struct page *page = netmem_to_page(netmem);
1021
+
1020
1022
page = compound_head(page);
1021
1023
1022
1024
/* page->pp_magic is OR'ed with PP_SIGNATURE after the allocation
···
1031
1029
if (unlikely(!is_pp_page(page)))
1032
1030
return false;
1033
1031
1034
-
page_pool_put_full_page(page->pp, page, false);
1032
+
page_pool_put_full_netmem(page->pp, page_to_netmem(page), false);
1035
1033
1036
1034
return true;
1037
1035
}
···
1042
1040
{
1043
1041
if (!IS_ENABLED(CONFIG_PAGE_POOL) || !skb->pp_recycle)
1044
1042
return false;
1045
-
return napi_pp_put_page(virt_to_page(data));
1043
+
return napi_pp_put_page(page_to_netmem(virt_to_page(data)));
1046
1044
}
1047
1045
1048
1046
/**
+5
-1
tools/net/ynl/Makefile
+5
-1
tools/net/ynl/Makefile
···
2
2
3
3
SUBDIRS = lib generated samples
4
4
5
-
all: $(SUBDIRS)
5
+
all: $(SUBDIRS) libynl.a
6
6
7
7
samples: | lib generated
8
+
libynl.a: | lib generated
9
+
@echo -e "\tAR $@"
10
+
@ar rcs $@ lib/ynl.o generated/*-user.o
8
11
9
12
$(SUBDIRS):
10
13
@if [ -f "$@/Makefile" ] ; then \
···
20
17
$(MAKE) -C $$dir $@; \
21
18
fi \
22
19
done
20
+
rm -f libynl.a
23
21
24
22
.PHONY: all clean distclean $(SUBDIRS)
+3
-1
tools/net/ynl/lib/Makefile
+3
-1
tools/net/ynl/lib/Makefile
+21
tools/testing/selftests/net/ynl.mk
+21
tools/testing/selftests/net/ynl.mk
···
1
+
# SPDX-License-Identifier: GPL-2.0
2
+
3
+
# YNL selftest build snippet
4
+
5
+
# Inputs:
6
+
#
7
+
# YNL_GENS: families we need in the selftests
8
+
# YNL_PROGS: TEST_PROGS which need YNL (TODO, none exist, yet)
9
+
# YNL_GEN_FILES: TEST_GEN_FILES which need YNL
10
+
11
+
YNL_OUTPUTS := $(patsubst %,$(OUTPUT)/%,$(YNL_GEN_FILES))
12
+
13
+
$(YNL_OUTPUTS): $(OUTPUT)/libynl.a
14
+
$(YNL_OUTPUTS): CFLAGS += \
15
+
-I$(top_srcdir)/usr/include/ $(KHDR_INCLUDES) \
16
+
-I$(top_srcdir)/tools/net/ynl/lib/ \
17
+
-I$(top_srcdir)/tools/net/ynl/generated/
18
+
19
+
$(OUTPUT)/libynl.a:
20
+
$(Q)$(MAKE) -C $(top_srcdir)/tools/net/ynl GENS="$(YNL_GENS)" libynl.a
21
+
$(Q)cp $(top_srcdir)/tools/net/ynl/libynl.a $(OUTPUT)/libynl.a